Add a script for cross building python

build-python.sh

@@ -0,0 +1,161 @@
+#!/bin/sh
+#
+# Copyright (c) 2018 Martin Storsjo
+#
+# Permission to use, copy, modify, and/or distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+set -e
+
+: ${LIBFFI_VERSION:=v3.4.2}
+: ${PYTHON_MAJOR:=3}
+: ${PYTHON_MINOR:=9}
+: ${PYTHON_PATCH:=6}
+: ${PYTHON_VERSION:=v${PYTHON_MAJOR}.${PYTHON_MINOR}.${PYTHON_PATCH}}
+: ${PYTHON_VERSION_MINGW:=61c5a4a14932539b31878de0b60bc98bcba791b9}
+
+unset HOST
+
+BUILDDIR=build
+
+while [ $# -gt 0 ]; do
+    case "$1" in
+    --host=*)
+        HOST="${1#*=}"
+        BUILDDIR=$BUILDDIR-$HOST
+        ;;
+    *)
+        PREFIX="$1"
+        ;;
+    esac
+    shift
+done
+
+if [ -z "$CHECKOUT_ONLY" ]; then
+    if [ -z "$PREFIX" ]; then
+        echo $0 --host=triple dest
+        exit 1
+    fi
+
+    mkdir -p "$PREFIX"
+    PREFIX="$(cd "$PREFIX" && pwd)"
+fi
+
+MAKE=make
+if [ -n "$(which gmake)" ]; then
+    MAKE=gmake
+fi
+
+: ${CORES:=$(nproc 2>/dev/null)}
+: ${CORES:=$(sysctl -n hw.ncpu 2>/dev/null)}
+: ${CORES:=4}
+
+if [ ! -d libffi ]; then
+    git clone https://github.com/libffi/libffi.git
+    CHECKOUT_LIBFFI=1
+fi
+
+if [ -n "$SYNC" ] || [ -n "$CHECKOUT_LIBFFI" ]; then
+    cd libffi
+    [ -z "$SYNC" ] || git fetch
+    git checkout $LIBFFI_VERSION
+    autoreconf -vfi
+    cd ..
+fi
+
+if [ -z "$HOST" ]; then
+    # Use a separate checkout for python for the native build;
+    # mingw builds use a separate fork, maintained by msys2
+    # which doesn't build on regular Unix
+    if [ ! -d cpython-native ]; then
+        git clone https://github.com/python/cpython.git cpython-native
+        CHECKOUT_PYTHON_NATIVE=1
+    fi
+
+    if [ -n "$SYNC" ] || [ -n "$CHECKOUT_PYTHON_NATIVE" ]; then
+        cd cpython-native
+        [ -z "$SYNC" ] || git fetch
+        git checkout $PYTHON_VERSION
+        cd ..
+    fi
+
+    [ -z "$CHECKOUT_ONLY" ] || exit 0
+
+    cd libffi
+    [ -z "$CLEAN" ] || rm -rf $BUILDDIR
+    mkdir -p $BUILDDIR
+    cd $BUILDDIR
+    ../configure --prefix="$PREFIX" --disable-symvers --disable-docs
+    $MAKE -j$CORES
+    $MAKE install
+    cd ../..
+
+    cd cpython-native
+    [ -z "$CLEAN" ] || rm -rf $BUILDDIR
+    mkdir -p $BUILDDIR
+    cd $BUILDDIR
+    ../configure --prefix="$PREFIX" \
+        CFLAGS="-I$PREFIX/include" CXXFLAGS="-I$PREFIX/include" LDFLAGS="-L$PREFIX/lib -Wl,-s" \
+        --without-ensurepip
+    $MAKE -j$CORES
+    $MAKE install
+    exit 0
+fi
+
+# Fetching
+if [ ! -d cpython-mingw ]; then
+    git clone https://github.com/msys2-contrib/cpython-mingw.git
+    CHECKOUT_PYTHON=1
+fi
+
+if [ -n "$SYNC" ] || [ -n "$CHECKOUT_PYTHON" ]; then
+    cd cpython-mingw
+    [ -z "$SYNC" ] || git fetch
+    git checkout $PYTHON_VERSION_MINGW
+    cat ../patches/python/*.patch | patch -Nup1
+    autoreconf -vfi
+    cd ..
+fi
+
+[ -z "$CHECKOUT_ONLY" ] || exit 0
+
+cd libffi
+[ -z "$CLEAN" ] || rm -rf $BUILDDIR
+mkdir -p $BUILDDIR
+cd $BUILDDIR
+../configure --prefix="$PREFIX" --host=$HOST --disable-symvers --disable-docs
+$MAKE -j$CORES
+$MAKE install
+cd ../..
+
+cd cpython-mingw
+[ -z "$CLEAN" ] || rm -rf $BUILDDIR
+mkdir -p $BUILDDIR
+cd $BUILDDIR
+BUILD=$(../config.guess) # Python configure requires build triplet for cross compilation
+
+# Avoid gcc workarounds in distutils
+export CC=$HOST-clang
+export CXX=$HOST-clang++
+
+../configure --prefix="$PREFIX" --build=$BUILD --host=$HOST \
+    CFLAGS="-I$PREFIX/include" CXXFLAGS="-I$PREFIX/include" LDFLAGS="-L$PREFIX/lib -Wl,-s" \
+    --enable-shared             \
+    --with-system-ffi           \
+    --without-ensurepip         \
+    --without-c-locale-coercion
+
+$MAKE -j$CORES
+$MAKE install
+rm -rf $PREFIX/lib/python*/test
+find $PREFIX/lib/python* -name __pycache__ | xargs rm -rf
+cd ../..

patches/python/distutils-clang-mingw.patch

@@ -0,0 +1,155 @@
+diff --git a/Lib/distutils/cygwinccompiler.py b/Lib/distutils/cygwinccompiler.py
+index 66c12dd..2f76fb2 100644
+--- a/Lib/distutils/cygwinccompiler.py
++++ b/Lib/distutils/cygwinccompiler.py
+@@ -44,6 +44,8 @@ cygwin in no-cygwin mode).
+ #   (ld supports -shared)
+ # * mingw gcc 3.2/ld 2.13 works
+ #   (ld supports -shared)
++# * llvm-mingw with Clang 11 works
++#   (lld supports -shared)
+ 
+ import os
+ import sys
+@@ -109,41 +111,46 @@ class CygwinCCompiler(UnixCCompiler):
+                 "Compiling may fail because of undefined preprocessor macros."
+                 % details)
+ 
+-        self.gcc_version, self.ld_version, self.dllwrap_version = \
+-            get_versions()
+-        self.debug_print(self.compiler_type + ": gcc %s, ld %s, dllwrap %s\n" %
+-                         (self.gcc_version,
+-                          self.ld_version,
+-                          self.dllwrap_version) )
+-
+-        # ld_version >= "2.10.90" and < "2.13" should also be able to use
+-        # gcc -mdll instead of dllwrap
+-        # Older dllwraps had own version numbers, newer ones use the
+-        # same as the rest of binutils ( also ld )
+-        # dllwrap 2.10.90 is buggy
+-        if self.ld_version >= "2.10.90":
+-            self.linker_dll = "gcc"
+-        else:
+-            self.linker_dll = "dllwrap"
++        self.cc = os.environ.get('CC', 'gcc')
++        self.cxx = os.environ.get('CXX', 'g++')
++
++        if ('gcc' in self.cc): # Start gcc workaround
++            self.gcc_version, self.ld_version, self.dllwrap_version = \
++                get_gcc_versions()
++            self.debug_print(self.compiler_type + ": gcc %s, ld %s, dllwrap %s\n" %
++                             (self.gcc_version,
++                              self.ld_version,
++                              self.dllwrap_version) )
++
++            # ld_version >= "2.10.90" and < "2.13" should also be able to use
++            # gcc -mdll instead of dllwrap
++            # Older dllwraps had own version numbers, newer ones use the
++            # same as the rest of binutils ( also ld )
++            # dllwrap 2.10.90 is buggy
++            if self.ld_version >= "2.10.90":
++                self.linker_dll = self.cc
++            else:
++                self.linker_dll = "dllwrap"
+ 
+-        # ld_version >= "2.13" support -shared so use it instead of
+-        # -mdll -static
+-        if self.ld_version >= "2.13":
++            # ld_version >= "2.13" support -shared so use it instead of
++            # -mdll -static
++            if self.ld_version >= "2.13":
++                shared_option = "-shared"
++            else:
++                shared_option = "-mdll -static"
++        else: # Assume linker is up to date
++            self.linker_dll = self.cc
+             shared_option = "-shared"
+-        else:
+-            shared_option = "-mdll -static"
+ 
+-        # Hard-code GCC because that's what this is all about.
+-        # XXX optimization, warnings etc. should be customizable.
+-        self.set_executables(compiler='gcc -mcygwin -O -Wall',
+-                             compiler_so='gcc -mcygwin -mdll -O -Wall',
+-                             compiler_cxx='g++ -mcygwin -O -Wall',
+-                             linker_exe='gcc -mcygwin',
++        self.set_executables(compiler='%s -mcygwin -O -Wall' % self.cc,
++                             compiler_so='%s -mcygwin -mdll -O -Wall' % self.cc,
++                             compiler_cxx='%s -mcygwin -O -Wall' % self.cxx,
++                             linker_exe='%s -mcygwin' % self.cc,
+                              linker_so=('%s -mcygwin %s' %
+                                         (self.linker_dll, shared_option)))
+ 
+         # cygwin and mingw32 need different sets of libraries
+-        if self.gcc_version == "2.91.57":
++        if ('gcc' in self.cc and self.gcc_version == "2.91.57"):
+             # cygwin shouldn't need msvcrt, but without the dlls will crash
+             # (gcc version 2.91.57) -- perhaps something about initialization
+             self.dll_libraries=["msvcrt"]
+@@ -281,26 +288,26 @@ class Mingw32CCompiler(CygwinCCompiler):
+ 
+         # ld_version >= "2.13" support -shared so use it instead of
+         # -mdll -static
+-        if self.ld_version >= "2.13":
+-            shared_option = "-shared"
+-        else:
++        if ('gcc' in self.cc and self.ld_version < "2.13"):
+             shared_option = "-mdll -static"
++        else:
++            shared_option = "-shared"
+ 
+         # A real mingw32 doesn't need to specify a different entry point,
+         # but cygwin 2.91.57 in no-cygwin-mode needs it.
+-        if self.gcc_version <= "2.91.57":
++        if ('gcc' in self.cc and self.gcc_version <= "2.91.57"):
+             entry_point = '--entry _DllMain@12'
+         else:
+             entry_point = ''
+ 
+-        if is_cygwingcc():
++        if is_cygwincc(self.cc):
+             raise CCompilerError(
+                 'Cygwin gcc cannot be used with --compiler=mingw32')
+ 
+-        self.set_executables(compiler='gcc -O2 -Wall',
+-                             compiler_so='gcc -mdll -O2 -Wall',
+-                             compiler_cxx='g++ -O2 -Wall',
+-                             linker_exe='gcc',
++        self.set_executables(compiler='%s -O2 -Wall' % self.cc,
++                             compiler_so='%s -mdll -O2 -Wall' % self.cc,
++                             compiler_cxx='%s -O2 -Wall' % self.cxx,
++                             linker_exe='%s' % self.cc,
+                              linker_so='%s %s %s'
+                                         % (self.linker_dll, shared_option,
+                                            entry_point))
+@@ -351,6 +358,10 @@ def check_config_h():
+     if "GCC" in sys.version:
+         return CONFIG_H_OK, "sys.version mentions 'GCC'"
+ 
++    # Clang would also work
++    if "Clang" in sys.version:
++        return CONFIG_H_OK, "sys.version mentions 'Clang'"
++
+     # let's see if __GNUC__ is mentioned in python.h
+     fn = sysconfig.get_config_h_filename()
+     try:
+@@ -389,7 +400,7 @@ def _find_exe_version(cmd):
+     # so we need to decode our bytes
+     return LooseVersion(result.group(1).decode())
+ 
+-def get_versions():
++def get_gcc_versions():
+     """ Try to find out the versions of gcc, ld and dllwrap.
+ 
+     If not possible it returns None for it.
+@@ -397,7 +408,7 @@ def get_versions():
+     commands = [gcc+' -dumpversion', ld+' -v', 'dllwrap --version']
+     return tuple([_find_exe_version(cmd) for cmd in commands])
+ 
+-def is_cygwingcc():
+-    '''Try to determine if the gcc that would be used is from cygwin.'''
+-    out_string = check_output(['gcc', '-dumpmachine'])
++def is_cygwincc(cc):
++    '''Try to determine if the compiler that would be used is from cygwin.'''
++    out_string = check_output([cc, '-dumpmachine'])
+     return out_string.strip().endswith(b'cygwin')

patches/python/fix-python-build-detection.patch

@@ -0,0 +1,13 @@
+diff --git a/Lib/distutils/command/build_ext.py b/Lib/distutils/command/build_ext.py
+index dbcd9d1..274b1d8 100644
+--- a/Lib/distutils/command/build_ext.py
++++ b/Lib/distutils/command/build_ext.py
+@@ -219,7 +219,7 @@ class build_ext(Command):
+         # For extensions under Cygwin, Python's library directory must be
+         # appended to library_dirs
+         if sys.platform[:6] == 'cygwin' or self.plat_name.startswith(('mingw')):
+-            if sys.executable.startswith(os.path.join(sys.exec_prefix, "bin")):
++            if not sysconfig.python_build:
+                 # building third party extensions
+                 self.library_dirs.append(os.path.join(sys.prefix, "lib",
+                                                       "python" + get_python_version(),