From 8ff26ac253f0d408b73c4f4adbe2e19040e56e23 Mon Sep 17 00:00:00 2001 From: Long Nguyen Date: Mon, 8 Feb 2021 11:21:22 +0700 Subject: [PATCH] Add a script for cross building python --- build-python.sh | 161 ++++++++++++++++++ patches/python/distutils-clang-mingw.patch | 155 +++++++++++++++++ .../python/fix-python-build-detection.patch | 13 ++ 3 files changed, 329 insertions(+) create mode 100755 build-python.sh create mode 100644 patches/python/distutils-clang-mingw.patch create mode 100644 patches/python/fix-python-build-detection.patch diff --git a/build-python.sh b/build-python.sh new file mode 100755 index 0000000..96ebcbc --- /dev/null +++ b/build-python.sh @@ -0,0 +1,161 @@ +#!/bin/sh +# +# Copyright (c) 2018 Martin Storsjo +# +# Permission to use, copy, modify, and/or distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +set -e + +: ${LIBFFI_VERSION:=v3.4.2} +: ${PYTHON_MAJOR:=3} +: ${PYTHON_MINOR:=9} +: ${PYTHON_PATCH:=6} +: ${PYTHON_VERSION:=v${PYTHON_MAJOR}.${PYTHON_MINOR}.${PYTHON_PATCH}} +: ${PYTHON_VERSION_MINGW:=61c5a4a14932539b31878de0b60bc98bcba791b9} + +unset HOST + +BUILDDIR=build + +while [ $# -gt 0 ]; do + case "$1" in + --host=*) + HOST="${1#*=}" + BUILDDIR=$BUILDDIR-$HOST + ;; + *) + PREFIX="$1" + ;; + esac + shift +done + +if [ -z "$CHECKOUT_ONLY" ]; then + if [ -z "$PREFIX" ]; then + echo $0 --host=triple dest + exit 1 + fi + + mkdir -p "$PREFIX" + PREFIX="$(cd "$PREFIX" && pwd)" +fi + +MAKE=make +if [ -n "$(which gmake)" ]; then + MAKE=gmake +fi + +: ${CORES:=$(nproc 2>/dev/null)} +: ${CORES:=$(sysctl -n hw.ncpu 2>/dev/null)} +: ${CORES:=4} + +if [ ! -d libffi ]; then + git clone https://github.com/libffi/libffi.git + CHECKOUT_LIBFFI=1 +fi + +if [ -n "$SYNC" ] || [ -n "$CHECKOUT_LIBFFI" ]; then + cd libffi + [ -z "$SYNC" ] || git fetch + git checkout $LIBFFI_VERSION + autoreconf -vfi + cd .. +fi + +if [ -z "$HOST" ]; then + # Use a separate checkout for python for the native build; + # mingw builds use a separate fork, maintained by msys2 + # which doesn't build on regular Unix + if [ ! -d cpython-native ]; then + git clone https://github.com/python/cpython.git cpython-native + CHECKOUT_PYTHON_NATIVE=1 + fi + + if [ -n "$SYNC" ] || [ -n "$CHECKOUT_PYTHON_NATIVE" ]; then + cd cpython-native + [ -z "$SYNC" ] || git fetch + git checkout $PYTHON_VERSION + cd .. + fi + + [ -z "$CHECKOUT_ONLY" ] || exit 0 + + cd libffi + [ -z "$CLEAN" ] || rm -rf $BUILDDIR + mkdir -p $BUILDDIR + cd $BUILDDIR + ../configure --prefix="$PREFIX" --disable-symvers --disable-docs + $MAKE -j$CORES + $MAKE install + cd ../.. + + cd cpython-native + [ -z "$CLEAN" ] || rm -rf $BUILDDIR + mkdir -p $BUILDDIR + cd $BUILDDIR + ../configure --prefix="$PREFIX" \ + CFLAGS="-I$PREFIX/include" CXXFLAGS="-I$PREFIX/include" LDFLAGS="-L$PREFIX/lib -Wl,-s" \ + --without-ensurepip + $MAKE -j$CORES + $MAKE install + exit 0 +fi + +# Fetching +if [ ! -d cpython-mingw ]; then + git clone https://github.com/msys2-contrib/cpython-mingw.git + CHECKOUT_PYTHON=1 +fi + +if [ -n "$SYNC" ] || [ -n "$CHECKOUT_PYTHON" ]; then + cd cpython-mingw + [ -z "$SYNC" ] || git fetch + git checkout $PYTHON_VERSION_MINGW + cat ../patches/python/*.patch | patch -Nup1 + autoreconf -vfi + cd .. +fi + +[ -z "$CHECKOUT_ONLY" ] || exit 0 + +cd libffi +[ -z "$CLEAN" ] || rm -rf $BUILDDIR +mkdir -p $BUILDDIR +cd $BUILDDIR +../configure --prefix="$PREFIX" --host=$HOST --disable-symvers --disable-docs +$MAKE -j$CORES +$MAKE install +cd ../.. + +cd cpython-mingw +[ -z "$CLEAN" ] || rm -rf $BUILDDIR +mkdir -p $BUILDDIR +cd $BUILDDIR +BUILD=$(../config.guess) # Python configure requires build triplet for cross compilation + +# Avoid gcc workarounds in distutils +export CC=$HOST-clang +export CXX=$HOST-clang++ + +../configure --prefix="$PREFIX" --build=$BUILD --host=$HOST \ + CFLAGS="-I$PREFIX/include" CXXFLAGS="-I$PREFIX/include" LDFLAGS="-L$PREFIX/lib -Wl,-s" \ + --enable-shared \ + --with-system-ffi \ + --without-ensurepip \ + --without-c-locale-coercion + +$MAKE -j$CORES +$MAKE install +rm -rf $PREFIX/lib/python*/test +find $PREFIX/lib/python* -name __pycache__ | xargs rm -rf +cd ../.. diff --git a/patches/python/distutils-clang-mingw.patch b/patches/python/distutils-clang-mingw.patch new file mode 100644 index 0000000..a41fe47 --- /dev/null +++ b/patches/python/distutils-clang-mingw.patch @@ -0,0 +1,155 @@ +diff --git a/Lib/distutils/cygwinccompiler.py b/Lib/distutils/cygwinccompiler.py +index 66c12dd..2f76fb2 100644 +--- a/Lib/distutils/cygwinccompiler.py ++++ b/Lib/distutils/cygwinccompiler.py +@@ -44,6 +44,8 @@ cygwin in no-cygwin mode). + # (ld supports -shared) + # * mingw gcc 3.2/ld 2.13 works + # (ld supports -shared) ++# * llvm-mingw with Clang 11 works ++# (lld supports -shared) + + import os + import sys +@@ -109,41 +111,46 @@ class CygwinCCompiler(UnixCCompiler): + "Compiling may fail because of undefined preprocessor macros." + % details) + +- self.gcc_version, self.ld_version, self.dllwrap_version = \ +- get_versions() +- self.debug_print(self.compiler_type + ": gcc %s, ld %s, dllwrap %s\n" % +- (self.gcc_version, +- self.ld_version, +- self.dllwrap_version) ) +- +- # ld_version >= "2.10.90" and < "2.13" should also be able to use +- # gcc -mdll instead of dllwrap +- # Older dllwraps had own version numbers, newer ones use the +- # same as the rest of binutils ( also ld ) +- # dllwrap 2.10.90 is buggy +- if self.ld_version >= "2.10.90": +- self.linker_dll = "gcc" +- else: +- self.linker_dll = "dllwrap" ++ self.cc = os.environ.get('CC', 'gcc') ++ self.cxx = os.environ.get('CXX', 'g++') ++ ++ if ('gcc' in self.cc): # Start gcc workaround ++ self.gcc_version, self.ld_version, self.dllwrap_version = \ ++ get_gcc_versions() ++ self.debug_print(self.compiler_type + ": gcc %s, ld %s, dllwrap %s\n" % ++ (self.gcc_version, ++ self.ld_version, ++ self.dllwrap_version) ) ++ ++ # ld_version >= "2.10.90" and < "2.13" should also be able to use ++ # gcc -mdll instead of dllwrap ++ # Older dllwraps had own version numbers, newer ones use the ++ # same as the rest of binutils ( also ld ) ++ # dllwrap 2.10.90 is buggy ++ if self.ld_version >= "2.10.90": ++ self.linker_dll = self.cc ++ else: ++ self.linker_dll = "dllwrap" + +- # ld_version >= "2.13" support -shared so use it instead of +- # -mdll -static +- if self.ld_version >= "2.13": ++ # ld_version >= "2.13" support -shared so use it instead of ++ # -mdll -static ++ if self.ld_version >= "2.13": ++ shared_option = "-shared" ++ else: ++ shared_option = "-mdll -static" ++ else: # Assume linker is up to date ++ self.linker_dll = self.cc + shared_option = "-shared" +- else: +- shared_option = "-mdll -static" + +- # Hard-code GCC because that's what this is all about. +- # XXX optimization, warnings etc. should be customizable. +- self.set_executables(compiler='gcc -mcygwin -O -Wall', +- compiler_so='gcc -mcygwin -mdll -O -Wall', +- compiler_cxx='g++ -mcygwin -O -Wall', +- linker_exe='gcc -mcygwin', ++ self.set_executables(compiler='%s -mcygwin -O -Wall' % self.cc, ++ compiler_so='%s -mcygwin -mdll -O -Wall' % self.cc, ++ compiler_cxx='%s -mcygwin -O -Wall' % self.cxx, ++ linker_exe='%s -mcygwin' % self.cc, + linker_so=('%s -mcygwin %s' % + (self.linker_dll, shared_option))) + + # cygwin and mingw32 need different sets of libraries +- if self.gcc_version == "2.91.57": ++ if ('gcc' in self.cc and self.gcc_version == "2.91.57"): + # cygwin shouldn't need msvcrt, but without the dlls will crash + # (gcc version 2.91.57) -- perhaps something about initialization + self.dll_libraries=["msvcrt"] +@@ -281,26 +288,26 @@ class Mingw32CCompiler(CygwinCCompiler): + + # ld_version >= "2.13" support -shared so use it instead of + # -mdll -static +- if self.ld_version >= "2.13": +- shared_option = "-shared" +- else: ++ if ('gcc' in self.cc and self.ld_version < "2.13"): + shared_option = "-mdll -static" ++ else: ++ shared_option = "-shared" + + # A real mingw32 doesn't need to specify a different entry point, + # but cygwin 2.91.57 in no-cygwin-mode needs it. +- if self.gcc_version <= "2.91.57": ++ if ('gcc' in self.cc and self.gcc_version <= "2.91.57"): + entry_point = '--entry _DllMain@12' + else: + entry_point = '' + +- if is_cygwingcc(): ++ if is_cygwincc(self.cc): + raise CCompilerError( + 'Cygwin gcc cannot be used with --compiler=mingw32') + +- self.set_executables(compiler='gcc -O2 -Wall', +- compiler_so='gcc -mdll -O2 -Wall', +- compiler_cxx='g++ -O2 -Wall', +- linker_exe='gcc', ++ self.set_executables(compiler='%s -O2 -Wall' % self.cc, ++ compiler_so='%s -mdll -O2 -Wall' % self.cc, ++ compiler_cxx='%s -O2 -Wall' % self.cxx, ++ linker_exe='%s' % self.cc, + linker_so='%s %s %s' + % (self.linker_dll, shared_option, + entry_point)) +@@ -351,6 +358,10 @@ def check_config_h(): + if "GCC" in sys.version: + return CONFIG_H_OK, "sys.version mentions 'GCC'" + ++ # Clang would also work ++ if "Clang" in sys.version: ++ return CONFIG_H_OK, "sys.version mentions 'Clang'" ++ + # let's see if __GNUC__ is mentioned in python.h + fn = sysconfig.get_config_h_filename() + try: +@@ -389,7 +400,7 @@ def _find_exe_version(cmd): + # so we need to decode our bytes + return LooseVersion(result.group(1).decode()) + +-def get_versions(): ++def get_gcc_versions(): + """ Try to find out the versions of gcc, ld and dllwrap. + + If not possible it returns None for it. +@@ -397,7 +408,7 @@ def get_versions(): + commands = [gcc+' -dumpversion', ld+' -v', 'dllwrap --version'] + return tuple([_find_exe_version(cmd) for cmd in commands]) + +-def is_cygwingcc(): +- '''Try to determine if the gcc that would be used is from cygwin.''' +- out_string = check_output(['gcc', '-dumpmachine']) ++def is_cygwincc(cc): ++ '''Try to determine if the compiler that would be used is from cygwin.''' ++ out_string = check_output([cc, '-dumpmachine']) + return out_string.strip().endswith(b'cygwin') diff --git a/patches/python/fix-python-build-detection.patch b/patches/python/fix-python-build-detection.patch new file mode 100644 index 0000000..1541e95 --- /dev/null +++ b/patches/python/fix-python-build-detection.patch @@ -0,0 +1,13 @@ +diff --git a/Lib/distutils/command/build_ext.py b/Lib/distutils/command/build_ext.py +index dbcd9d1..274b1d8 100644 +--- a/Lib/distutils/command/build_ext.py ++++ b/Lib/distutils/command/build_ext.py +@@ -219,7 +219,7 @@ class build_ext(Command): + # For extensions under Cygwin, Python's library directory must be + # appended to library_dirs + if sys.platform[:6] == 'cygwin' or self.plat_name.startswith(('mingw')): +- if sys.executable.startswith(os.path.join(sys.exec_prefix, "bin")): ++ if not sysconfig.python_build: + # building third party extensions + self.library_dirs.append(os.path.join(sys.prefix, "lib", + "python" + get_python_version(),