build-mingw-w64: Split out building tools to a separate script

This simplifies build-mingw-w64.sh. When building on native
windows (where we also provide unprefixed tools, we can't have
the yet incomplete compiler toolchain in $PATH when building
the mingw-w64 tools. (Logically, build-mingw-w64-tools belong
with build-llvm and install-wrappers; using the original system
compiler to build tools that are supposed to run on the current
machine, as opposed to the cross target, like when building the
runtimes.)

Dockerfile

@@ -20,12 +20,13 @@ ARG TOOLCHAIN_ARCHS="i686 x86_64 armv7 aarch64"
 ARG DEFAULT_CRT=ucrt
 
 # Build everything that uses the llvm monorepo. We need to build the mingw runtime before the compiler-rt/libunwind/libcxxabi/libcxx runtimes.
-COPY build-llvm.sh strip-llvm.sh install-wrappers.sh build-mingw-w64.sh build-compiler-rt.sh build-mingw-w64-libraries.sh build-libcxx.sh ./
+COPY build-llvm.sh strip-llvm.sh install-wrappers.sh build-mingw-w64.sh build-mingw-w64-tools.sh build-compiler-rt.sh build-mingw-w64-libraries.sh build-libcxx.sh ./
 COPY wrappers/*.sh wrappers/*.c wrappers/*.h ./wrappers/
 RUN ./build-llvm.sh $TOOLCHAIN_PREFIX && \
     ./strip-llvm.sh $TOOLCHAIN_PREFIX && \
     ./install-wrappers.sh $TOOLCHAIN_PREFIX && \
     ./build-mingw-w64.sh $TOOLCHAIN_PREFIX --with-default-msvcrt=$DEFAULT_CRT && \
+    ./build-mingw-w64-tools.sh $TOOLCHAIN_PREFIX && \
     ./build-compiler-rt.sh $TOOLCHAIN_PREFIX && \
     ./build-mingw-w64-libraries.sh $TOOLCHAIN_PREFIX && \
     ./build-libcxx.sh $TOOLCHAIN_PREFIX && \

Dockerfile.cross

@@ -37,8 +37,8 @@ RUN ./strip-llvm.sh $CROSS_TOOLCHAIN_PREFIX --host=$HOST
 
 ARG TOOLCHAIN_ARCHS="i686 x86_64 armv7 aarch64"
 
-COPY build-mingw-w64.sh ./
+COPY build-mingw-w64.sh build-mingw-w64-tools.sh ./
-RUN ./build-mingw-w64.sh $CROSS_TOOLCHAIN_PREFIX --skip-include-triplet-prefix --host=$HOST
+RUN ./build-mingw-w64-tools.sh $CROSS_TOOLCHAIN_PREFIX --skip-include-triplet-prefix --host=$HOST
 
 COPY wrappers/*.sh wrappers/*.c wrappers/*.h ./wrappers/
 COPY install-wrappers.sh .

Dockerfile.dev

@@ -42,6 +42,9 @@ ARG DEFAULT_CRT=ucrt
 COPY build-mingw-w64.sh ./
 RUN ./build-mingw-w64.sh $TOOLCHAIN_PREFIX --with-default-msvcrt=$DEFAULT_CRT
 
+COPY build-mingw-w64-tools.sh ./
+RUN ./build-mingw-w64-tools.sh $TOOLCHAIN_PREFIX
+
 # Build compiler-rt
 COPY build-compiler-rt.sh ./
 RUN ./build-compiler-rt.sh $TOOLCHAIN_PREFIX

build-all.sh

@@ -25,6 +25,7 @@ PREFIX="$1"
 ./build-llvm.sh $PREFIX
 ./install-wrappers.sh $PREFIX
 ./build-mingw-w64.sh $PREFIX
+./build-mingw-w64-tools.sh $PREFIX
 ./build-compiler-rt.sh $PREFIX
 ./build-mingw-w64-libraries.sh $PREFIX
 ./build-libcxx.sh $PREFIX

build-mingw-w64-tools.sh

@@ -0,0 +1,107 @@
+#!/bin/sh
+#
+# Copyright (c) 2018 Martin Storsjo
+#
+# Permission to use, copy, modify, and/or distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+set -e
+
+unset HOST
+
+while [ $# -gt 0 ]; do
+    case "$1" in
+    --skip-include-triplet-prefix)
+        SKIP_INCLUDE_TRIPLET_PREFIX=1
+        ;;
+    --host=*)
+        HOST="${1#*=}"
+        ;;
+    *)
+        PREFIX="$1"
+        ;;
+    esac
+    shift
+done
+if [ -z "$CHECKOUT_ONLY" ]; then
+    if [ -z "$PREFIX" ]; then
+        echo $0 [--skip-include-triplet-prefix] [--host=<triple>] dest
+        exit 1
+    fi
+
+    mkdir -p "$PREFIX"
+    PREFIX="$(cd "$PREFIX" && pwd)"
+fi
+
+if [ ! -d mingw-w64 ] || [ -n "$SYNC" ]; then
+    CHECKOUT_ONLY=1 ./build-mingw-w64.sh
+fi
+
+cd mingw-w64
+
+MAKE=make
+if [ "$(which gmake)" != "" ]; then
+    MAKE=gmake
+fi
+
+: ${CORES:=$(nproc 2>/dev/null)}
+: ${CORES:=$(sysctl -n hw.ncpu 2>/dev/null)}
+: ${CORES:=4}
+: ${ARCHS:=${TOOLCHAIN_ARCHS-i686 x86_64 armv7 aarch64}}
+
+if [ -n "$HOST" ]; then
+    CONFIGFLAGS="$CONFIGFLAGS --host=$HOST"
+    CROSS_NAME=$HOST-
+    EXEEXT=.exe
+else
+    case $(uname) in
+    MINGW*)
+        EXEEXT=.exe
+        ;;
+    *)
+        ;;
+    esac
+fi
+if [ -n "$SKIP_INCLUDE_TRIPLET_PREFIX" ]; then
+    CONFIGFLAGS="$CONFIGFLAGS --with-widl-includedir=$PREFIX/include"
+    # If using the same includedir for all archs, it's enough to
+    # build one single binary.
+    ALL_ARCHS="$ARCHS"
+    ARCHS=x86_64
+fi
+
+cd mingw-w64-tools/widl
+for arch in $ARCHS; do
+    [ -z "$CLEAN" ] || rm -rf build-$CROSS_NAME$arch
+    mkdir -p build-$CROSS_NAME$arch
+    cd build-$CROSS_NAME$arch
+    ../configure --prefix="$PREFIX" --target=$arch-w64-mingw32 $CONFIGFLAGS
+    $MAKE -j$CORES
+    $MAKE install-strip
+    cd ..
+done
+cd "$PREFIX/bin"
+if [ -n "$SKIP_INCLUDE_TRIPLET_PREFIX" ]; then
+    for arch in $ALL_ARCHS; do
+        if [ "$arch" != "$ARCHS" ]; then
+            ln -sf $ARCHS-w64-mingw32-widl$EXEEXT $arch-w64-mingw32-widl$EXEEXT
+        fi
+    done
+fi
+if [ -n "$EXEEXT" ]; then
+    if [ -z "$HOST" ] && [ -f clang$EXEEXT ]; then
+        HOST=$(./clang -dumpmachine | sed 's/-.*//')-w64-mingw32
+    fi
+    if [ -n "$HOST" ]; then
+        ln -sf $HOST-widl$EXEEXT widl$EXEEXT
+    fi
+fi

build-mingw-w64.sh

@@ -19,7 +19,6 @@ set -e
 : ${DEFAULT_WIN32_WINNT:=0x601}
 : ${DEFAULT_MSVCRT:=ucrt}
 : ${MINGW_W64_VERSION:=76e56e6fb70d3a21e054c2c0de8822c108807d28}
-unset HOST
 
 while [ $# -gt 0 ]; do
     case "$1" in
@@ -32,9 +31,6 @@ while [ $# -gt 0 ]; do
     --with-default-msvcrt=*)
         DEFAULT_MSVCRT="${1#*=}"
         ;;
-    --host=*)
-        HOST="${1#*=}"
-        ;;
     *)
         PREFIX="$1"
         ;;
@@ -43,7 +39,7 @@ while [ $# -gt 0 ]; do
 done
 if [ -z "$CHECKOUT_ONLY" ]; then
     if [ -z "$PREFIX" ]; then
-        echo $0 [--skip-include-triplet-prefix] [--with-default-win32-winnt=0x601] [--with-default-msvcrt=ucrt] [--host=<triple>] dest
+        echo $0 [--skip-include-triplet-prefix] [--with-default-win32-winnt=0x601] [--with-default-msvcrt=ucrt] dest
         exit 1
     fi
 
@@ -70,123 +66,59 @@ if [ "$(which gmake)" != "" ]; then
     MAKE=gmake
 fi
 
-ORIGPATH="$PATH"
+export PATH="$PREFIX/bin:$PATH"
-if [ -z "$HOST" ]; then
-    # The newly built toolchain isn't crosscompiled; add it to the path.
-    export PATH="$PREFIX/bin:$PATH"
-else
-    # Crosscompiling the toolchain itself; the cross compiler is
-    # expected to already be in $PATH.
-    true
-fi
 
 : ${CORES:=$(nproc 2>/dev/null)}
 : ${CORES:=$(sysctl -n hw.ncpu 2>/dev/null)}
 : ${CORES:=4}
 : ${ARCHS:=${TOOLCHAIN_ARCHS-i686 x86_64 armv7 aarch64}}
 
-# If crosscompiling the toolchain itself, we already have a mingw-w64
+if [ -z "$SKIP_INCLUDE_TRIPLET_PREFIX" ]; then
-# runtime and don't need to rebuild it.
+    HEADER_ROOT="$PREFIX/generic-w64-mingw32"
-if [ -z "$HOST" ]; then
-    if [ -z "$SKIP_INCLUDE_TRIPLET_PREFIX" ]; then
-        HEADER_ROOT="$PREFIX/generic-w64-mingw32"
-    else
-        HEADER_ROOT="$PREFIX"
-    fi
-
-    cd mingw-w64-headers
-    [ -z "$CLEAN" ] || rm -rf build
-    mkdir -p build
-    cd build
-    ../configure --prefix="$HEADER_ROOT" \
-        --enable-idl --with-default-win32-winnt=$DEFAULT_WIN32_WINNT --with-default-msvcrt=$DEFAULT_MSVCRT INSTALL="install -C"
-    $MAKE install
-    cd ../..
-    if [ -z "$SKIP_INCLUDE_TRIPLET_PREFIX" ]; then
-        for arch in $ARCHS; do
-            mkdir -p "$PREFIX/$arch-w64-mingw32"
-            if [ ! -e "$PREFIX/$arch-w64-mingw32/include" ]; then
-                ln -sfn ../generic-w64-mingw32/include "$PREFIX/$arch-w64-mingw32/include"
-            fi
-        done
-    fi
-
-    cd mingw-w64-crt
-    for arch in $ARCHS; do
-        [ -z "$CLEAN" ] || rm -rf build-$arch
-        mkdir -p build-$arch
-        cd build-$arch
-        case $arch in
-        armv7)
-            FLAGS="--disable-lib32 --disable-lib64 --enable-libarm32"
-            ;;
-        aarch64)
-            FLAGS="--disable-lib32 --disable-lib64 --enable-libarm64"
-            ;;
-        i686)
-            FLAGS="--enable-lib32 --disable-lib64"
-            ;;
-        x86_64)
-            FLAGS="--disable-lib32 --enable-lib64"
-            ;;
-        esac
-        FLAGS="$FLAGS --with-default-msvcrt=$DEFAULT_MSVCRT"
-        ../configure --host=$arch-w64-mingw32 --prefix="$PREFIX/$arch-w64-mingw32" $FLAGS
-        $MAKE -j$CORES
-        $MAKE install
-        cd ..
-    done
-    cd ..
-fi
-
-if [ -n "$HOST" ]; then
-    CONFIGFLAGS="$CONFIGFLAGS --host=$HOST"
-    CROSS_NAME=$HOST-
-    EXEEXT=.exe
 else
-    case $(uname) in
+    HEADER_ROOT="$PREFIX"
-    MINGW*)
-        EXEEXT=.exe
-        ;;
-    *)
-        ;;
-    esac
-fi
-if [ -n "$SKIP_INCLUDE_TRIPLET_PREFIX" ]; then
-    CONFIGFLAGS="$CONFIGFLAGS --with-widl-includedir=$PREFIX/include"
-    # If using the same includedir for all archs, it's enough to
-    # build one single binary.
-    ALL_ARCHS="$ARCHS"
-    ARCHS=x86_64
 fi
 
-# If building on windows, we've installed prefixless wrappers - these break
+cd mingw-w64-headers
-# building widl, as the toolchain isn't functional yet. Restore the original
+[ -z "$CLEAN" ] || rm -rf build
-# path.
+mkdir -p build
-export PATH="$ORIGPATH"
+cd build
-cd mingw-w64-tools/widl
+../configure --prefix="$HEADER_ROOT" \
-for arch in $ARCHS; do
+    --enable-idl --with-default-win32-winnt=$DEFAULT_WIN32_WINNT --with-default-msvcrt=$DEFAULT_MSVCRT INSTALL="install -C"
-    [ -z "$CLEAN" ] || rm -rf build-$CROSS_NAME$arch
+$MAKE install
-    mkdir -p build-$CROSS_NAME$arch
+cd ../..
-    cd build-$CROSS_NAME$arch
+if [ -z "$SKIP_INCLUDE_TRIPLET_PREFIX" ]; then
-    ../configure --prefix="$PREFIX" --target=$arch-w64-mingw32 $CONFIGFLAGS
+    for arch in $ARCHS; do
-    $MAKE -j$CORES
+        mkdir -p "$PREFIX/$arch-w64-mingw32"
-    $MAKE install-strip
+        if [ ! -e "$PREFIX/$arch-w64-mingw32/include" ]; then
-    cd ..
+            ln -sfn ../generic-w64-mingw32/include "$PREFIX/$arch-w64-mingw32/include"
-done
-cd "$PREFIX/bin"
-if [ -n "$SKIP_INCLUDE_TRIPLET_PREFIX" ]; then
-    for arch in $ALL_ARCHS; do
-        if [ "$arch" != "$ARCHS" ]; then
-            ln -sf $ARCHS-w64-mingw32-widl$EXEEXT $arch-w64-mingw32-widl$EXEEXT
         fi
     done
 fi
-if [ -n "$EXEEXT" ]; then
+
-    if [ -z "$HOST" ] && [ -f clang$EXEEXT ]; then
+cd mingw-w64-crt
-        HOST=$(./clang -dumpmachine | sed 's/-.*//')-w64-mingw32
+for arch in $ARCHS; do
-    fi
+    [ -z "$CLEAN" ] || rm -rf build-$arch
-    if [ -n "$HOST" ]; then
+    mkdir -p build-$arch
-        ln -sf $HOST-widl$EXEEXT widl$EXEEXT
+    cd build-$arch
-    fi
+    case $arch in
-fi
+    armv7)
+        FLAGS="--disable-lib32 --disable-lib64 --enable-libarm32"
+        ;;
+    aarch64)
+        FLAGS="--disable-lib32 --disable-lib64 --enable-libarm64"
+        ;;
+    i686)
+        FLAGS="--enable-lib32 --disable-lib64"
+        ;;
+    x86_64)
+        FLAGS="--disable-lib32 --enable-lib64"
+        ;;
+    esac
+    FLAGS="$FLAGS --with-default-msvcrt=$DEFAULT_MSVCRT"
+    ../configure --host=$arch-w64-mingw32 --prefix="$PREFIX/$arch-w64-mingw32" $FLAGS
+    $MAKE -j$CORES
+    $MAKE install
+    cd ..
+done
+cd ..