Check out llvm sources using the new llvm git monorepo

This updates compiler-rt, libunwind, libcxxabi and libcxx to a newer
version, matching the compiler.

This will be the official, canonical repo in the future.

For the non-dev Dockerfile build, this means that all the llvm based
subprojects need to be built in one RUN to avoid storing the llvm
checkout in the docker layers.

Dockerfile

@@ -2,8 +2,8 @@ FROM ubuntu:16.04
 
 RUN apt-get update -qq && apt-get install -qqy --no-install-recommends \
     git wget bzip2 file unzip libtool pkg-config cmake build-essential \
-    automake yasm gettext autopoint vim python git-svn ninja-build \
+    automake yasm gettext autopoint vim python ninja-build subversion \
-    subversion ca-certificates && \
+    ca-certificates && \
     apt-get clean -y && \
     rm -rf /var/lib/apt/lists/*
 
@@ -15,35 +15,19 @@ WORKDIR /build
 
 ENV TOOLCHAIN_PREFIX=/opt/llvm-mingw
 
-# Build and strip the LLVM installation
-COPY build-llvm.sh strip-llvm.sh ./
-RUN ./build-llvm.sh $TOOLCHAIN_PREFIX && \
-    ./strip-llvm.sh $TOOLCHAIN_PREFIX && \
-    rm -rf /build/*
-
 ARG TOOLCHAIN_ARCHS="i686 x86_64 armv7 aarch64"
 
-# Install the usual $TUPLE-clang binaries
+# Build everything that uses the llvm monorepo. We need to build the mingw runtime before the compiler-rt/libunwind/libcxxabi/libcxx runtimes.
+COPY build-llvm.sh strip-llvm.sh install-wrappers.sh build-mingw-w64.sh build-compiler-rt.sh build-mingw-w64-libraries.sh build-libcxx.sh ./
 COPY wrappers/*.sh wrappers/*.c wrappers/*.h ./wrappers/
-COPY install-wrappers.sh ./
+RUN ./build-llvm.sh $TOOLCHAIN_PREFIX && \
-RUN ./install-wrappers.sh $TOOLCHAIN_PREFIX && \
+    ./strip-llvm.sh $TOOLCHAIN_PREFIX && \
-    rm -rf /build/*
+    ./install-wrappers.sh $TOOLCHAIN_PREFIX && \
-
+    ./build-mingw-w64.sh $TOOLCHAIN_PREFIX && \
-# Build MinGW-w64, compiler-rt and mingw-w64's extra libraries
-COPY build-mingw-w64.sh build-compiler-rt.sh build-mingw-w64-libraries.sh ./
-RUN ./build-mingw-w64.sh $TOOLCHAIN_PREFIX && \
     ./build-compiler-rt.sh $TOOLCHAIN_PREFIX && \
     ./build-mingw-w64-libraries.sh $TOOLCHAIN_PREFIX && \
-    rm -rf /build/*
+    ./build-libcxx.sh $TOOLCHAIN_PREFIX && \
-
+    ./build-compiler-rt.sh $TOOLCHAIN_PREFIX --build-sanitizers && \
-# Build libunwind/libcxxabi/libcxx
-COPY build-libcxx.sh ./
-RUN ./build-libcxx.sh $TOOLCHAIN_PREFIX && \
-    rm -rf /build/*
-
-# Build sanitizers
-COPY build-compiler-rt.sh ./
-RUN ./build-compiler-rt.sh $TOOLCHAIN_PREFIX --build-sanitizers && \
     rm -rf /build/*
 
 # Build libssp

Dockerfile.dev

@@ -2,8 +2,8 @@ FROM ubuntu:16.04
 
 RUN apt-get update -qq && apt-get install -qqy --no-install-recommends \
     git wget bzip2 file unzip libtool pkg-config cmake build-essential \
-    automake yasm gettext autopoint vim python git-svn ninja-build \
+    automake yasm gettext autopoint vim python ninja-build subversion \
-    subversion ca-certificates && \
+    ca-certificates && \
     apt-get clean -y && \
     rm -rf /var/lib/apt/lists/*
 

build-compiler-rt.sh

@@ -31,9 +31,8 @@ export PATH=$PREFIX/bin:$PATH
 
 CLANG_VERSION=$(basename $(dirname $(dirname $(dirname $($PREFIX/bin/clang --print-libgcc-file-name -rtlib=compiler-rt)))))
 
-if [ ! -d compiler-rt ]; then
+if [ ! -d llvm-project/compiler-rt ] || [ -n "$SYNC" ]; then
-    git clone -b master https://github.com/llvm-mirror/compiler-rt.git
+    CHECKOUT_ONLY=1 ./build-llvm.sh
-    CHECKOUT=1
 fi
 
 # Add a symlink for i386 -> i686; we normally name the toolchain
@@ -43,12 +42,7 @@ if [ ! -e $PREFIX/i386-w64-mingw32 ]; then
     ln -sfn i686-w64-mingw32 $PREFIX/i386-w64-mingw32 || true
 fi
 
-cd compiler-rt
+cd llvm-project/compiler-rt
-
-if [ -n "$SYNC" ] || [ -n "$CHECKOUT" ]; then
-    [ -z "$SYNC" ] || git fetch
-    git checkout 806498c6546a07c65bb5c808a7d8964216cc24ca
-fi
 
 for arch in $ARCHS; do
     buildarchname=$arch

build-libcxx.sh

@@ -34,37 +34,12 @@ export PATH=$PREFIX/bin:$PATH
 : ${CORES:=4}
 : ${ARCHS:=${TOOLCHAIN_ARCHS-i686 x86_64 armv7 aarch64}}
 
-if [ ! -d libunwind ]; then
+if [ ! -d llvm-project/libunwind ] || [ -n "$SYNC" ]; then
-    git clone -b master https://github.com/llvm-mirror/libunwind.git
+    CHECKOUT_ONLY=1 ./build-llvm.sh
-    CHECKOUT_LIBUNWIND=1
-fi
-if [ ! -d libcxxabi ]; then
-    git clone -b master https://github.com/llvm-mirror/libcxxabi.git
-    CHECKOUT_LIBCXXABI=1
-fi
-if [ ! -d libcxx ]; then
-    git clone -b master https://github.com/llvm-mirror/libcxx.git
-    CHECKOUT_LIBCXX=1
-fi
-if [ -n "$SYNC" ] || [ -n "$CHECKOUT_LIBUNWIND" ]; then
-    cd libunwind
-    [ -z "$SYNC" ] || git fetch
-    git checkout df9c0cfd896524ae16dd7283cdf722f300c2b45d
-    cd ..
-fi
-if [ -n "$SYNC" ] || [ -n "$CHECKOUT_LIBCXXABI" ]; then
-    cd libcxxabi
-    [ -z "$SYNC" ] || git fetch
-    git checkout cac80b29da529d44ceb63930679e3a1af9cace37
-    cd ..
-fi
-if [ -n "$SYNC" ] || [ -n "$CHECKOUT_LIBCXX" ]; then
-    cd libcxx
-    [ -z "$SYNC" ] || git fetch
-    git checkout f16a595ec76a383f08380c81af3e21a3065364c1
-    cd ..
 fi
 
+cd llvm-project
+
 LIBCXX=$(pwd)/libcxx
 
 case $(uname) in

build-llvm.sh

@@ -31,46 +31,21 @@ PREFIX="$(cd "$PREFIX" && pwd)"
 : ${CORES:=$(sysctl -n hw.ncpu 2>/dev/null)}
 : ${CORES:=4}
 
-if [ ! -d llvm ]; then
+if [ ! -d llvm-project ]; then
     # When cloning master and checking out a pinned old hash, we can't use --depth=1.
-    git clone -b master https://github.com/llvm-mirror/llvm.git
+    git clone -b master https://github.com/llvm/llvm-project.git
-    cd llvm/tools
-    git clone -b master https://github.com/llvm-mirror/clang.git
-    git clone -b master https://github.com/llvm-mirror/lld.git
-    cd ..
-    set +e
-    # Do the git-svn rebase to populate git-svn information, to make
-    # "clang --version" produce SVN based version numbers.
-    # This is optional - don't error out here if git-svn is unavailable.
-    git svn init https://llvm.org/svn/llvm-project/llvm/trunk
-    git config svn-remote.svn.fetch :refs/remotes/origin/master
-    git svn rebase -l
-    cd tools/clang
-    git svn init https://llvm.org/svn/llvm-project/cfe/trunk
-    git config svn-remote.svn.fetch :refs/remotes/origin/master
-    git svn rebase -l
-    cd ../lld
-    git svn init https://llvm.org/svn/llvm-project/lld/trunk
-    git config svn-remote.svn.fetch :refs/remotes/origin/master
-    git svn rebase -l
-    cd ../../..
-    set -e
     CHECKOUT=1
 fi
 
 if [ -n "$SYNC" ] || [ -n "$CHECKOUT" ]; then
-    cd llvm
+    cd llvm-project
     [ -z "$SYNC" ] || git fetch
-    git checkout 66495492c0af53c4597cace26c50f0dce44033fc
+    git checkout e698958ad8031e0f17202e06f5de53989852bb66
-    cd tools/clang
+    cd ..
-    [ -z "$SYNC" ] || git fetch
-    git checkout 6fddf7789c74ae74d695dd571024915ad319db1b
-    cd ../lld
-    [ -z "$SYNC" ] || git fetch
-    git checkout 1e90cb9d20d4fcdb5d1448853370e969035d8014
-    cd ../../..
 fi
 
+[ -z "$CHECKOUT_ONLY" ] || exit 0
+
 if [ -n "$(which ninja)" ]; then
     CMAKE_GENERATOR="Ninja"
     NINJA=1
@@ -86,12 +61,12 @@ fi
 
 if [ -n "$HOST" ]; then
     find_native_tools() {
-        if [ -d llvm/build/bin ]; then
+        if [ -d llvm-project/llvm/build/bin ]; then
-            echo $(pwd)/llvm/build/bin
+            echo $(pwd)/llvm-project/llvm/build/bin
-        elif [ -d llvm/build-asserts/bin ]; then
+        elif [ -d llvm-project/llvm/build-asserts/bin ]; then
-            echo $(pwd)/llvm/build-asserts/bin
+            echo $(pwd)/llvm-project/llvm/build-asserts/bin
-        elif [ -d llvm/build-noasserts/bin ]; then
+        elif [ -d llvm-project/llvm/build-noasserts/bin ]; then
-            echo $(pwd)/llvm/build-noasserts/bin
+            echo $(pwd)/llvm-project/llvm/build-noasserts/bin
         elif [ -n "$(which llvm-tblgen)" ]; then
             echo $(dirname $(which llvm-tblgen))
         fi
@@ -130,7 +105,7 @@ if [ -n "$FULL_LLVM" ]; then
     TOOLCHAIN_ONLY=OFF
 fi
 
-cd llvm
+cd llvm-project/llvm
 mkdir -p $BUILDDIR
 cd $BUILDDIR
 cmake \
@@ -138,6 +113,7 @@ cmake \
     -DCMAKE_INSTALL_PREFIX="$PREFIX" \
     -DCMAKE_BUILD_TYPE=Release \
     -DLLVM_ENABLE_ASSERTIONS=$ASSERTS \
+    -DLLVM_ENABLE_PROJECTS="clang;lld" \
     -DLLVM_TARGETS_TO_BUILD="ARM;AArch64;X86" \
     -DLLVM_INSTALL_TOOLCHAIN_ONLY=$TOOLCHAIN_ONLY \
     -DLLVM_TOOLCHAIN_TOOLS="llvm-ar;llvm-ranlib;llvm-objdump;llvm-rc;llvm-cvtres;llvm-nm;llvm-strings;llvm-readobj;llvm-dlltool;llvm-pdbutil;llvm-objcopy;llvm-strip;llvm-cov;llvm-profdata;llvm-addr2line" \