Files
llvm-mingw/.github/workflows/build.yml
T
Martin Storsjö c49af4399a github: Skip dependencies (libxml2, terminfo) in linux builds that have little value in our context
The fewer dynamic library dependencies we have, the easier it is
to run the binaries on other linux distributions than the one it
was built on.

In the docker based releases, these libraries haven't been installed
and thus haven't been used so far.

The docker based releases haven't linked against zlib though, but
keep linking against that, since that should have a quite stable
ABI and not cause much binary compatibility issues.
2023-04-26 10:01:42 +03:00

588 lines
23 KiB
YAML

name: Builds
on:
push:
branches:
- '**'
schedule:
- cron: '0 0 * * *'
permissions:
contents: read
jobs:
# For scheduled builds, get the commit ids of the latest versions to build.
#
# Doing this as one single job, which then propagates the information further
# to the other jobs, so that all build jobs in one workflow builds the exact
# same version. This allows trusting builds without assertions enabled
# slightly more, when we know a separate build of the same version but with
# assertions enabled, has passed some amount of tests.
prepare:
if: (github.event_name != 'schedule') || (github.repository == 'mstorsjo/llvm-mingw')
runs-on: ubuntu-latest
outputs:
LLVM_VERSION: ${{steps.get-versions.outputs.LLVM_VERSION}}
MINGW_W64_VERSION: ${{steps.get-versions.outputs.MINGW_W64_VERSION}}
PYTHON_VERSION_MINGW: ${{steps.get-versions.outputs.PYTHON_VERSION_MINGW}}
TAG: ${{steps.get-tag.outputs.TAG}}
steps:
- name: Select build tag
id: get-tag
run: |
if ${{github.event_name == 'schedule'}}; then
TAG=nightly
else
TAG=$(TZ=UTC date +%Y%m%d)
fi
echo TAG=$TAG >> $GITHUB_OUTPUT
cat $GITHUB_OUTPUT
- name: Check latest version
if: github.event_name == 'schedule'
id: get-versions
run: |
echo LLVM_VERSION=$(git ls-remote https://github.com/llvm/llvm-project.git | grep 'refs/heads/main$' | awk '{print $1}') >> $GITHUB_OUTPUT
echo MINGW_W64_VERSION=$(git ls-remote https://github.com/mingw-w64/mingw-w64.git | grep 'refs/heads/master$' | awk '{print $1}') >> $GITHUB_OUTPUT
echo PYTHON_VERSION_MINGW=$(git ls-remote https://github.com/msys2-contrib/cpython-mingw.git | grep 'refs/heads/mingw-v3.10.9$' | awk '{print $1}') >> $GITHUB_OUTPUT
cat $GITHUB_OUTPUT
# Build a cross compiler for Linux, targeting Windows.
linux:
if: (github.event_name != 'schedule') || (github.repository == 'mstorsjo/llvm-mingw')
needs: [prepare]
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v3
- name: Build
env:
LLVM_VERSION: ${{needs.prepare.outputs.LLVM_VERSION}}
MINGW_W64_VERSION: ${{needs.prepare.outputs.MINGW_W64_VERSION}}
TAG: ${{needs.prepare.outputs.TAG}}
run: |
sudo apt-get update && sudo apt-get install ninja-build
# Skip dynamic library dependencies that might make it harder to
# run the binaries on other distros (and that have little use within
# llvm-mingw).
LLVM_CMAKEFLAGS="-DLLVM_ENABLE_LIBXML2=OFF -DLLVM_ENABLE_TERMINFO=OFF" ./build-all.sh $(pwd)/install/llvm-mingw --disable-clang-tools-extra --disable-lldb
.github/workflows/store-version.sh install/llvm-mingw/versions.txt
cd install
DISTRO=ubuntu-$(grep DISTRIB_RELEASE /etc/lsb-release | cut -f 2 -d =)-$(uname -m)
NAME=llvm-mingw-$TAG-ucrt-$DISTRO
mv llvm-mingw $NAME
tar -Jcf ../$NAME.tar.xz $NAME
- uses: actions/upload-artifact@v3
with:
name: linux-toolchain
path: |
llvm-mingw-*.tar.xz
retention-days: 7
# Build a cross compiler for Linux, targeting Windows, with assertions enabled.
#
# The scheduled builds are made with the latest git version of llvm-project
# and mingw-w64. When using random git snapshot builds of llvm, there's
# always a risk for bugs - thus build such versions with assertions enabled,
# to better catch such bugs early. This makes the first-stage toolchain built
# here in scheduled builds somewhat slower.
linux-asserts:
if: (github.event_name == 'schedule') && (github.repository == 'mstorsjo/llvm-mingw')
needs: [prepare]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Build
env:
LLVM_VERSION: ${{needs.prepare.outputs.LLVM_VERSION}}
MINGW_W64_VERSION: ${{needs.prepare.outputs.MINGW_W64_VERSION}}
run: |
sudo apt-get update && sudo apt-get install ninja-build
# Skip dynamic library dependencies that might make it harder to
# run the binaries on other distros (and that have little use within
# llvm-mingw).
LLVM_CMAKEFLAGS="-DLLVM_ENABLE_LIBXML2=OFF -DLLVM_ENABLE_TERMINFO=OFF" ./build-all.sh $(pwd)/install/llvm-mingw --disable-clang-tools-extra --disable-lldb --enable-asserts
.github/workflows/store-version.sh install/llvm-mingw/versions.txt
cd install
tar -Jcf ../llvm-mingw-linux.tar.xz llvm-mingw
- uses: actions/upload-artifact@v3
with:
name: linux-asserts-toolchain
path: |
llvm-mingw-linux.tar.xz
retention-days: 7
# Build a cross compiler for macOS, targeting Windows.
macos:
if: (github.event_name != 'schedule') || (github.repository == 'mstorsjo/llvm-mingw')
needs: [prepare]
runs-on: macos-latest
steps:
- uses: actions/checkout@v3
- name: Build
env:
LLVM_VERSION: ${{needs.prepare.outputs.LLVM_VERSION}}
MINGW_W64_VERSION: ${{needs.prepare.outputs.MINGW_W64_VERSION}}
TAG: ${{needs.prepare.outputs.TAG}}
run: |
brew install ninja
# Building with MACOS_REDIST=1 (building universal binaries) without
# --disable-lldb fails in Github Actions, because python is available
# (and gets picked up), but only exists in native form.
# Disable use of zstd too - it's also available, but only in native
# form.
MACOS_REDIST=1 LLVM_CMAKEFLAGS="-DLLVM_ENABLE_ZSTD=OFF" ./build-all.sh $(pwd)/install/llvm-mingw --disable-clang-tools-extra --disable-lldb
.github/workflows/store-version.sh install/llvm-mingw/versions.txt
cd install
NAME=llvm-mingw-$TAG-ucrt-macos-universal
mv llvm-mingw $NAME
tar -Jcf ../$NAME.tar.xz $NAME
- uses: actions/upload-artifact@v3
with:
name: macos-toolchain
path: |
llvm-mingw-*.tar.xz
retention-days: 7
# Test building the toolchain on msys2 (in the mingw64 and clang64
# environments). The binaries built here rely on the runtime libraries from
# the host environment (libstdc++ or libc++). No artifacts are stored from
# these builds, but llvm-mingw's tests are run.
msys2:
if: (github.event_name != 'schedule') || (github.repository == 'mstorsjo/llvm-mingw')
needs: [prepare]
runs-on: windows-latest
defaults:
run:
shell: msys2 {0}
strategy:
fail-fast: false
matrix:
sys:
- mingw64
- clang64
steps:
- uses: msys2/setup-msys2@v2
with:
msystem: ${{matrix.sys}}
install: >-
git
make
diffutils
pacboy: >-
toolchain:p
cmake:p
ninja:p
- uses: actions/checkout@v3
- name: Build
env:
LLVM_VERSION: ${{needs.prepare.outputs.LLVM_VERSION}}
MINGW_W64_VERSION: ${{needs.prepare.outputs.MINGW_W64_VERSION}}
run: |
./build-all.sh $(pwd)/install/llvm-mingw --disable-clang-tools-extra --disable-lldb
.github/workflows/store-version.sh install/llvm-mingw/versions.txt
./run-tests.sh $(pwd)/install/llvm-mingw
# Use the Linux cross compilers built in the first step to cross compile
# llvm and make a proper standalone toolchain for Windows (for all 4
# architectures). The binaries built here match actual releases quite closely.
linux-cross:
if: (github.event_name != 'schedule') || (github.repository == 'mstorsjo/llvm-mingw')
needs: [linux, prepare]
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
arch:
- i686
- x86_64
- armv7
- aarch64
steps:
- uses: actions/download-artifact@v3
with:
name: linux-toolchain
- name: Unpack cross toolchain
run: |
tar -Jxf llvm-mingw-*.tar.xz
rm llvm-mingw-*.tar.xz
sudo mv llvm-mingw* /opt/llvm-mingw
echo /opt/llvm-mingw/bin >> $GITHUB_PATH
- uses: actions/checkout@v3
- name: Build
env:
LLVM_VERSION: ${{needs.prepare.outputs.LLVM_VERSION}}
MINGW_W64_VERSION: ${{needs.prepare.outputs.MINGW_W64_VERSION}}
PYTHON_VERSION_MINGW: ${{needs.prepare.outputs.PYTHON_VERSION_MINGW}}
TAG: ${{needs.prepare.outputs.TAG}}
run: |
sudo apt-get update && sudo apt-get install autoconf-archive ninja-build
./build-cross-tools.sh /opt/llvm-mingw $(pwd)/install/llvm-mingw ${{matrix.arch}} --with-python
.github/workflows/store-version.sh install/llvm-mingw/versions.txt
cd install
NAME=llvm-mingw-$TAG-ucrt-${{matrix.arch}}
mv llvm-mingw $NAME
zip -9rq ../$NAME.zip $NAME
- uses: actions/upload-artifact@v3
with:
name: windows-${{matrix.arch}}-toolchain
path: |
llvm-mingw-*.zip
retention-days: 7
# Run llvm-mingw's tests on x86_64 (and i686) with the cross-built x86_64
# toolchain from above.
test-toolchain:
if: (github.event_name != 'schedule') || (github.repository == 'mstorsjo/llvm-mingw')
needs: [linux-cross]
runs-on: windows-latest
defaults:
run:
shell: msys2 {0}
strategy:
fail-fast: false
matrix:
arch:
- x86_64
- i686
steps:
- uses: msys2/setup-msys2@v2
with:
msystem: mingw64
install: >-
unzip
make
- uses: actions/download-artifact@v3
with:
name: windows-${{matrix.arch}}-toolchain
- name: Unpack toolchain
run: |
unzip -q llvm-mingw-*.zip
rm llvm-mingw-*.zip
mv llvm-mingw-* /llvm-mingw
echo /llvm-mingw/bin >> $GITHUB_PATH
- uses: actions/checkout@v3
- name: Run tests
run: |
./run-tests.sh /llvm-mingw
./run-lldb-tests.sh /llvm-mingw
# Run libcxx's tests with the cross-built i686/x86_64 toolchains from above.
# (This builds its own copy of libcxx, but it should be pretty much
# identical to the one shipped - and tests that the toolchain works
# for running the libcxx tests.)
# This also forces testing the bundled python executables.
test-libcxx:
# This test is kinda slow, and kinda redundant (libcxx does contain
# llvm-mingw configurations in its own CI, so the only difference with
# testing here is different runners with different OS setups, and
# testing with the latest compiler instead of an older release).
# Therefore, keep the test disabled by default; it's easy to enable
# temporarily on a branch for testing.
# The libcxxabi and libunwind tests don't pass on the pinned 16.x branch,
# only on git main.
if: false
needs: [linux-cross, prepare]
runs-on: windows-latest
strategy:
fail-fast: false
matrix:
arch:
- x86_64
- i686
steps:
- uses: actions/download-artifact@v3
with:
name: windows-${{matrix.arch}}-toolchain
- name: Unpack toolchain
run: |
Expand-Archive llvm-mingw-*.zip -DestinationPath .
del llvm-mingw-*.zip
mv llvm-mingw-* c:\llvm-mingw
echo "c:\llvm-mingw\bin" | Out-File -FilePath $Env:GITHUB_PATH -Encoding utf8 -Append
echo "c:\llvm-mingw\python\bin" | Out-File -FilePath $Env:GITHUB_PATH -Encoding utf8 -Append
echo "PYTHON_EXE=c:/llvm-mingw/python/bin/python3.exe" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append
- uses: actions/checkout@v3
- name: Checkout llvm-project
env:
LLVM_VERSION: ${{needs.prepare.outputs.LLVM_VERSION}}
run: |
bash -c "CHECKOUT_ONLY=1 ./build-llvm.sh"
- name: Build and test libcxx
run: |
cd llvm-project
mkdir build
cd build
cmake ../runtimes `
-G Ninja `
-DCMAKE_BUILD_TYPE=Release `
-DPython3_EXECUTABLE="$Env:PYTHON_EXE" `
-DLIBCXX_ENABLE_WERROR=YES `
-DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi;libunwind" `
-DLIBCXX_CXX_ABI=libcxxabi `
-DCMAKE_C_COMPILER=clang `
-DCMAKE_CXX_COMPILER=clang++ `
-DLIBCXXABI_ENABLE_SHARED=NO `
-DLIBCXX_ENABLE_STATIC_ABI_LIBRARY=YES `
-DLIBCXX_USE_COMPILER_RT=YES `
-DLIBCXXABI_USE_COMPILER_RT=YES `
-DLIBUNWIND_USE_COMPILER_RT=YES `
-DLIBCXXABI_USE_LLVM_UNWINDER=YES `
-DLIBCXX_EXTRA_SITE_DEFINES="__USE_MINGW_ANSI_STDIO=1;TEST_IS_EXECUTED_IN_A_SLOW_ENVIRONMENT"
ninja
ninja check-cxx check-cxxabi check-unwind
# Run the OpenMP tests with the cross-built i686/x86_64 toolchains from above.
# This also forces testing the bundled python executables.
test-openmp:
# Only running these tests in scheduled builds.
if: (github.event_name == 'schedule') && (github.repository == 'mstorsjo/llvm-mingw')
needs: [linux-cross, prepare]
runs-on: windows-latest
strategy:
fail-fast: false
matrix:
include:
- { arch: i686, asmflag: }
- { arch: x86_64, asmflag: -m64 }
steps:
- uses: actions/download-artifact@v3
with:
name: windows-${{matrix.arch}}-toolchain
- name: Unpack toolchain
run: |
Expand-Archive llvm-mingw-*.zip -DestinationPath .
del llvm-mingw-*.zip
mv llvm-mingw-* c:\llvm-mingw
echo "c:\llvm-mingw\bin" | Out-File -FilePath $Env:GITHUB_PATH -Encoding utf8 -Append
echo "c:\llvm-mingw\python\bin" | Out-File -FilePath $Env:GITHUB_PATH -Encoding utf8 -Append
echo "PYTHON_EXE=c:/llvm-mingw/python/bin/python3.exe" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append
- uses: actions/checkout@v3
- name: Checkout llvm-project
env:
LLVM_VERSION: ${{needs.prepare.outputs.LLVM_VERSION}}
run: |
bash -c "CHECKOUT_ONLY=1 ./build-llvm.sh"
- name: Build test executables
run: |
cd llvm-project
mkdir build-tools
cd build-tools
cmake ../llvm `
-G Ninja `
-DCMAKE_BUILD_TYPE=Release `
-DLLVM_TARGETS_TO_BUILD=X86 `
-DCMAKE_C_COMPILER=clang `
-DCMAKE_CXX_COMPILER=clang++
ninja not FileCheck
echo "TOOLS=$PWD\bin" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append
- name: Build and test OpenMP
run: |
cd llvm-project
mkdir build-openmp
cd build-openmp
cmake ../openmp `
-G Ninja `
-DCMAKE_BUILD_TYPE=Release `
-DPython3_EXECUTABLE="$Env:PYTHON_EXE" `
-DOPENMP_LLVM_TOOLS_DIR="$Env:TOOLS" `
-DCMAKE_C_COMPILER=clang `
-DCMAKE_CXX_COMPILER=clang++ `
-DCMAKE_ASM_MASM_COMPILER=llvm-ml `
-DLIBOMP_ASMFLAGS=${{matrix.asmflag}}
ninja
ninja check-openmp
# Run the compiler-rt tests with the cross-built i686/x86_64 toolchains from
# above. This also forces testing the bundled python executables.
test-compiler-rt:
# These tests don't pass on the pinned 16.x branch, but only on latest
# git main - thus only run it for scheduled jobs.
if: (github.event_name == 'schedule') && (github.repository == 'mstorsjo/llvm-mingw')
needs: [linux-cross, prepare]
runs-on: windows-latest
strategy:
fail-fast: false
matrix:
arch:
- x86_64
- i686
steps:
- uses: actions/download-artifact@v3
with:
name: windows-${{matrix.arch}}-toolchain
- name: Unpack toolchain
run: |
Expand-Archive llvm-mingw-*.zip -DestinationPath .
del llvm-mingw-*.zip
mv llvm-mingw-* c:\llvm-mingw
echo "c:\llvm-mingw\bin" | Out-File -FilePath $Env:GITHUB_PATH -Encoding utf8 -Append
echo "c:\llvm-mingw\python\bin" | Out-File -FilePath $Env:GITHUB_PATH -Encoding utf8 -Append
echo "PYTHON_EXE=c:/llvm-mingw/python/bin/python3.exe" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append
- uses: actions/checkout@v3
- name: Checkout llvm-project
env:
LLVM_VERSION: ${{needs.prepare.outputs.LLVM_VERSION}}
run: |
git config --global core.autocrlf false
bash -c "CHECKOUT_ONLY=1 ./build-llvm.sh"
- name: Build test executables
run: |
cd llvm-project
mkdir build-tools
cd build-tools
cmake ../llvm `
-G Ninja `
-DCMAKE_BUILD_TYPE=Release `
-DLLVM_TARGETS_TO_BUILD=X86 `
-DCMAKE_C_COMPILER=clang `
-DCMAKE_CXX_COMPILER=clang++
ninja not FileCheck llvm-config KillTheDoctor count
echo "LLVM_CONFIG=$PWD\bin\llvm-config.exe" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append
echo "CMAKE_DIR=$PWD\lib\cmake\llvm" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append
- name: Build and test compiler-rt
# Skipping libfuzzer; all tests fail since the clang mingw driver
# doesn't support -fsanitize=fuzzer.
# Skipping ORC; the tests rely on llvm-jitlink being available,
# which requires building a large part of LLVM among the test
# tools above, and jitlink requires an MSVC toolchain to be available.
run: |
cd llvm-project
mkdir build-compiler-rt
cd build-compiler-rt
cmake ../compiler-rt `
-G Ninja `
-DCMAKE_BUILD_TYPE=Release `
-DPython3_EXECUTABLE="$Env:PYTHON_EXE" `
-DCMAKE_C_COMPILER=clang `
-DCMAKE_CXX_COMPILER=clang++ `
-DCMAKE_C_COMPILER_TARGET=${{matrix.arch}}-w64-windows-gnu `
-DCOMPILER_RT_DEFAULT_TARGET_ONLY=TRUE `
-DCOMPILER_RT_USE_BUILTINS_LIBRARY=TRUE `
-DLLVM_CONFIG_PATH="$Env:LLVM_CONFIG" `
-DLLVM_CMAKE_DIR="$Env:CMAKE_DIR" `
-DSANITIZER_CXX_ABI=libc++ `
-DCOMPILER_RT_INCLUDE_TESTS=ON `
-DCOMPILER_RT_BUILD_LIBFUZZER=OFF `
-DCOMPILER_RT_BUILD_ORC=OFF
ninja
# Ideally we'd run check-compiler-rt here, to include all potential
# tests. However, for i686 we want to skip check-asan-dynamic
# (which has stray failures) and check-interception (which always
# seems to fail in mingw i686 configurations). For x86_64, we also
# specify the individual testsuites to run, since check-compiler-rt
# also includes check-asan, which tests asan for the static CRT,
# which isn't supported on mingw targets.
ninja ${{ (matrix.arch == 'x86_64') && 'check-asan-dynamic check-interception' || '' }} check-builtins check-profile check-sanitizer check-ubsan
# Test cross-building ffmpeg for all 4 targeted architectures from linux.
# This is done only on scheduled builds, with a toolchain with assertions
# enabled, to catch code generation bugs that might trigger asserts, to
# find such regressions early.
linux-test-cross-build-ffmpeg:
if: (github.event_name == 'schedule') && (github.repository == 'mstorsjo/llvm-mingw')
needs: [linux-asserts]
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
arch:
- i686
- x86_64
- armv7
- aarch64
steps:
- uses: actions/download-artifact@v3
with:
name: linux-asserts-toolchain
- name: Unpack cross toolchain
run: |
tar -Jxf llvm-mingw-*.tar.xz
rm llvm-mingw-*.tar.xz
sudo mv llvm-mingw* /opt/llvm-mingw
echo /opt/llvm-mingw/bin >> $GITHUB_PATH
- name: Checkout ffmpeg
uses: actions/checkout@v3
with:
repository: ffmpeg/ffmpeg
ref: n6.0
path: ffmpeg
- name: Build ffmpeg
run: |
sudo apt-get update && sudo apt-get install nasm
mkdir ffmpeg-build
cd ffmpeg-build
# Set a limit of 3 minutes per process/object file; with unproven
# versions from latest git, the compiler may occasionally hang.
# Make sure we exit in a reasonable time instead of waiting for
# the whole github actions timeout of 6h before exiting.
ulimit -t 180
../ffmpeg/configure --arch=${{matrix.arch}} --target-os=mingw32 --cross-prefix=${{matrix.arch}}-w64-mingw32- --enable-gpl
make -j$(nproc)
# Test building ffmpeg for natively on x86_64 Windows and run its tests,
# to find cases of compiler bugs don't show up as failed asserts in the
# compiler itself, but that only show up at runtime. This is only done
# for scheduled builds.
test-ffmpeg:
if: (github.event_name == 'schedule') && (github.repository == 'mstorsjo/llvm-mingw')
needs: [linux-cross]
runs-on: windows-latest
defaults:
run:
shell: msys2 {0}
steps:
- name: Avoid git checking out files with CRLF
shell: cmd
run: |
git config --global core.autocrlf false
- uses: msys2/setup-msys2@v2
with:
msystem: mingw64
install: >-
unzip
make
rsync
diffutils
pacboy: >-
nasm:p
- uses: actions/download-artifact@v3
with:
name: windows-x86_64-toolchain
- name: Unpack toolchain
run: |
unzip -q llvm-mingw-*.zip
rm llvm-mingw-*.zip
mv llvm-mingw-* /llvm-mingw
echo /llvm-mingw/bin >> $GITHUB_PATH
- name: Checkout ffmpeg
uses: actions/checkout@v3
with:
repository: ffmpeg/ffmpeg
ref: n6.0
path: ffmpeg
- name: Build & test ffmpeg
run: |
export PATH=/llvm-mingw/bin:$PATH
mkdir ffmpeg-build
cd ffmpeg-build
../ffmpeg/configure --samples=../fate-samples --enable-gpl
make -j$(nproc)
make fate-rsync
make -j$(nproc) fate
upload-nightly:
if: (github.event_name == 'schedule') && (github.repository == 'mstorsjo/llvm-mingw')
permissions:
contents: write
needs: [linux, macos, linux-cross, test-toolchain, linux-test-cross-build-ffmpeg, test-ffmpeg]
runs-on: ubuntu-latest
steps:
- name: Download all artifacts
uses: actions/download-artifact@v3
- name: Rearrange files
run: |
mv windows-*-toolchain/*.zip linux-toolchain/*.tar.xz macos-toolchain/*.tar.xz .
- name: Upload binaries
env:
GITHUB_TOKEN: ${{github.token}}
run: |
gh release upload nightly *.tar.xz *.zip --clobber -R ${{github.repository}}