From e16ba0772fc48048daaf6d3de536f221756252a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Storsj=C3=B6?= Date: Wed, 26 Oct 2022 14:48:34 +0300 Subject: [PATCH] run-tests: Split out building and running of tests into a Makefile This allows building and running the tests in parallel. --- .github/workflows/build.yml | 1 + run-tests.sh | 322 +++++---------------------------- test/Makefile | 352 ++++++++++++++++++++++++++++++++++++ 3 files changed, 396 insertions(+), 279 deletions(-) create mode 100644 test/Makefile diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ec7ee5c..f56cb25 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -258,6 +258,7 @@ jobs: msystem: mingw64 install: >- unzip + make - uses: actions/download-artifact@v3 with: name: windows-${{matrix.arch}}-toolchain diff --git a/run-tests.sh b/run-tests.sh index 128b0a8..23014d0 100755 --- a/run-tests.sh +++ b/run-tests.sh @@ -24,11 +24,27 @@ PREFIX="$1" PREFIX="$(cd "$PREFIX" && pwd)" export PATH=$PREFIX/bin:$PATH +: ${CORES:=$(nproc 2>/dev/null)} +: ${CORES:=$(sysctl -n hw.ncpu 2>/dev/null)} +: ${CORES:=4} : ${ARCHS:=${TOOLCHAIN_ARCHS-i686 x86_64 armv7 aarch64}} +MAKE=make +if command -v gmake >/dev/null; then + MAKE=gmake +fi + +case $(uname -s) in +Darwin) + ;; +*) + # Assume everything except macOS has got GNU make >= 4.0 + MAKEOPTS="-O" +esac + cd test -DEFAULT_OSES="mingw32 mingw32uwp" +HAVE_UWP=1 cat< is-ucrt.c #include #if __MSVCRT_VERSION__ < 0x1400 && !defined(_UCRT) @@ -38,7 +54,7 @@ EOF ANY_ARCH=$(echo $ARCHS | awk '{print $1}') if ! $ANY_ARCH-w64-mingw32-gcc -E is-ucrt.c > /dev/null 2>&1; then # If the default CRT isn't UCRT, we can't build for mingw32uwp. - DEFAULT_OSES="mingw32" + unset HAVE_UWP fi rm -f is-ucrt.c @@ -62,10 +78,8 @@ if [ -z "$RUN_X86_64" ] && [ -z "$RUN_I686" ]; then case $(uname) in MINGW*|MSYS*) NATIVE_X86=1 - # A non-empty string to trigger running, even if no wrapper is needed. - RUN_X86_64=" " - RUN_I686=" " - export PATH=.:$PATH + RUN_X86_64=true + RUN_I686=true ;; *) case $(uname -m) in @@ -79,306 +93,56 @@ if [ -z "$RUN_X86_64" ] && [ -z "$RUN_I686" ]; then fi -TESTS_C="hello hello-tls crt-test setjmp" -TESTS_C_DLL="autoimport-lib" -TESTS_C_LINK_DLL="autoimport-main" -TESTS_C_NO_BUILTIN="crt-test" -TESTS_C_ANSI_STDIO="crt-test" -TESTS_CPP="hello-cpp global-terminate tlstest-main longjmp-cleanup" -TESTS_CPP_EXCEPTIONS="hello-exception exception-locale exception-reduced" -TESTS_CPP_STATIC="hello-exception" -TESTS_CPP_DLL="tlstest-lib throwcatch-lib" -TESTS_CPP_LINK_DLL="throwcatch-main" -TESTS_SSP="stacksmash" -TESTS_ASAN="stacksmash" -TESTS_FORTIFY="bufferoverflow crt-test" -TESTS_UBSAN="ubsan" -TESTS_OMP="hello-omp" -TESTS_UWP="uwp-error" -TESTS_IDL="idltest" -TESTS_OTHER_TARGETS="hello" -if [ -n "$HAVE_CFGUARD" ]; then - TESTS_CFGUARD="cfguard-test" -fi for arch in $ARCHS; do case $arch in i686) RUN="$RUN_I686" COPY="$COPY_I686" NATIVE="$NATIVE_X86" + HAVE_SANITIZERS=1 + HAVE_OPENMP=1 ;; x86_64) RUN="$RUN_X86_64" COPY="$COPY_X86_64" NATIVE="$NATIVE_X86" + HAVE_SANITIZERS=1 + HAVE_OPENMP=1 ;; armv7) RUN="$RUN_ARMV7" COPY="$COPY_ARMV7" NATIVE="$NATIVE_ARMV7" + unset HAVE_SANITIZERS + unset HAVE_OPENMP ;; aarch64) RUN="$RUN_AARCH64" COPY="$COPY_AARCH64" NATIVE="$NATIVE_AARCH64" + unset HAVE_SANITIZERS + unset HAVE_OPENMP ;; esac - TEST_DIR="$arch" - TESTS_EXTRA="" - FAILURE_TESTS="" - [ -z "$CLEAN" ] || rm -rf $TEST_DIR - # A leftover libc++.dll from a previous round will cause the linker to find it (and error out) instead of - # locating libc++.dll.a in a later include directory. The same goes with - # libunwind.dll. - rm -f $TEST_DIR/libc++.dll $TEST_DIR/libunwind.dll - mkdir -p $TEST_DIR - for test in $TESTS_C; do - $arch-w64-mingw32-clang $test.c -o $TEST_DIR/$test.exe - done - for target_os in $TARGET_OSES; do - # Check that some generic tests build successfully for other targets. - # These tests are included in other groups, so skip the for the default - # mingw32 target, only test them for e.g. UWP. - if [ "$target_os" = "mingw32" ]; then - continue - fi - for test in $TESTS_OTHER_TARGETS; do - $arch-w64-$target_os-clang $test.c -o $TEST_DIR/$test-$target_os.exe - TESTS_EXTRA="$TESTS_EXTRA $test-$target_os" - done - done - for test in $TESTS_C_DLL; do - $arch-w64-mingw32-clang $test.c -shared -o $TEST_DIR/$test.dll -Wl,--out-implib,$TEST_DIR/lib$test.dll.a - done - for test in $TESTS_C_LINK_DLL; do - $arch-w64-mingw32-clang $test.c -o $TEST_DIR/$test.exe -L$TEST_DIR -l${test%-main}-lib - done - for test in $TESTS_C_NO_BUILTIN; do - $arch-w64-mingw32-clang $test.c -o $TEST_DIR/$test-no-builtin.exe -fno-builtin - TESTS_EXTRA="$TESTS_EXTRA $test-no-builtin" - done - for test in $TESTS_C_ANSI_STDIO; do - $arch-w64-mingw32-clang $test.c -o $TEST_DIR/$test-ansi-stdio.exe -D__USE_MINGW_ANSI_STDIO=1 - TESTS_EXTRA="$TESTS_EXTRA $test-ansi-stdio" - done - for test in $TESTS_CPP; do - $arch-w64-mingw32-clang++ $test.cpp -o $TEST_DIR/$test.exe - done - for test in $TESTS_CPP_EXCEPTIONS; do - $arch-w64-mingw32-clang++ $test.cpp -o $TEST_DIR/$test.exe - $arch-w64-mingw32-clang++ $test.cpp -O2 -o $TEST_DIR/$test-opt.exe - TESTS_EXTRA="$TESTS_EXTRA $test $test-opt" - done - for test in $TESTS_CPP_STATIC; do - $arch-w64-mingw32-clang++ $test.cpp -static -o $TEST_DIR/$test-static.exe - TESTS_EXTRA="$TESTS_EXTRA $test-static" - done - for test in $TESTS_CPP_DLL; do - $arch-w64-mingw32-clang++ $test.cpp -shared -o $TEST_DIR/$test.dll -Wl,--out-implib,$TEST_DIR/lib$test.dll.a - done - for test in $TESTS_CPP_LINK_DLL; do - $arch-w64-mingw32-clang++ $test.cpp -o $TEST_DIR/$test.exe -L$TEST_DIR -l${test%-main}-lib - done - for test in $TESTS_SSP; do - $arch-w64-mingw32-clang $test.c -o $TEST_DIR/$test.exe -fstack-protector-strong - FAILURE_TESTS="$FAILURE_TESTS $test" - done - for test in $TESTS_CFGUARD; do - $arch-w64-mingw32-clang $test.c -o $TEST_DIR/$test.exe -mguard=cf - done - for test in $TESTS_FORTIFY; do - $arch-w64-mingw32-clang $test.c -o $TEST_DIR/$test-fortify.exe -O2 -D_FORTIFY_SOURCE=2 -lssp - TESTS_EXTRA="$TESTS_EXTRA $test-fortify" - if [ "$test" != "crt-test" ]; then - # crt-test doesn't trigger failures - FAILURE_TESTS="$FAILURE_TESTS $test-fortify" - fi - done - for test in $TESTS_IDL; do - # This is primary a build-only test, so no need to execute it. - # The IDL output isn't arch specific, but we want to test the - # individual widl frontends. - $arch-w64-mingw32-widl $test.idl -h -o $TEST_DIR/$test.h - $arch-w64-mingw32-clang $test.c -I$TEST_DIR -o $TEST_DIR/$test.exe -lole32 - done - for target_os in $TARGET_OSES; do - for test in $TESTS_UWP; do - set +e - # compilation should fail for UWP and WinRT - $arch-w64-$target_os-clang $test.c -o $TEST_DIR/$test-$target_os.exe -Wimplicit-function-declaration -Werror > /dev/null 2>&1 - UWP_ERROR=$? - set -e - case $target_os in - mingw32uwp) - if [ $UWP_ERROR -eq 0 ]; then - echo "UWP compilation should have failed for test $test!" - exit 1 - fi - ;; - *) - if [ $UWP_ERROR -eq 0 ]; then - TESTS_EXTRA="$TESTS_EXTRA $test-$target_os" - else - echo "$test failed to compile for non-UWP target!" - exit 1 - fi - ;; - esac - done - done - for test in $TESTS_ASAN; do - case $arch in - # Sanitizers on windows only support x86. - i686|x86_64) ;; - *) continue ;; - esac - $arch-w64-mingw32-clang $test.c -o $TEST_DIR/$test-asan.exe -fsanitize=address -g -gcodeview -Wl,--pdb= - # Only run these tests on native windows; asan doesn't run in wine. - if [ -n "$NATIVE" ]; then - TESTS_EXTRA="$TESTS_EXTRA $test-asan" - FAILURE_TESTS="$FAILURE_TESTS $test-asan" - fi - if [ -n "$HAVE_CFGUARD" ]; then - # Smoke test ASAN with CFGuard to make sure it doesn't trip. - $arch-w64-mingw32-clang $test.c -o $TEST_DIR/$test-asan-cfguard.exe -fsanitize=address -g -gcodeview -Wl,--pdb= -mguard=cf - if [ -n "$NATIVE" ]; then - TESTS_EXTRA="$TESTS_EXTRA $test-asan-cfguard" - FAILURE_TESTS="$FAILURE_TESTS $test-asan-cfguard" - fi - fi - done - for test in $TESTS_UBSAN; do - case $arch in - # Ubsan might not require anything too x86 specific, but we don't - # build any of the sanitizer libs for anything else than x86. - i686|x86_64) ;; - *) continue ;; - esac - $arch-w64-mingw32-clang $test.c -o $TEST_DIR/$test.exe -fsanitize=undefined -fno-sanitize-recover=all - TESTS_EXTRA="$TESTS_EXTRA $test" - FAILURE_TESTS="$FAILURE_TESTS $test" - done - for test in $TESTS_OMP; do - case $arch in - # OpenMP on windows only supports x86. - i686|x86_64) ;; - *) continue ;; - esac - $arch-w64-mingw32-clang $test.c -o $TEST_DIR/$test.exe -fopenmp=libomp - TESTS_EXTRA="$TESTS_EXTRA $test" - done - DLL="$TESTS_C_DLL $TESTS_CPP_DLL" - compiler_rt_arch=$arch - if [ "$arch" = "i686" ]; then - compiler_rt_arch=i386 - fi - for i in libc++ libunwind libssp-0 libclang_rt.asan_dynamic-$compiler_rt_arch libomp; do - if [ -f $PREFIX/$arch-w64-mingw32/bin/$i.dll ]; then - cp $PREFIX/$arch-w64-mingw32/bin/$i.dll $TEST_DIR - DLL="$DLL $i" - fi - done - RUN_TESTS="$TESTS_C $TESTS_C_LINK_DLL $TESTS_CPP $TESTS_CPP_LINK_DLL $TESTS_EXTRA $TESTS_SSP" - cd $TEST_DIR - if [ -n "$COPY" ]; then - COPYFILES="" - for i in $DLL; do - COPYFILES="$COPYFILES $i.dll" - done - for i in $RUN_TESTS; do - COPYFILES="$COPYFILES $i.exe" - done - $COPY $COPYFILES - fi + TARGET=all if [ -n "$RUN" ] && [ "$RUN" != "false" ]; then - for test in $RUN_TESTS; do - file=$test.exe - $RUN $file - done - - # These don't strictly require running native instead of in Wine - # (except for sanitizers, but they are already filtered out at this - # point), but some of the error situations trigger crashes, which - # might not work robustly on all exotic Wine configurations - thus - # only run these tests on native Windows. - if [ -n "$NATIVE" ]; then - for test in $FAILURE_TESTS; do - file=$test.exe - OUT=cmdoutput - rm -f $OUT - if $RUN $file trigger > $OUT 2>&1; then - cat $OUT - echo $file trigger should have failed - exit 1 - else - ret=$? - cat $OUT - echo $file trigger failed expectedly, returned $ret - - case $test in - stacksmash-asan|stacksmash-asan-cfguard) - grep -q stack-buffer-overflow $OUT - grep -q "func.*stacksmash.c" $OUT - ;; - ubsan) - grep -q "signed integer overflow" $OUT - ;; - stacksmash) - # GNU libssp writes this directly to the console, - # and it can't be redirected, so we can't check for its presence. - #grep -q "stack smashing detected" $OUT - ;; - bufferoverflow-*) - # GNU libssp writes this directly to the console, - # and it can't be redirected, so we can't check for its presence. - #grep -q "buffer overflow detected" $OUT - ;; - *) - echo Unhandled failure test $test - exit 1 - ;; - esac - rm -f $OUT - fi - done - # Run all testcases for the bufferoverflow test. - file=bufferoverflow-fortify.exe - OUT=cmdoutput - i=0 - while [ $i -le 10 ]; do - rm -f $OUT - if $RUN $file $i > $OUT 2>&1; then - cat $OUT - echo $file $i should have failed - exit 1 - else - ret=$? - cat $OUT - echo $file $i failed expectedly, returned $ret - #grep -q "buffer overflow detected" $OUT - rm -f $OUT - fi - i=$(($i+1)) - done - for test in $TESTS_CFGUARD; do - file=$test.exe - OUT=cmdoutput - rm -f $OUT - if $RUN $test.exe check_enabled; then - $RUN $test.exe normal_icall - $RUN $test.exe invalid_icall_nocf || [ $? = 2 ] - # We want to check the exit code to be 0xc0000409 - # (STATUS_STACK_BUFFER_OVERRUN aka fail fast exception). - # MSYS2 bash does not give us the full 32-bit exit code, so - # we have to rely on cmd.exe to perform the check. - # (This probably doesn't work on Wine, but Wine doesn't - # support CFG anyway, at least not for now...) - $RUN cmd //v:on //c "$test.exe invalid_icall & if !errorlevel! equ -1073740791 (exit 0) else (exit 1)" - fi - done + TARGET=test + if [ "$RUN" = "true" ]; then + unset RUN fi fi + COPYARG="" + if [ -n "$COPY" ]; then + COPYARG="COPY=$COPY" + fi + + TEST_DIR="$arch" + [ -z "$CLEAN" ] || rm -rf $TEST_DIR + mkdir -p $TEST_DIR + cd $TEST_DIR + $MAKE -f ../Makefile ARCH=$arch HAVE_UWP=$HAVE_UWP HAVE_CFGUARD=$HAVE_CFGUARD HAVE_SANITIZERS=$HAVE_SANITIZERS HAVE_OPENMP=$HAVE_OPENMP NATIVE=$NATIVE RUNTIMES_SRC=$PREFIX/$arch-w64-mingw32/bin clean + $MAKE -f ../Makefile ARCH=$arch HAVE_UWP=$HAVE_UWP HAVE_CFGUARD=$HAVE_CFGUARD HAVE_SANITIZERS=$HAVE_SANITIZERS HAVE_OPENMP=$HAVE_OPENMP NATIVE=$NATIVE RUNTIMES_SRC=$PREFIX/$arch-w64-mingw32/bin RUN="$RUN" $COPYARG $MAKEOPTS -j$CORES $TARGET cd .. done echo All tests succeeded diff --git a/test/Makefile b/test/Makefile new file mode 100644 index 0000000..f978dc4 --- /dev/null +++ b/test/Makefile @@ -0,0 +1,352 @@ +# +# Copyright (c) 2022 Martin Storsjo +# +# Permission to use, copy, modify, and/or distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +SRC_PATH=$(word 1, $(dir $(MAKEFILE_LIST))) +vpath %.c $(SRC_PATH) +vpath %.cpp $(SRC_PATH) +vpath %.idl $(SRC_PATH) + +ifneq ($(filter %-mingw32, $(MAKE_HOST)),) + # If we're running as mingw32-make, we're executing everything in + # cmd.exe, so adapt/filter out things that require a POSIX shell. + CMD = 1 + NATIVE = 1 + TOUCH_TARGET = + RM_F = del +else + TOUCH_TARGET = @touch $@ + RM_F = rm -f +endif + +# Run with TOOLEXT=.exe to run testing with Windows tools, from Linux make +# in WSL. + +ifneq ($(ARCH),) + CROSS = $(ARCH)-w64-mingw32- + CROSS_UWP = $(ARCH)-w64-mingw32uwp- +endif +CC = $(CROSS)clang$(TOOLEXT) +CXX = $(CROSS)clang++$(TOOLEXT) +WIDL = $(CROSS)widl$(TOOLEXT) +CC_UWP = $(CROSS_UWP)clang$(TOOLEXT) + +ifneq ($(COPY),) + COPY_TARGET = $(COPY) $@ + COPY_DEP = $(COPY) $< +else + COPY_TARGET = @: + COPY_DEP = @: +endif +ifneq ($(RUN),) + DO_RUN = $(RUN) # trailing whitespace +else + DO_RUN = ./ +endif + +EXEEXT = .exe +DLLEXT = .dll + +TESTS_C = hello hello-tls crt-test setjmp +TESTS_C_DLL = autoimport-lib +TESTS_C_LINK_DLL = autoimport-main +TESTS_C_NO_BUILTIN = crt-test +TESTS_C_ANSI_STDIO = crt-test +TESTS_CPP = hello-cpp global-terminate tlstest-main longjmp-cleanup +TESTS_CPP_EXCEPTIONS = hello-exception exception-locale exception-reduced +TESTS_CPP_STATIC = hello-exception +TESTS_CPP_DLL = tlstest-lib throwcatch-lib +TESTS_CPP_LINK_DLL = throwcatch-main +TESTS_SSP = stacksmash +TESTS_FORTIFY = bufferoverflow crt-test +ifneq ($(HAVE_SANITIZERS),) + TESTS_ASAN = stacksmash + TESTS_UBSAN = ubsan +endif +ifneq ($(HAVE_OPENMP),) + TESTS_OMP = hello-omp +endif +TESTS_UWP = uwp-error +TESTS_IDL = idltest +ifneq ($(HAVE_UWP),) + TESTS_OTHER_TARGETS = hello +endif +ifneq ($(HAVE_CFGUARD),) + TESTS_CFGUARD = cfguard-test + TESTS_ASAN_CFGUARD = $(TESTS_ASAN) +endif + +TARGETS_C = $(addsuffix $(EXEEXT), $(TESTS_C)) +TARGETS_C_DLL = $(addsuffix $(DLLEXT), $(TESTS_C_DLL)) +TARGETS_C_LINK_DLL = $(addsuffix $(EXEEXT), $(TESTS_C_LINK_DLL)) +TARGETS_C_NO_BUILTIN = $(addsuffix -no-builtin$(EXEEXT), $(TESTS_C_NO_BUILTIN)) +TARGETS_C_ANSI_STDIO = $(addsuffix -ansi-stdio$(EXEEXT), $(TESTS_C_ANSI_STDIO)) +TARGETS_CPP = $(addsuffix $(EXEEXT), $(TESTS_CPP)) +TARGETS_CPP_EXCEPTIONS = $(addsuffix $(EXEEXT), $(TESTS_CPP_EXCEPTIONS)) +TARGETS_CPP_EXCEPTIONS_OPT = $(addsuffix -opt$(EXEEXT), $(TESTS_CPP_EXCEPTIONS)) +TARGETS_CPP_STATIC = $(addsuffix -static$(EXEEXT), $(TESTS_CPP_STATIC)) +TARGETS_CPP_DLL = $(addsuffix $(DLLEXT), $(TESTS_CPP_DLL)) +TARGETS_CPP_LINK_DLL = $(addsuffix $(EXEEXT), $(TESTS_CPP_LINK_DLL)) +TARGETS_SSP = $(addsuffix $(EXEEXT), $(TESTS_SSP)) +TARGETS_CFGUARD = $(addsuffix $(EXEEXT), $(TESTS_CFGUARD)) +TARGETS_FORTIFY = $(addsuffix -fortify$(EXEEXT), $(TESTS_FORTIFY)) +TARGETS_IDL = $(addsuffix $(EXEEXT), $(TESTS_IDL)) +TARGETS_OTHER_TARGETS = $(addsuffix -mingw32uwp$(EXEEXT), $(TESTS_OTHER_TARGETS)) +TARGETS_UWP = $(addsuffix -mingw32$(EXEEXT), $(TESTS_UWP)) +ifneq ($(HAVE_UWP),) +ifeq ($(CMD),) + TARGETS_UWP_FAIL = $(addprefix .tested.build., $(addsuffix -mingw32uwp$(EXEEXT), $(TESTS_UWP))) +endif +endif +TARGETS_ASAN = $(addsuffix -asan$(EXEEXT), $(TESTS_ASAN)) +TARGETS_UBSAN = $(addsuffix $(EXEEXT), $(TESTS_UBSAN)) +TARGETS_ASAN_CFGUARD = $(addsuffix -asan-cfguard$(EXEEXT), $(TESTS_ASAN_CFGUARD)) +TARGETS_OMP = $(addsuffix $(EXEEXT), $(TESTS_OMP)) + +TARGETS = \ + $(TARGETS_C) $(TARGETS_C_DLL) $(TARGETS_C_LINK_DLL) $(TARGETS_C_NO_BUILTIN) $(TARGETS_C_ANSI_STDIO) \ + $(TARGETS_CPP) $(TARGETS_CPP_EXCEPTIONS) $(TARGETS_CPP_EXCEPTIONS_OPT) $(TARGETS_CPP_STATIC) $(TARGETS_CPP_DLL) $(TARGETS_CPP_LINK_DLL) \ + $(TARGETS_SSP) $(TARGETS_CFGUARD) $(TARGETS_FORTIFY) \ + $(TARGETS_IDL) \ + $(TARGETS_OTHER_TARGETS) $(TARGETS_UWP) $(TARGETS_UWP_FAIL) \ + $(TARGETS_ASAN) $(TARGETS_UBSAN) $(TARGETS_ASAN_CFGUARD) \ + $(TARGETS_OMP) + +# crt-test-fortify doesn't trigger failures +FAILURE_TESTS = \ + $(TARGETS_SSP) \ + $(filter-out crt-test-fortify$(EXEEXT), $(TARGETS_FORTIFY)) \ + $(TARGETS_ASAN) $(TARGETS_UBSAN) $(TARGETS_ASAN_CFGUARD) + +EXTRAFILES = \ + $(addprefix lib, $(addsuffix .dll.a, $(TESTS_C_DLL))) \ + $(addprefix lib, $(addsuffix .dll.a, $(TESTS_CPP_DLL))) \ + $(addsuffix .h, $(TESTS_IDL)) \ + $(addsuffix -asan.pdb, $(TESTS_ASAN)) \ + $(addsuffix -asan-cfguard.pdb, $(TESTS_ASAN_CFGUARD)) + +all: $(TARGETS) + +# A custom dependency outside of the generic patterns +tlstest-main$(EXEEXT): tlstest-lib$(DLLEXT) + +$(TARGETS_C): %$(EXEEXT): %.c + $(CC) $(CPPFLAGS) $(CFLAGS) $< -o $@ + +$(TARGETS_C_DLL): %$(DLLEXT): %.c + $(CC) $(CPPFLAGS) $(CFLAGS) $< -shared -o $@ -Wl,--out-implib,lib$*.dll.a + $(COPY_TARGET) + +$(TARGETS_CPP_DLL): %$(DLLEXT): %.cpp + $(CXX) $(CPPFLAGS) $(CFLAGS) $< -shared -o $@ -Wl,--out-implib,lib$*.dll.a + $(COPY_TARGET) + +$(TARGETS_C_NO_BUILTIN): %-no-builtin$(EXEEXT): %.c + $(CC) $(CPPFLAGS) $(CFLAGS) $< -o $@ -fno-builtin + +$(TARGETS_C_ANSI_STDIO): %-ansi-stdio$(EXEEXT): %.c + $(CC) $(CPPFLAGS) $(CFLAGS) $< -o $@ -D__USE_MINGW_ANSI_STDIO=1 + +$(TARGETS_CPP_EXCEPTIONS_OPT): %-opt$(EXEEXT): %.cpp + $(CXX) $(CPPFLAGS) $(CXXFLAGS) $< -o $@ -O2 + +$(TARGETS_CPP) $(TARGETS_CPP_EXCEPTIONS): %$(EXEEXT): %.cpp + $(CXX) $(CPPFLAGS) $(CXXFLAGS) $< -o $@ + +$(TARGETS_CPP_STATIC): %-static$(EXEEXT): %.cpp + $(CXX) -static $< -o $@ + +$(TARGETS_SSP): %$(EXEEXT): %.c + $(CC) $(CPPFLAGS) $(CFLAGS) $< -o $@ -fstack-protector-strong + +$(TARGETS_CFGUARD): %$(EXEEXT): %.c + $(CC) $(CPPFLAGS) $(CFLAGS) $< -o $@ -mguard=cf + +$(TARGETS_FORTIFY): %-fortify$(EXEEXT): %.c + $(CC) $(CPPFLAGS) $(CFLAGS) $< -o $@ -O2 -D_FORTIFY_SOURCE=2 -lssp + +$(TARGETS_ASAN): %-asan$(EXEEXT): %.c + $(CC) $(CPPFLAGS) $(CFLAGS) $< -o $@ -fsanitize=address -g -gcodeview -Wl,--pdb= + +$(TARGETS_UBSAN): %$(EXEEXT): %.c + $(CC) $(CPPFLAGS) $(CFLAGS) $< -o $@ -fsanitize=undefined -fno-sanitize-recover=all + +# Smoke test ASAN with CFGuard to make sure it doesn't trip. +$(TARGETS_ASAN_CFGUARD): %-asan-cfguard$(EXEEXT): %.c + $(CC) $(CPPFLAGS) $(CFLAGS) $< -o $@ -fsanitize=address -g -gcodeview -mguard=cf -Wl,--pdb= + +$(TARGETS_OMP): %$(EXEEXT): %.c + $(CC) $(CPPFLAGS) $(CFLAGS) $< -o $@ -fopenmp=libomp + +%.h: %.idl + $(WIDL) $< -h -o $@ + +$(TARGETS_IDL): %$(EXEEXT): %.c %.h + $(CC) $(CPPFLAGS) $(CFLAGS) $< -o $@ -I. -lole32 + +$(TARGETS_OTHER_TARGETS): %-mingw32uwp$(EXEEXT): %.c + $(CC_UWP) $< -o $@ + +$(TARGETS_UWP): %-mingw32$(EXEEXT): %.c + $(CC) $< -o $@ -Wimplicit-function-declaration -Werror + +$(TARGETS_UWP_FAIL): .tested.build.%-mingw32uwp$(EXEEXT): %.c + @echo $(CC_UWP) $< -o $@ -Wimplicit-function-declaration -Werror + @if $(CC_UWP) $< -o $@ -Wimplicit-function-declaration -Werror; then echo ERROR: $@ should have failed; rm -f $@; exit 1; else echo OK: UWP build failed intentionally; touch $@; fi + +.SECONDEXPANSION: +$(TARGETS_C_LINK_DLL): %$(EXEEXT): %.c $$(subst -main,-lib,$$*)$(DLLEXT) + $(CC) $< -o $@ -L. -l$(subst -main,-lib,$*) + +$(TARGETS_CPP_LINK_DLL): %$(EXEEXT): %.cpp $$(subst -main,-lib,$$*)$(DLLEXT) + $(CXX) $< -o $@ -L. -l$(subst -main,-lib,$*) + +COMPILER_RT_ARCH = $(ARCH) +ifeq ($(ARCH), i686) + COMPILER_RT_ARCH = i386 +endif + +RUNTIMES = libc++ libunwind libssp-0 libclang_rt.asan_dynamic-$(COMPILER_RT_ARCH) libomp +ifneq ($(RUNTIMES_SRC),) +LOCAL_RUNTIMES = $(patsubst $(RUNTIMES_SRC)/%, %, $(wildcard $(addprefix $(RUNTIMES_SRC)/, $(addsuffix $(DLLEXT), $(RUNTIMES))))) + +$(LOCAL_RUNTIMES): %$(DLLEXT): $(RUNTIMES_SRC)/%$(DLLEXT) + cp -a $< $@ + $(COPY_TARGET) +endif + +TESTS = $(filter-out %$(DLLEXT), $(TARGETS)) +# TARGETS_UWP_FAIL isn't a real executable, but a placeholder file to indicate +# that we tested (and failed) to compile the file. +TESTS := $(filter-out $(TARGETS_UWP_FAIL), $(TESTS)) +# TARGETS_IDL is a build-only test. +TESTS := $(filter-out $(TARGETS_IDL), $(TESTS)) +ifeq ($(NATIVE),) + # Only test asan on native Windows, as it doesn't run in Wine. + TESTS := $(filter-out $(TARGETS_ASAN) $(TARGETS_ASAN_CFGUARD), $(TESTS)) + FAILURE_TESTS := $(filter-out $(TARGETS_ASAN) $(TARGETS_ASAN_CFGUARD), $(FAILURE_TESTS)) +endif +TEST_TARGETS = $(addprefix .tested., $(TESTS)) + +ifneq ($(NATIVE),) +ifeq ($(CMD),) + # These don't strictly require running native instead of in Wine + # (except for asan, but that's are already filtered out at this + # point), but some of the error situations trigger crashes, which + # might not work robustly on all exotic Wine configurations - thus + # only run these tests on native Windows. + # These tests don't support being run in cmd.exe. + FAILURE_TEST_TARGETS = $(addprefix .failtested., $(FAILURE_TESTS)) + CUSTOM1_FAILURE_TEST_TARGETS = .customtest1.bufferoverflow-fortify$(EXEEXT) + ifneq ($(HAVE_CFGUARD),) + CUSTOM2_FAILURE_TEST_TARGETS = .customtest2.cfguard-test$(EXEEXT) + endif +endif +endif + +ALL_TEST_TARGETS = $(TEST_TARGETS) $(FAILURE_TEST_TARGETS) $(CUSTOM1_FAILURE_TEST_TARGETS) $(CUSTOM2_FAILURE_TEST_TARGETS) + +COPY_TARGETS = $(patsubst .tested.%, .copied.%, $(TEST_TARGETS)) + +test: all $(ALL_TEST_TARGETS) + +.copied.%: % + $(COPY_DEP) + $(TOUCH_TARGET) + +$(TEST_TARGETS): .tested.%$(EXEEXT): %$(EXEEXT) .copied.%$(EXEEXT) $(LOCAL_RUNTIMES) + $(DO_RUN)$< + $(TOUCH_TARGET) + +# GNU libssp writes the error messages directly to the console, and it can't be +# redirected, so we can't check for the presence of those messages. +# grep -q "stack smashing detected" $$OUT || exit 1 +# grep -q "buffer overflow detected" $$OUT || exit 1 +$(FAILURE_TEST_TARGETS): .failtested.%$(EXEEXT): %$(EXEEXT) .copied.%$(EXEEXT) $(LOCAL_RUNTIMES) + @echo $(DO_RUN)$< trigger; \ + OUT=log1-$*; \ + if $(DO_RUN)$< trigger > $$OUT 2>&1; then \ + cat $$OUT; \ + rm -f $$OUT; \ + echo ERROR: $< trigger should have failed; \ + exit 1; \ + else \ + ret=$$?; \ + cat $$OUT; \ + echo OK: $< trigger failed expectedly, returned $$ret; \ + case $* in \ + stacksmash-asan|stacksmash-asan-cfguard) \ + grep -q stack-buffer-overflow $$OUT || exit 1; \ + grep -q "func.*stacksmash.c" $$OUT || exit 1; \ + ;; \ + ubsan) \ + grep -q "signed integer overflow" $$OUT || exit 1; \ + ;; \ + stacksmash) \ + ;; \ + bufferoverflow-*) \ + ;; \ + *) \ + echo Unhandled failure test $*; \ + exit 1; \ + ;; \ + esac; \ + rm -f $$OUT; \ + fi + @touch $@ + +# grep -q "buffer overflow detected" $$OUT || exit 1 +.customtest1.%$(EXEEXT): %$(EXEEXT) .copied.%$(EXEEXT) $(LOCAL_RUNTIMES) + @i=0; while [ $$i -le 10 ]; do \ + OUT=log2-$*; \ + rm -f $$OUT; \ + echo $(DO_RUN)$< $$i; \ + if $(DO_RUN)$< $$i > $$OUT 2>&1; then \ + cat $$OUT; \ + echo ERROR: $* $$i should have failed; \ + rm -f $$OUT; \ + exit 1; \ + else \ + ret=$$?; \ + cat $$OUT; \ + echo OK: $* $$i failed expectedly, returned $$ret; \ + rm -f $$OUT; \ + fi; \ + i=$$(($$i+1)); \ + done + @touch $@ + +# We want to check the exit code to be 0xc0000409 (STATUS_STACK_BUFFER_OVERRUN +# aka fail fast exception). MSYS2 bash does not give us the full 32-bit exit +# code, so we have to rely on cmd.exe to perform the check. (This probably +# doesn't work on Wine, but Wine doesn't support CFG anyway, at least not for +# now...) +.customtest2.%$(EXEEXT): %$(EXEEXT) .copied.%$(EXEEXT) $(LOCAL_RUNTIMES) + @echo $(DO_RUN)$< check_enabled + @if $(DO_RUN)$< check_enabled; then \ + echo $(DO_RUN)$< normal_icall; \ + $(DO_RUN)$< normal_icall || exit 1; \ + echo $(DO_RUN)$< invalid_icall_nocf; \ + $(DO_RUN)$< invalid_icall_nocf; \ + [ $$? = 2 ] || exit 1; \ + echo $(DO_RUN)$< invalid_icall; \ + cmd.exe //v:on //c "$< invalid_icall & if !errorlevel! equ -1073740791 (exit 0) else (exit 1)"; \ + [ $$? = 0 ] || exit 1; \ + fi + @touch $@ + +clean: + $(RM_F) $(TARGETS) $(EXTRAFILES) $(ALL_TEST_TARGETS) $(COPY_TARGETS) $(LOCAL_RUNTIMES) + +.PHONY: all test clean