Signed-off-by: Glenn Washburn <developm...@efficientek.com> --- .ci/build.sh | 66 +++ .ci/functions.gitlab.sh | 30 + .ci/functions.sh | 30 + .ci/make-images.sh | 71 +++ .ci/process-tests.sh | 108 ++++ .ci/test.sh | 106 ++++ .gitlab-ci.yml | 1184 +++++++++++++++++++++++++++++++++++++++ 7 files changed, 1595 insertions(+) create mode 100755 .ci/build.sh create mode 100644 .ci/functions.gitlab.sh create mode 100644 .ci/functions.sh create mode 100755 .ci/make-images.sh create mode 100755 .ci/process-tests.sh create mode 100755 .ci/test.sh create mode 100644 .gitlab-ci.yml
diff --git a/.ci/build.sh b/.ci/build.sh new file mode 100755 index 000000000..14dd7f7c5 --- /dev/null +++ b/.ci/build.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +set -eo pipefail + +# Environment variables +# JOBS: Number of concurrent jobs while building, defaults to number of +# processors plus 1, unless number of processors is 1 in which case its 1. +# SRCDIR: Path to source files +# BUILDDIR: Directory in which to place the build +# INSTALLDIR: Directory to install binaries +# ARCH: Architecture to build for +# PLATFORM: Platform to build for +# CONFIGURE_OPTS: Extra configure options +# SHELL_TRACE: Set to 'y' to enable shell tracing + +[ "x${SHELL_TRACE}" = "xy" ] && set -x + +[ -f "$(dirname "$0")/functions.sh" ] && +. "$(dirname "$0")/functions.sh" + +[ -f "$(dirname "$0")/functions.$CI_TYPE.sh" ] && +. "$(dirname "$0")/functions.$CI_TYPE.sh" + +JOBS=${JOBS:-`getconf _NPROCESSORS_ONLN 2> /dev/null || echo 1`}; +[ "$JOBS" == 1 ] || JOBS=$(($JOBS + 1)); + +TARGET="${ARCH}-${PLATFORM}" + +mkdir -pv ${BUILDDIR}; + +RET=0; +cd ${BUILDDIR}; + +# echo -e "#!$SHELL\nREAL_SHELL=\${REAL_SHELL:-\$(readlink /proc/\$\$/exe)}\nexec \$REAL_SHELL \$SHELL_OPTS \"\$@\"" >${BUILDDIR}/shell-wrapper.sh; +# cat <<EOF >${BUILDDIR}/shell-wrapper.sh +# #!$SHELL +# REAL_SHELL=\${REAL_SHELL:-$(readlink /proc/\$\$/exe)} +# exec \$REAL_SHELL \$SHELL_OPTS \"\$@\" +# EOF + +# chmod +x ${BUILDDIR}/shell-wrapper.sh; +# export CONFIG_SHELL=${BUILDDIR}/shell-wrapper.sh; + +start_log -c -n "configure-$TARGET" "Configuring $TARGET"; +ls -la $SRCDIR; +[ -x "$SRCDIR/configure" ] && $SRCDIR/configure --help; +$SRCDIR/configure --target=$ARCH --with-platform=$PLATFORM \ + --prefix=${INSTALLDIR} $CONFIGURE_OPTS || RET=$?; +end_log -n "configure-$TARGET"; + +if [ "$RET" -ne 0 ]; then + start_log -c -n "showlogs-$TARGET" "Configuring $TARGET failed, showing logs"; + cat ${BUILDDIR}/config.log; + end_log -n "showlogs-$TARGET"; + exit $RET; +fi; + +start_log -c -n "build-$TARGET" "Building $TARGET"; +make -j$JOBS || RET=$?; +end_log -n "build-$TARGET"; +[ "$RET" -ne 0 ] && exit $RET; + +start_log -c -n "install-$TARGET" "Installing $TARGET"; +make -j$JOBS install || RET=$?; +end_log -n "install-$TARGET"; +exit $RET; \ No newline at end of file diff --git a/.ci/functions.gitlab.sh b/.ci/functions.gitlab.sh new file mode 100644 index 000000000..e9cd2f9be --- /dev/null +++ b/.ci/functions.gitlab.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +TXT_RED="\e[31m" +TXT_YELLOW="\e[33m" +TXT_CLEAR="\e[0m" + +function start_log() { + while [ "$#" -gt 0 ]; do + case "$1" in + -c) LOG_COLLAPSE=1; shift;; + -n) LOG_NAME="$2"; shift;; + *) [ "$#" -eq 1 ] && LOG_MSG="$1"; shift;; + esac + done + + echo -e "section_start:`date +%s`:${LOG_NAME}${LOG_COLLAPSE:+[collapsed=true]}\r\e[0K${LOG_MSG}" +} + +function end_log() { + while [ "$#" -gt 0 ]; do + case "$1" in + -n) LOG_NAME=$2; shift;; + *) shift;; + esac + done + + echo -e "section_end:`date +%s`:${LOG_NAME}\r\e[0K" +} + +:; diff --git a/.ci/functions.sh b/.ci/functions.sh new file mode 100644 index 000000000..bba33fb5f --- /dev/null +++ b/.ci/functions.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +TXT_RED="\e[31m" +TXT_YELLOW="\e[33m" +TXT_CLEAR="\e[0m" + +function start_log() { + while [ "$#" -gt 0 ]; do + case "$1" in + -c) LOG_COLLAPSE=1; shift;; + -n) LOG_NAME="$2"; shift;; + *) [ "$#" -eq 1 ] && LOG_MSG="$1"; shift;; + esac + done + + echo -e "Start:${LOG_NAME} ${LOG_MSG}" +} + +function end_log() { + while [ "$#" -gt 0 ]; do + case "$1" in + -n) LOG_NAME=$2; shift;; + *) shift;; + esac + done + + echo -e "End:${LOG_NAME}" +} + +:; diff --git a/.ci/make-images.sh b/.ci/make-images.sh new file mode 100755 index 000000000..3c7d2f88c --- /dev/null +++ b/.ci/make-images.sh @@ -0,0 +1,71 @@ +#!/bin/bash + +set -eo pipefail + +# Environment variables +# SRCDIR: Path to source files +# BUILDDIR: Directory in which to place the build +# TARGET: Target to test +# MAKE_ALL_IMAGE_TARGETS: If set to 'y', then all image targets, even disabled +# ones, will be run. +# DISABLED_IMAGES: String of target formats to disable when building grub +# images delimited by new lines. +# SHELL_TRACE: Set to 'y' to enable shell tracing + +if [ "x${SHELL_TRACE}" = "xy" ]; then + # If we export SHELL_OPTS, then all shells will be traced, most of which we do not want + SHELL_OPTS="-x"; + set -x; +fi; + +[ -f "$(dirname "$0")/functions.sh" ] && +. "$(dirname "$0")/functions.sh" + +[ -f "$(dirname "$0")/functions.$CI_TYPE.sh" ] && +. "$(dirname "$0")/functions.$CI_TYPE.sh" + +start_log -c -n "images-$TARGET" "Making images for $TARGET"; + +function build_tformat() { + TARGET=$1; + FORMATS=$2; + for fmt in $FORMATS; do + if [ -z "${fmt%~*}" ]; then + echo "${TARGET}"; + else + echo "${TARGET}-${fmt}"; + fi; + done; +} + +tformats="~"; +case "$TARGET" in + mipsel-loongson) + tformats="mipsel-loongson-elf mipsel-yeeloong-flash mipsel-fuloong2f-flash" ;; + *) + case "$TARGET" in + i386-pc) tformats="~ pxe eltorito" ;; + sparc64-ieee1275) tformats="aout cdcore raw" ;; + mips-qemu_mips | \ + mipsel-qemu_mips) tformats="elf flash" ;; + arm-coreboot) tformats="vexpress veyron" ;; + esac; + tformats=$(build_tformat "${TARGET}" "$tformats"); + ;; +esac; + +RET=0; +mkdir -pv "${BUILDDIR}/images"; +for tfmt in $tformats; do + if [ "x${MAKE_ALL_IMAGE_TARGETS}" != "xy" ] && ( echo "${DISABLED_IMAGES}" | grep -q "^${tfmt}$" ); then + echo "Image build disabled for target format $tfmt"; + continue; + fi; + + echo "Making image for target format: ${tfmt}"; + ${BUILDDIR}/grub-mkimage -v -p / -O "${tfmt}" -o "${BUILDDIR}/images/grub-${tfmt}.img" echo reboot normal || _RET=$?; + [ "${_RET:-0}" -ne 0 ] && RET=$_RET; +done; + +end_log -n "images-$TARGET"; +exit $RET; diff --git a/.ci/process-tests.sh b/.ci/process-tests.sh new file mode 100755 index 000000000..ab030cfc2 --- /dev/null +++ b/.ci/process-tests.sh @@ -0,0 +1,108 @@ +#!/bin/bash + +set -eo pipefail + +# Environment variables +# SRCDIR: Path to source files +# BUILDDIR: Directory in which to place the build +# TARGET: Target to test +# TEST_ALL_TARGETS: If set to 'y', then all tests, even disabled ones will be +# run. +# DISABLED_TESTS: String of disabled tests delimited by new lines. +# TESTS_TIMEOUT: Maximum timeout to allow for 'make check'. Set this slightly +# below total job timeout to allow for cleanup and log packaging code. +# TEST_VERBOSITY: Number to set verbosity level to +# TEST_DATA_PREFIX: Filename prefix to use when saving test data. If not set, +# test data will not be saved. +# STRACE_TESTS: Set to 'y' to strace every tests for extreme debugging +# SHELL_TRACE: Set to 'y' to enable shell tracing + +[ "x${SHELL_TRACE}" = "xy" ] && set -x + +[ -f "$(dirname "$0")/functions.sh" ] && +. "$(dirname "$0")/functions.sh" + +[ -f "$(dirname "$0")/functions.$CI_TYPE.sh" ] && +. "$(dirname "$0")/functions.$CI_TYPE.sh" + +# TARGET="${ARCH}-${PLATFORM}" + +test "x${SHELL_TRACE}" = "xy" && set -x; +if [ -f ${BUILDDIR}/test.success ]; then + echo "Not processing test logs because make check ran successfully"; + exit 0; +elif [ "x${TEST_ALL_TARGETS}" != "xy" ] && ( echo "${DISABLED_TESTS}" | grep -q "^${TARGET}$" ); then + echo "No test output processing because testing was disabled"; + exit 0; +fi; + +start_log -c -n "process-test-$TARGET" "Processing test output of $TARGET"; +if [ -f ${BUILDDIR}/test-suite.log ]; then + ( grep -E "^(FAIL|ERROR|SKIP):" ${BUILDDIR}/test-suite.log || ':' ) | + while read STATUS TESTNAME; do + STATUS=${STATUS%%:}; + + if [ "$STATUS" = "SKIP" ]; then + TYPE=skip; + elif [ "$STATUS" = "ERROR" ]; then + TYPE=error; + elif echo "${EXPECTED_FAILURES}" | grep -qE "^(${TESTNAME}|${TARGET}:${TESTNAME})$"; then + echo 'Expected failed test:' "$TESTNAME"; + TYPE=expected; + # If any unexpected failures in the functional tests, count a failure in + # grub_func_test as a true failure. + elif [ "$TESTNAME" = "grub_func_test" ]; then + grep -E 'test:'' FAIL' ${BUILDDIR}/${TESTNAME}.log | sort -u | + while read FTESTNAME STATUS; do + FTESTNAME=${FTESTNAME%:*}; + if echo "${EXPECTED_FUNCTIONAL_FAILURES}" | grep -qE "^(${FTESTNAME}|${TARGET}:${FTESTNAME})$"; then + echo 'Expected failed functional test:' "$FTESTNAME"; + TYPE=expected; + else + echo -e "${TXT_RED}"'Unexpected failed functional test:' "${FTESTNAME}$TXT_CLEAR"; + TYPE=unexpected; + fi; + echo "${TARGET}:${TYPE}:${TESTNAME}:${FTESTNAME}" >> ${BUILDDIR}/test.failures.log; + done; + continue; + elif [ "x${IGNORE_TIMEDOUT_TEST_FAILURE}" = "xy" ] && + tail -n1 "${BUILDDIR}/${TESTNAME}.log" | grep -q 'exit status:'' 137'; then + echo -e "${TXT_RED}"'Ignoring Timed-out test:' "${TESTNAME}$TXT_CLEAR"; + echo -e -n "\e[97;100m"; + echo "Test failed due to a timeout. This is likely due to"; + echo "insufficient runner resources, and NOT a real failure."; + echo -e -n "$TXT_CLEAR"; + TYPE=timeout; + else + echo -e "${TXT_RED}"'Unexpected failed test:' "${TESTNAME}$TXT_CLEAR"; + echo -e -n "\e[97;100m"; + echo "Last 100 lines of ${BUILDDIR}/${TESTNAME}.log"; + tail -n 100 ${BUILDDIR}/${TESTNAME}.log; + echo -e -n "$TXT_CLEAR"; + TYPE=unexpected; + fi; + echo "${TARGET}:${TYPE}:${TESTNAME}" >> ${BUILDDIR}/test.failures.log; + done; +else + echo "No success canary and no test-suite.log, perhaps a timeout"; +fi; + +# If there are any unexpected errors return exit with error +FAILCOND="${TARGET}:unexpected"; +if [ "x${FAIL_ON_HARD_ERRORS}" = "xy" ]; then + FAILCOND="${TARGET}:(unexpected|error)"; +fi; + if grep -E -q "${FAILCOND}" ${BUILDDIR}/test.failures.log; then + RET=1; +fi; +end_log -n "process-test-$TARGET"; +# if [ -f $CI_PROJECT_DIR/${TARGET}.timedout-failures ]; then +# sed 's/^/${TARGET}:/' $CI_PROJECT_DIR/${TARGET}.timedout-failures +# >> $CI_PROJECT_DIR/build/test.timedout.log; +# fi; +# if [ -f $CI_PROJECT_DIR/${TARGET}.unexpected-failures ]; then +# sed 's/^/${TARGET}:/' $CI_PROJECT_DIR/${TARGET}.unexpected-failures +# >> $CI_PROJECT_DIR/build/test.failed.log; +# exit 1; +# fi; +exit ${RET:-0}; diff --git a/.ci/test.sh b/.ci/test.sh new file mode 100755 index 000000000..3f51de58b --- /dev/null +++ b/.ci/test.sh @@ -0,0 +1,106 @@ +#!/bin/bash + +set -eo pipefail + +# Environment variables +# SRCDIR: Path to source files +# BUILDDIR: Directory in which to place the build +# TARGET: Target to test +# TEST_ALL_TARGETS: If set to 'y', then all tests, even disabled ones will be +# run. +# DISABLED_TESTS: String of disabled tests delimited by new lines. +# TESTS_TIMEOUT: Maximum timeout to allow for 'make check'. Set this slightly +# below total job timeout to allow for cleanup and log packaging code. +# TEST_VERBOSITY: Number to set verbosity level to +# TEST_DATA_PREFIX: Filename prefix to use when saving test data. If not set, +# test data will not be saved. +# STRACE_TESTS: Set to 'y' to strace every tests for extreme debugging +# SHELL_TRACE: Set to 'y' to enable shell tracing + +if [ "x${SHELL_TRACE}" = "xy" ]; then + # If we export SHELL_OPTS, then all shells will be traced, most of which we do not want + SHELL_OPTS="-x"; + set -x; +fi; + +[ -f "$(dirname "$0")/functions.sh" ] && +. "$(dirname "$0")/functions.sh" + +[ -f "$(dirname "$0")/functions.$CI_TYPE.sh" ] && +. "$(dirname "$0")/functions.$CI_TYPE.sh" + +if [ "x${TEST_ALL_TARGETS}" != "xy" ] && ( echo "${DISABLED_TESTS}" | grep -q "^${TARGET}$" ); then + echo "Tests are disabled for target $TARGET"; + exit 0; +fi; + +start_log -c -n "test-$TARGET" "Testing $TARGET"; + +TESTTMPDIR="${TESTTMPDIR:-${TMPDIR:-/tmp}/grub-test-$TARGET}"; +COMP=xz; +STRACE_LOG="${BUILDDIR}/xxx.strace.$COMP"; + +if [ "x${STRACE_TESTS}" = "xy" ]; then + STRACE="strace -y -yy -f -v -s4096 -o >($COMP -9 > $STRACE_LOG)"; +fi; + +cat >${BUILDDIR}/log-tester.sh <<EOF +#!$SHELL $SHELL_OPTS + +STRACE_LOG="$STRACE_LOG"; +STRACE="$STRACE"; +TESTNAME=\$(basename "\$1"); +export SHELL_OPTS="$SHELL_OPTS"; +export TMPDIR=$TESTTMPDIR/\$TESTNAME; +mkdir -p "\$TMPDIR"; + +# if not a shell script, run normally +if [ "\$(head -c2 \$1)" == "#!" ]; then + TEST_SHELL="$(head -n1 "${BUILDDIR}/grub-shell" | tail -c+3 | tr -d ' ')"; + # Only turn on tracing if the shell if the one used by grub-shell + if head -n1 \$1 | grep -q \$TEST_SHELL; then + if [ "${TEST_VERBOSITY:-0}" -gt 0 ] || [ "x${SHELL_TRACE}" = "xy" ]; then + TEST_SHELL="\$TEST_SHELL -x"; + fi; + fi; +fi; +eval \${STRACE/xxx/\$TESTNAME} \$TEST_SHELL "\$@" && { rm -rf \${STRACE_LOG/xxx/\$TESTNAME} \$TMPDIR; } +EOF +export LOG_COMPILER="${BUILDDIR}/log-tester.sh"; +chmod +x ${BUILDDIR}/log-tester.sh; +echo -n -e "${TXT_YELLOW}"; +cat ${BUILDDIR}/log-tester.sh; +echo -n -e "${TXT_CLEAR}"; + +if [ "${TEST_VERBOSITY:-0}" -gt 0 ]; then + export V=${TEST_VERBOSITY}; + export GRUB_SHELL_DEFAULT_DEBUG=${TEST_VERBOSITY}; + export GRUB_FSTEST_DEFAULT_DEBUG=${TEST_VERBOSITY}; +fi; + +if [ -n "$GRUB_SHELL_DEFAULT_DEBUG" ]; then + ### Turning on grub-shell debugging, turns on a graphical qemu window. + ### But we want to run in headless environments, so turn off graphics. + export GRUB_QEMU_OPTS="-nographic -monitor file:/dev/null -fw_cfg name=etc/sercon-port,string=0"; +fi; +TIMEOUT=${TESTS_TIMEOUT}; + +function func_strace () { + strace -y -yy -f -v -s4096 -o >($COMP -9 > ${BUILDDIR}/test.strace.$RANDOM.$COMP) "$@"; +}; +':' STRACE="func_strace"; + +':' $STRACE ${TIMEOUT:+timeout $TIMEOUT} make -C ${BUILDDIR} -j$JOBS SUBDIRS= check || RET=$?; +${TIMEOUT:+timeout $TIMEOUT} make -C ${BUILDDIR} -j$JOBS SUBDIRS= check || RET=$?; +':' timeout $TIMEOUT $STRACE make -C ${BUILDDIR} -j$JOBS SUBDIRS= LOG_COMPILER="${BUILDDIR}/log-tester.sh" check || RET=$?; +[ "$RET" -eq 124 -o "$RET" -eq 137 ] && echo "ERROR":" make check timed out"; +':' gzip -9 ${BUILDDIR}/test.strace.*; + +end_log -n "test-$TARGET"; + +[ "$RET" -eq 0 ] && touch ${BUILDDIR}/test.success || :; +if [ -n "$TEST_DATA_PREFIX" ]; then + tar -S -cJf "${TEST_DATA_PREFIX}.tar.xz" -C $(dirname "$TESTTMPDIR") $(basename "$TESTTMPDIR"); +fi + +exit $RET; diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 000000000..d4fc2337e --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,1184 @@ +# SPDX-License-Identifier: GPL-3.0+ +# Copyright Glenn Washburn <developm...@efficientek.com> +# +# Build GRUB on GitLab CI - https://www.gitlab.org/ +# + +workflow: + rules: + - if: '$CI_PIPELINE_SOURCE == "web"' + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + - if: $CI_MERGE_REQUEST_IID + - if: $CI_COMMIT_TAG + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + - when: never + +# This allows running CI/CD pipelines for merge requests +# include: +# - template: 'Workflows/MergeRequest-Pipelines.gitlab-ci.yml' + +# Variables that can be set by runners: +# CONFIGURE_OPTS - Extra configure options +# TESTS_TIMEOUT - Set timeout for make check tests (see man timeout) +# FAIL_ON_HARD_ERRORS - If set, Hard errors will cause a failure +# DISABLE_ALL_TESTS - If set, make check tests will be disabled +# TEST_ALL_TARGETS - If set, failing targets which have been marked disabled will be enabled +# TEST_VERBOSITY - Verbosity level when running tests: 1-3 +# STRACE_TESTS - If set, strace individual tests. WARNING: This will cause testing to take much longer +# which can cause some test to fail by timing out or having incorrect timing +# SHELL_TRACE - If set, turn on shell tracing of everything. NOTE: TEST_VERBOSITY=3 turns on more targeted +# shell tracing + +variables: + CI_TYPE: gitlab + # CI_DEBUG_TRACE: 'true' + # GIT_CLONE_PATH: $CI_BUILDS_DIR/$CI_CONCURRENT_ID/grub-$CI_COMMIT_SHORT_SHA + GIT_CLONE_PATH: $CI_BUILDS_DIR/grub-${CI_COMMIT_SHORT_SHA}.git + GIT_STRATEGY: fetch + + # Include all cross toolchain paths, so we can just call them later down. + #PATH: "/tmp/qemu-install/bin:/usr/bin:/bin:$CROSS_DIR/gcc-8.1.0-nolibc/aarch64-linux/bin:$CROSS_DIR/gcc-8.1.0-nolibc/arm-linux-gnueabi/bin:$CROSS_DIR/gcc-8.1.0-nolibc/ia64-linux/bin:$CROSS_DIR/gcc-8.1.0-nolibc/mips64-linux/bin:$CROSS_DIR/gcc-8.1.0-nolibc/powerpc64-linux/bin:$CROSS_DIR/gcc-8.1.0-nolibc/riscv32-linux/bin:$CROSS_DIR/gcc-8.1.0-nolibc/riscv64-linux/bin:$CROSS_DIR/gcc-8.1.0-nolibc/sparc64-linux/bin" +# PACKAGE_CACHE: $CI_BUILDS_DIR/pkgs + PACKAGE_CACHE: $CI_PROJECT_DIR/pkgs + CROSS_DIR: $CI_PROJECT_DIR/cross + SRCDIR: $CI_PROJECT_DIR + + ### Below are values that are meant to be configurable + CROSS_VERSION: + # value: "4.8.0" + # value: "4.8.5" + # value: "4.9.0" + # value: "4.9.4" + # value: "5.5.0" + # Start having riscv32/64 builds + # value: "7.3.0" + # value: "7.5.0" + # value: "8.1.0" + # value: "9.3.0" + value: "10.1.0" + description: "Version of cross-compiler to use (suggested: 10.1.0, 9.3.0, 8.1.0, 7.5.0 [riscv32/64 builds fail before 7.3.0])" + + PACKAGES: | + libsdl1.2-dev + xz-utils + lzop + ovmf + python + qemu-system + unifont + wget + libfreetype6-dev + libdevmapper-dev + liblzma-dev + libfuse-dev + libzfslinux-dev + make + autoconf + automake + autopoint + git + bison + flex + gettext + pkg-config +# gcc-9-multilib + + CONFIGURE_OPTS: + value: "--enable-boot-time" + description: "Extra options to pass to configure" + + BUILD_ALL_TARGETS: + value: "n" + description: "If set 'y', all targets defined in the build matrix will be built unconditionally." + + DISABLED_BUILDS: + value: | + ### qemu-system-mips64: Could not load MIPS bios 'mips_bios.bin', + ### and no -kernel argument was specified + mipsel-arc + mipsel-qemu_mips + mipsel-loongson + description: > + A list of targets for which builds are disabled, one per line. These + default targets are disabled generally because they do not work yet. + Lines beginning with '#' may be used as comments and are ignored. Some + hints as to why the tests fail are included in comment lines. Patches + which get these tests working are very welcome. + + TESTS_TIMEOUT: + value: "" + description: > + Set timeout for completion of all make check tests (see man timeout + duration argument). This is most useful when the make check job is + taking longer than the job timeout configured by the runner. In this + case, set this to 5-10 minutes less than runner timeout so that there + is time to package up the logs for debugging. + + IGNORE_TIMEDOUT_TEST_FAILURE: + value: "y" + description: > + If set to 'y', tests which fail due to a timeout in grub-shell of qemu + will not be counted as a failure. These failures are almost always the + result of insufficient runner resources to complete the execution of qemu + within the timeout period. + + FAIL_ON_HARD_ERRORS: + value: "n" + description: > + If set to 'y', hard errors will cause a failure. A hard error indicates + that the test was not able to be run (eg. dependencies not found), and + as such a hard error does not constitute a true failure of the test. + + DISABLE_ALL_TESTS: + value: "n" + description: > + If set to 'y', make check tests will be disabled. Effectively only test + whether the building is successful for configured platforms. + + TEST_ALL_TARGETS: + value: "n" + description: "If set 'y', failing targets which have been marked disabled will be enabled" + + MAKE_ALL_IMAGE_TARGETS: + value: "n" + description: "If set 'y', all target image formats will be built unconditionally" + + TEST_VERBOSITY: + value: "2" + description: "Verbosity level when running tests [supported values: '', 1, 2]" + + STRACE_TESTS: + value: "n" + description: > + If set, strace individual tests. WARNING: This will cause testing to take + much longer which can cause some test to fail by timing out or having + incorrect timing + + SHELL_TRACE: + value: "n" + description: > + If set, turn on shell tracing of everything. NOTE: TEST_VERBOSITY=3 turns + on more targeted shell tracing + + GRUB_QEMU_OPTS: + value: "" + description: "Add options to qemu when used in tests (ex: -m size=64M)" + + GRUB_SHELL_DEFAULT_TIMEOUT: + value: "600" + description: > + Set grub-shell timeout for qemu instance (see man timeout duration + argument). The timeout has been raised from the default of 60 seconds + in the grub-shell script to 600 seconds because CI runners may run in + a slow virtual machine. This may need to be raised further if grub-shell + is failing with a timeout (exit code 137). + + EXPECTED_FAILURES: + value: | + ### Usually this test passes, but occasionally the virtual machine is + ### too slow and the time delta threshold is exceeded. So allow failures. + grub_cmd_sleep + ### Need version of mkfs.minix which has -B option, where to find it? + # SKIP: minixfs_test + ### TODO: Gitlab shared runners use coreos kernel 4.19.78, perhaps we + ### can compile needed modules and load at runtime, or use our own + ### dedicated runner with these modules loaded. + ### Need mac-roman kernel module + # SKIP: hfs_test + ### Can not load the zfs modules in gitlab shared runners + zfs_test + reiserfs_test + f2fs_test + nilfs2_test + hfsplus_test + jfs_test + ### This should not fail, but ignoring for until it gets fixed + ### ./grub-fstest: error: cannot open `(loop0)/sym': invalid symlink. + udf_test + ### sparc64-ieee1275 fails these tests, but the rest should succeed + ### https://marc.info/?i=20201219002902.4490b844%20()%20crass-HP-ZBook-15-G2 + sparc64-ieee1275:grub_script_expansion + sparc64-ieee1275:gzcompress_test + sparc64-ieee1275:grub_func_test + sparc64-ieee1275:file_filter_test + ### Not sure why this is failing, but not qemu output + powerpc-ieee1275:pseries_test + ### This test is skipped by sparc64-ieee1275 due to OpenBIOS not + ### implementing RTC, could this be the same for powerpc-ieee1275? + powerpc-ieee1275:grub_cmd_date + description: > + A list of make check tests allowed to fail, one per line. Test may be + prefixed with "$ARCHITECTURE-$PLATFORM:" to only apply to certain + targets. Lines beginning with '#' may be used as comments and are + ignored. + + EXPECTED_FUNCTIONAL_FAILURES: + value: | + ### These have not worked for a very long time + videotest_checksum + gfxterm_menu + cmdline_cat_test + ### Sometimes the machine the test are running on are slow causing + ### timing tests to fail. + sleep_test + description: > + A list of functional tests from the grub_func_test make check that are + allowed to fail, one per line. Lines beginning with '#' may be used as + comments and are ignored. + + DISABLED_TESTS: + value: | + ### Failing on unreachable target error when invoking grub-mkimage + riscv32-efi + ### Need to get coreboot ROM and cbfstool + i386-coreboot + ### qemu-system-mips64: Could not load MIPS bios 'mips_bios.bin', + ### and no -kernel argument was specified + mips-arc + mips-qemu_mips + mipsel-arc + mipsel-qemu_mips + ### Need an i386 built open firmware image, otherwise uses regular bios + ### and stalls with "Booting from Hard Disk..." + i386-ieee1275 + ### Qemu logs "Boot failed: Could not read from CDROM (code 0004)", which + ### according to https://lists.gnu.org/archive/html/qemu-devel/2008-06/msg00476.html + ### means "cd is not eltorito (BRVD)" + i386-multiboot + ### TODO: These have no support in grub-shell and use the default + ### qemu-system-i386, which is clearly not correct + arm-coreboot + arm-uboot + ia64-efi + riscv64-efi + ### Also not supported in grub-shell + i386-xen + i386-xen_pvh + x86_64-xen + description: > + A list of targets for which make check is not run, one per line. These + targets are disabled because tests mostly do not work. Lines beginning + with '#' may be used as comments and are ignored. Some hints as to why + the tests fail are included in comment lines. Patches which get these + tests working are very welcome. + + DISABLED_IMAGES: + value: | + ### Failing on unreachable target error when invoking grub-mkimage + riscv32-efi + ### FIXME: There is a bug in grub-mkimage which causes the modules + ### directory for these target formats to be not found + arm-coreboot-vexpress + arm-coreboot-veyron + description: > + A list of target formats as accepted by grub-mkimage for which images + are not built. These target formats are disabled generally because they + do not work. Lines beginning with '#' may be used as comments and are + ignored. Some hints as to why the tests fail are included in comment + lines. Patches which get these tests working are very welcome. + +default: + image: ubuntu:xenial + # image: ubuntu:focal +# cache: +# # cache all untracked files +# untracked: true +# # policy: pull +# # Each branch will use the same cache +# key: ${CI_COMMIT_REF_SLUG}-${CI_JOB_IMAGE} +# paths: +# - ccache/ +# # - gnulib + +stages: + - cache + - setup + - build + - install + - test + +.needs-cross-path: + variables: +# PATH: "$CI_BUILDS_DIR/pkgroot/sbin:$CI_BUILDS_DIR/pkgroot/usr/sbin:$CI_BUILDS_DIR/pkgroot/bin:$CI_BUILDS_DIR/pkgroot/usr/bin:$CROSS_DIR/gcc-$CROSS_VERSION-nolibc/$CROSS_TARGETS/bin:/usr/sbin:/sbin:/usr/bin:/bin" + PATH: "$CROSS_DIR/gcc-$CROSS_VERSION-nolibc/$CROSS_TARGETS/bin:/usr/sbin:/sbin:/usr/bin:/bin" + +.print_debug_info: + before_script: &print_debug_info-before_script + - echo -e "section_start:`date +%s`:debug_info[collapsed=true]\r\e[0KDebug Information" + - uname -a + - ls -l /proc/$$/exe + - export + - set + - dpkg-query -l + - find /lib/modules || ':' + - '`which xorriso || echo ":"` --version' + - find /usr/lib/locale || ':' + - ls -lR /usr/share/OVMF* || ':' + - echo -e "section_end:`date +%s`:debug_info\r\e[0K" + +.needs-packages: &needs-packages + before_script: + - add-apt-repository -yu ppa:ubuntu-toolchain-r/test +# - apt-get update -yqq + - apt-get -o Dir::Cache::archives=$PACKAGE_CACHE install -yqq $PACKAGES + +.build:ccache: &build-ccache + # The ccache dir needs to be cached else its pointless, but set the path in + # the global cache, because merging cache path entries isn't possible + before_script: &build-ccache-before_script + - apt-get update -yqq + - apt-get -o Dir::Cache::archives=$PACKAGE_CACHE install -yqq ccache + # CCache Config + - mkdir -p ccache/bin + - export CCACHE_BASEDIR=${PWD} + - export CCACHE_DIR=${PWD}/ccache + - export CCACHE_CONFIGPATH=${CCACHE_DIR}/ccache.conf + - export CCACHE_HARDLINK=true + - export CCACHE_TEMPDIR=/tmp/.ccache + - export CCACHE_LOGFILE="${BUILDDIR}/ccache.log" + # - export CCACHE_PATH="$CROSS_DIR/gcc-$CROSS_VERSION-nolibc/$CROSS_TARGETS/bin" + - export CC="ccache gcc" + # - export CC="strace -y -yy -v -s4096 -o "${BUILDDIR}/ccache.strace.log" ccache gcc" + # - export CC="ccache ${CROSS_TARGETS}-gcc" + # - export CC=/usr/lib/ccache/gcc + - ccache --show-stats + # - ln -s /usr/bin/ccache /usr/local/bin/gcc + # ln -s /usr/bin/ccache /usr/local/bin/g++ + # ln -s /usr/bin/ccache /usr/local/bin/cc + # ln -s /usr/bin/ccache /usr/local/bin/c++ + after_script: &build-ccache-after_script + # The cache should be large enough to be useful but it shouldn't take + # too long to restore+save each run. + # NOTE: Envvars appear not to be preserved in after_script, so we must + # set the CCACHE_DIR + - export CCACHE_DIR=${PWD}/ccache + - ccache --show-stats + - ccache --max-size $( du --summarize --block-size=1M "$CI_PROJECT_DIR/build" | awk '{printf ("%dM", $1 * 1.5)}' ) + + +.install-packages: + before_script: &install-packages-before_script + # This is needed because tzdata is requesting user input to set the timezone + # and stalling the script + - TZ=UTC; ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + - mkdir -vp $PACKAGE_CACHE/partial + - start_log -c -n "install_env" "Install pkgs log" + - apt-get update -y + - PACKAGES=$(grep -v '^#' <<<"$PACKAGES"); + echo -e 'Installing packages:' "$PACKAGES"; + apt-get -o Dir::Cache::archives=$PACKAGE_CACHE install -yqq $PACKAGES; + - end_log -n "install_env" + +# These are packages or versions which we can not find in the default repo +.install-non-repo-packages: + before_script: &install-non-repo-packages-before_script + - start_log -c -n "install_non_repo_env" "Install non-repo pkgs log" + # We need to get at least bionic's version of e2fsprogs because before that the + # version is < 1.43, which does not have the encrypt option + # - apt-get -t bionic -o Dir::Cache::archives=$PACKAGE_CACHE install -yqq e2fsprogs + - if dpkg --compare-versions "$(dpkg-query -l e2fsprogs |tail -n1| awk '{print $3;}')" '<=' 1.43; then + wget -nv -P $PACKAGE_CACHE/archives/e2fsprogs + http://security.ubuntu.com/ubuntu/pool/main/e/e2fsprogs/e2fsprogs_1.44.1-1ubuntu1.3_amd64.deb + http://security.ubuntu.com/ubuntu/pool/main/e/e2fsprogs/e2fslibs_1.44.1-1ubuntu1.3_amd64.deb + http://security.ubuntu.com/ubuntu/pool/main/e/e2fsprogs/libext2fs2_1.44.1-1ubuntu1.3_amd64.deb + http://security.ubuntu.com/ubuntu/pool/main/e/e2fsprogs/libcom-err2_1.44.1-1ubuntu1.3_amd64.deb + http://security.ubuntu.com/ubuntu/pool/main/e/e2fsprogs/libcomerr2_1.44.1-1ubuntu1.3_amd64.deb; + apt-get install -y "$PACKAGE_CACHE"/archives/e2fsprogs/*.deb; + fi + - end_log -n "install_non_repo_env" + +# These are extra qemu packages that we can't get through the installer +.install-extra-qemu-packages: + variables: &install-extra-qemu-packages-vars + DEB_URLS: | + http://mirrors.kernel.org/ubuntu/pool/main/e/edk2/ovmf-ia32_2020.11-2_all.deb + http://mirrors.kernel.org/ubuntu/pool/main/e/edk2/qemu-efi-arm_2020.11-2_all.deb + http://mirrors.kernel.org/ubuntu/pool/main/e/edk2/qemu-efi-aarch64_2020.11-2_all.deb + before_script: &install-extra-qemu-packages-before_script + - for DEB_URL in $DEB_URLS; do + DEB_PATH="$PACKAGE_CACHE/archives/$(basename "$DEB_URL")"; + if [ ! -f "$DEB_PATH" ]; then + wget -nv -P $PACKAGE_CACHE/archives $DEB_URL; + fi; + dpkg -i $DEB_PATH; + done + # TODO: i386-efi: Unfortunately the non-Ubuntu package ovmf-ia32_2020.11-2_all.deb + # package installs OVMF images that are not usable by qemu. This is the + # most official looking build found. Perhaps we should build our own? + - wget -nv -t 3 -O "${SRCDIR}/OVMF-ia32.fd" + https://github.com/retrage/edk2-nightly/raw/master/bin/RELEASEIa32_OVMF.fd + - cp -va /usr/share/qemu-efi-aarch64/QEMU_EFI.fd "${SRCDIR}/OVMF-aarch64.fd" + - cp -va /usr/share/AAVMF/AAVMF32_CODE.fd "${SRCDIR}/OVMF-arm.fd" + +.pkg-cache: + cache: + # Each branch will use the same cache + key: ${CI_COMMIT_REF_SLUG}-${CI_JOB_IMAGE} + # Set to push pull so that changes update the cache + policy: pull-push + paths: + - $PACKAGE_CACHE + +.setup-loopback: + before_script: &setup-loopback-before_script + # Gitlab docker runners do not setup the loopback devices, but it runs + # our code in a privileged mode so we can set it up ourselves + - if [ ! -e /dev/loop-control ]; then + mknod -m 0660 /dev/loop-control c 10 237; + fi + - for i in $(seq 0 64); do + mknod -m 0660 "/dev/loop$i" b 7 "$i"; + done + +.all-targets: + parallel: &target_matrix + matrix: + - ARCH: x86_64 + GRUB_TARGETS: [x86_64-efi, x86_64-xen] + CROSS_TARGETS: [x86_64-linux] + - ARCH: i386 + GRUB_TARGETS: [i386-coreboot, i386-efi, i386-ieee1275, i386-multiboot, i386-pc, i386-qemu, i386-xen, i386-xen_pvh] + CROSS_TARGETS: [i386-linux] + - ARCH: powerpc + GRUB_TARGETS: [powerpc-ieee1275] + CROSS_TARGETS: [powerpc64-linux] + - ARCH: sparc64 + GRUB_TARGETS: [sparc64-ieee1275] + CROSS_TARGETS: [sparc64-linux] + - ARCH: ia64 + GRUB_TARGETS: [ia64-efi] + CROSS_TARGETS: [ia64-linux] + - ARCH: mips + GRUB_TARGETS: [mips-arc, mips-qemu_mips, mipsel-arc, mipsel-qemu_mips, mipsel-loongson] + CROSS_TARGETS: [mips64-linux] + - ARCH: arm + GRUB_TARGETS: [arm-coreboot, arm-efi, arm-uboot] + CROSS_TARGETS: [arm-linux-gnueabi] + - ARCH: arm64 + GRUB_TARGETS: [arm64-efi] + CROSS_TARGETS: [aarch64-linux] + - ARCH: riscv32 + GRUB_TARGETS: [riscv32-efi] + CROSS_TARGETS: [riscv32-linux] + - ARCH: riscv64 + GRUB_TARGETS: [riscv64-efi] + CROSS_TARGETS: [riscv64-linux] + +.cache:packages: + stage: cache + extends: .pkg-cache + script: + # Install required package dependencies with package manager apt +# - apt-get install -o Dir::Cache::archives=$PACKAGE_CACHE -yqq software-properties-common +# - add-apt-repository -yu ppa:ubuntu-toolchain-r/test + - apt-get update -yqq + - mkdir -vp $PACKAGE_CACHE/partial +# - apt-get -o Dir::Cache::archives=$PACKAGE_CACHE install --download-only -yqq $PACKAGES + +.process-tests: + script: &process-tests-script + - export target=$GRUB_TARGETS + - test "x${SHELL_TRACE}" = "xy" && set -x; + if [ -f ${BUILDDIR}/test.success ]; then + exit 0; + elif [ "x${DISABLE_ALL_TESTS}" = "xy" ] || [ "x${TEST_ALL_TARGETS}" != "xy" -a -z "${DISABLED_TESTS##*$'\n'${target}$'\n'*}" ]; then + echo "No test output processing because testing was disabled"; + exit 0; + fi; + + echo -e "section_start:`date +%s`:process-test-$target[collapsed=true]\r\e[0KProcessing test output of $target"; + if [ -f ${BUILDDIR}/test-suite.log ]; then + FAILURE_COND='^FAIL:'; + if [ "x${FAIL_ON_HARD_ERRORS}" = "xy" ]; then + FAILURE_COND='^(FAIL|ERROR):'; + fi; + + ( grep -E "$FAILURE_COND" ${BUILDDIR}/test-suite.log || ':' ) | + while read _ TESTNAME; do + if echo "${EXPECTED_FAILURES}" | grep -qE "^(${TESTNAME}|${target}:${TESTNAME})$"; then + echo 'Expected failed test:' "$TESTNAME"; + continue; + fi; + + if [ "$TESTNAME" = "grub_func_test" ]; then + FAILURE=0; + grep -E 'test:'' FAIL' ${BUILDDIR}/${TESTNAME}.log | sort -u | + while read FTESTNAME STATUS; do + FTESTNAME=${FTESTNAME%:*}; + if echo "${EXPECTED_FUNCTIONAL_FAILURES}" | grep -qE "^(${FTESTNAME}|${target}:${FTESTNAME})$"; then + echo 'Expected failed functional test:' "$FTESTNAME"; + else + echo -e "${TXT_RED}"'Unexpected failed functional test:' "${FTESTNAME}${TXT_CLEAR}"; + FAILURE=1; + fi; + done; + [ "${FAILURE}" -eq 0 ] && continue; + fi; + + if [ "x${IGNORE_TIMEDOUT_TEST_FAILURE}" = "xy" ] && + tail -n1 "${BUILDDIR}/${TESTNAME}.log" | grep -q 'exit status:'' 137'; then + echo -e "${TXT_RED}"'Ignoring Timed-out test:' "${TESTNAME}${TXT_CLEAR}"; + echo -e -n "\e[97;100m"; + echo "Test failed due to a timeout. This is likely due to" + "insufficient runner resources, and NOT a real failure."; + echo -e -n "${TXT_CLEAR}"; + echo "${TESTNAME}" >> $CI_PROJECT_DIR/${target}.timedout-failures; + else + echo -e "${TXT_RED}"'Unexpected failed test:' "${TESTNAME}${TXT_CLEAR}"; + echo -e -n "\e[97;100m"; + echo "Last 100 lines of ${BUILDDIR}/${TESTNAME}.log"; + tail -n 100 ${BUILDDIR}/${TESTNAME}.log; + echo -e -n "${TXT_CLEAR}"; + echo "${TESTNAME}" >> $CI_PROJECT_DIR/${target}.unexpected-failures; + fi; + done; + else + echo "No success canary and no test-suite.log, perhaps a timeout"; + fi; + echo -e "section_end:`date +%s`:process-test-$target\r\e[0K"; + if [ -f $CI_PROJECT_DIR/${target}.timedout-failures ]; then + sed 's/^/${target}:/' $CI_PROJECT_DIR/${target}.timedout-failures + >> $CI_PROJECT_DIR/build/test.timedout.log; + fi; + if [ -f $CI_PROJECT_DIR/${target}.unexpected-failures ]; then + sed 's/^/${target}:/' $CI_PROJECT_DIR/${target}.unexpected-failures + >> $CI_PROJECT_DIR/build/test.failed.log; + exit 1; + fi; + exit 0; + +.cross-compilers: + variables: + PACKAGES: | + wget + xz-utils + CROSS_TARGETS: &cross_targets > + x86_64-linux + i386-linux + powerpc64-linux + sparc64-linux + ia64-linux + mips64-linux + arm-linux-gnueabi + aarch64-linux + riscv32-linux + riscv64-linux + before_script: &cross_compilers-setup_script + - mkdir -pv $CROSS_DIR + - export HOST_ARCH=`uname -m` + # These give us binaries like $CROSS_DIR/gcc-8.1.0-nolibc/ia64-linux/bin/ia64-linux-gcc + - for i in $CROSS_TARGETS; do + [ -d "${CROSS_BIN_DIR}" ] || + wget -nv -t 3 -O - https://mirrors.kernel.org/pub/tools/crosstool/files/bin/$HOST_ARCH/$CROSS_VERSION/$HOST_ARCH-gcc-$CROSS_VERSION-nolibc-$i.tar.xz | tar xJ -C "$CROSS_DIR"; + export CROSS_BIN_DIR="$CROSS_DIR/gcc-$CROSS_VERSION-nolibc/$CROSS_TARGETS/bin"; + export PATH="${CROSS_BIN_DIR}:${PATH}"; + test -d "$CROSS_BIN_DIR" || exit 1; + done + +.setup:cross-compilers: + stage: setup + when: manual + extends: + - .pkg-cache + - .install-packages + - .cross-compilers + artifacts: + name: "${CI_JOB_NAME}-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA}-${GRUB_TARGETS}" + paths: + - $CROSS_DIR + expire_in: 4 hours + + +setup:bootstrap: + stage: setup + extends: + - .pkg-cache + artifacts: + name: "${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA}-bootstrap" + untracked: true + expire_in: 1 week + exclude: + - $PACKAGE_CACHE +# needs: +# - cache:packages + variables: + PACKAGES: | + python + autoconf + automake + autopoint + git + gettext + pkg-config + locales + language-pack-en + kmod + wget + before_script: + - . ${SRCDIR}/.ci/functions.gitlab.sh + - *install-packages-before_script + script: + - *print_debug_info-before_script + - cat /proc/filesystems + # - modprobe hfsplus + # - modprobe f2fs + # - modprobe reiserfs + - start_log -c -n "bootstrap" "Bootstrap log"; + # We cache the bootstrap, so if the bootstrap canary exists, don't run the bootstrap + - '[ -f .bootstrap.finished ] || ( ./bootstrap && touch .bootstrap.finished )' + - end_log -n "bootstrap"; + +.build:compile: + stage: build + when: manual + extends: + - .needs-cross-path + - .pkg-cache + - .install-packages + artifacts: + name: "${CI_JOB_NAME}-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA}-${GRUB_TARGETS}" + paths: + - $CI_PROJECT_DIR/build + - $CI_PROJECT_DIR/install + - $CROSS_DIR + exclude: + - $CI_PROJECT_DIR/build/**/*.o + expire_in: 4 hours + needs: + - setup:cross-compilers + - setup:bootstrap + variables: +# CONFIGURE_OPTS: "--disable-silent-rules" + PACKAGES: | + unifont + libfreetype6-dev + libdevmapper-dev + liblzma-dev + libfuse-dev + libzfslinux-dev + python + make + automake + bison + flex + gettext + pkg-config +# libsdl1.2-dev +# gcc-9-multilib + script: + - echo -e "section_start:`date +%s`:debug_info\r\e[0KDebugging info" + - 'echo "pwd: `pwd`"' + - 'echo "PATH: $PATH"' + - 'echo "CI_BUILDS_DIR: $CI_BUILDS_DIR"' + - 'echo "CI_PROJECT_DIR: $CI_PROJECT_DIR"' + - echo -e "section_start:`date +%s`:vars[collapsed=true]\r\e[0KEnvironment Variables" + - export + - echo -e "section_end:`date +%s`:vars\r\e[0K" + - echo -e "section_end:`date +%s`:debug_info\r\e[0K" + # Build all selected GRUB targets. + - for target in $GRUB_TARGETS; do + set -x; + while [ -z ${PATH%%*\$*} ]; do + eval "PATH=$PATH"; + done; + plat=${target#*-}; + arch=${target%-*}; + case "$arch" in + arm64) arch=aarch64-linux;; + arm) arch=arm-linux-gnueabi;; + mipsel) arch=mips64-linux;; + powerpc) arch=powerpc64-linux;; + ia64|riscv32|riscv64|sparc64) arch=$arch-linux;; + i386|x86_64|mips) ;; + *) echo "Unkown architecture":" $arch"; exit 1;; + esac; + case "$target" in + sparc64-ieee1275) tformats="aout cdcore raw" ;; + Xmipsel-qemu_mips) tformats="elf flash" ;; + Xarm-coreboot) tformats="vexpress veyron" ;; + *) tformats='~' ;; + esac; + echo -e "section_start:`date +%s`:build-$target\r\e[0KBuilding $target"; + builddir=$CI_PROJECT_DIR/build/obj-$target; + installdir=$CI_PROJECT_DIR/install/grub-$target; + mkdir -pv $builddir $installdir; + JOBS=`getconf _NPROCESSORS_ONLN 2> /dev/null || echo 1`; + [ "$JOBS" == 1 ] || JOBS=$(($JOBS + 1)); + ( + cd $builddir && + $CI_PROJECT_DIR/configure --target=$arch --with-platform=$plat --prefix=$installdir $CONFIGURE_OPTS && + make -j$JOBS && + make -j$JOBS install; + for tfmt in $tformats; do + [ -z "${tfmt%~*}" ] || + $builddir/grub-mkimage -p / -O $target${tfmt:+-$tfmt} -o /tmp/grub-$target${tfmt:+-$tfmt} echo reboot normal; + done; + ) || ( + cd $builddir; + echo -e "section_start:`date +%s`:config_log-$target[collapsed=true]\r\e[0KBuild fail logs $target"; + cat config.log; + echo -e "section_end:`date +%s`:config_log-$target\r\e[0K"; + false; + ); + echo -e "section_end:`date +%s`:build-$target\r\e[0K"; + done + parallel: + <<: *target_matrix + +..build:install: + stage: install + allow_failure: true + when: manual + extends: + - .needs-cross-path + - .pkg-cache + - .install-packages + artifacts: + name: "${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA}-${GRUB_TARGETS}" + paths: + - $CI_PROJECT_DIR/install + expire_in: 4 hours + needs: + - setup:bootstrap + - build:compile + variables: + PACKAGES: | + python + make + automake + pkg-config +# unifont + script: + - echo -e "section_start:`date +%s`:vars[collapsed=true]\r\e[0KEnvironment Variables" + - export + - echo -e "section_end:`date +%s`:vars\r\e[0K" + # Build all selected GRUB targets. + - for target in $GRUB_TARGETS; do + while [ -z ${PATH%%*\$*} ]; do + eval "PATH=$PATH"; + done; + echo -e "section_start:`date +%s`:install-$target\r\e[0KInstalling $target"; + builddir=$CI_PROJECT_DIR/build/obj-$target; + installdir=$CI_PROJECT_DIR/install/grub-$target; + mkdir -pv $builddir $installdir; + JOBS=`getconf _NPROCESSORS_ONLN 2> /dev/null || echo 1`; + [ "$JOBS" == 1 ] || JOBS=$(($JOBS + 1)); + ( + cd $builddir && + make -j$JOBS install + ); + echo -e "section_end:`date +%s`:install-$target\r\e[0K"; + done + parallel: + <<: *target_matrix + +.build:rescue: + stage: install + allow_failure: true + when: manual + extends: + - .install-packages + artifacts: + name: "${CI_JOB_NAME}-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA}" + paths: + - $CI_PROJECT_DIR/rescue + expire_in: 4 hours + needs: + - build:compile + variables: + # EFI platforms use mformat from mtools when building the rescue image + PACKAGES: | + xorriso + mtools + script: + - echo -e "section_start:`date +%s`:vars[collapsed=true]\r\e[0KEnvironment Variables" + - export + - echo -e "section_end:`date +%s`:vars\r\e[0K" + # Build all selected GRUB targets. + - for target in $GRUB_TARGETS; do + echo -e "section_start:`date +%s`:mkrescue-$target\r\e[0KInstalling $target"; + builddir=$CI_PROJECT_DIR/build/obj-$target; + rescuedir=$CI_PROJECT_DIR/rescue; + mkdir -pv $rescuedir; + ( + echo -e "insmod normal; normal" > grub.cfg; + $builddir/grub-mkrescue -v -o $rescuedir/grub-rescue-$target.iso boot/grub/grub.cfg=grub.cfg; + ); + echo -e "section_end:`date +%s`:mkrescue-$target\r\e[0K"; + done + parallel: + <<: *target_matrix + +.build:docs: + stage: build + allow_failure: true + extends: + - .install-packages + variables: + # Use tex packages to build documentation + PACKAGES: | + texinfo + texlive + script: + - make html dvi pdf info ps man + +.test-all: +# image: ubuntu:focal + stage: test + when: manual + extends: + - .needs-cross-path + - .install-packages + artifacts: + name: test-artifacts + when: on_failure + paths: + - $CI_PROJECT_DIR/build/obj-*/*.{log,trs} + needs: + - setup:bootstrap + - build:compile + variables: + PACKAGES: | + ovmf + qemu-system + python + make + automake + pkg-config + bison + flex + wamerican + tar + cpio + gzip + lzop + xz-utils + parted + xorriso + util-linux + squashfs-tools + zfsutils-linux + dosfstools + exfat-utils + ntfs-3g + e2fsprogs + btrfs-progs + xfsprogs + hfsprogs + jfsutils + reiserfsprogs + udftools + nilfs-tools + f2fs-tools + genromfs + attr +# libsdl1.2-dev +# unifont +# libfreetype6-dev +# libdevmapper-dev +# liblzma-dev +# libfuse-dev +# libzfslinux-dev +# autoconf +# autopoint +# git +# gettext + script: + - for target in $GRUB_TARGETS; do + set -x; + while [ -z ${PATH%%*\$*} ]; do + eval "PATH=$PATH"; + done; + plat=${target#*-}; + plat=${plat%-*}; + arch=${target%%-*}; + echo -e "section_start:`date +%s`:test-$target[collapsed=true]\r\e[0KTesting $target"; + builddir=$CI_PROJECT_DIR/build/obj-$target; + installdir=$CI_PROJECT_DIR/install/grub-$target; + mkdir -pv $builddir $installdir; + ls $builddir; + JOBS=`getconf _NPROCESSORS_ONLN 2> /dev/null || echo 1`; + [ "$JOBS" == 1 ] || JOBS=$(($JOBS + 1)); + make -C $builddir -j$JOBS TMPDIR=/tmp SUBDIRS= V=3 check || ( + ls $builddir; + false; + ); + echo -e "section_end:`date +%s`:test-$target\r\e[0K"; + done + parallel: + <<: *target_matrix + +# Build and test everything in one (matrix) job +build:allinone: + stage: build + extends: + - .all-targets + - .install-packages + cache: + # Each architecture will use the same ccache + key: ${ARCH}-${CI_JOB_IMAGE} + paths: + - ccache/ + - $PACKAGE_CACHE + artifacts: + name: "${CI_JOB_NAME}-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA}-${GRUB_TARGETS}" + # when: on_failure + when: always + paths: &target-archive-paths + - $CI_PROJECT_DIR/build/obj-$GRUB_TARGETS/config.h + - $CI_PROJECT_DIR/build/obj-$GRUB_TARGETS/*.{log,trs} + - $CI_PROJECT_DIR/build/obj-$GRUB_TARGETS/*.strace* + - $CI_PROJECT_DIR/build/obj-$GRUB_TARGETS/test-data.tar.* + - $CI_PROJECT_DIR/build/obj-$GRUB_TARGETS/test.success + - $CI_PROJECT_DIR/build/obj-$GRUB_TARGETS/images + # - $CI_PROJECT_DIR/build/*.log + # - $CROSS_DIR + exclude: + - $CI_PROJECT_DIR/build/**/*.o + expire_in: 5 days + needs: + - setup:bootstrap + variables: + <<: *install-extra-qemu-packages-vars + BUILDDIR: $CI_PROJECT_DIR/build/obj-$GRUB_TARGETS + INSTALLDIR: $CI_PROJECT_DIR/install/grub-$GRUB_TARGETS + # CONFIGURE_OPTS: "--disable-silent-rules" + # TMPDIR: "/tmp" + TEST_DATA_PREFIX: $CI_PROJECT_DIR/build/obj-${GRUB_TARGETS}/test-data + # GRUB_QEMU_OPTS: -m size=64M + PACKAGES: | + wget + ### Grub build requirements + unifont + libfreetype6-dev + libdevmapper-dev + liblzma-dev + libfuse-dev + libzfslinux-dev + python + make + automake + bison + flex + gettext + pkg-config + ### Grub testing requirements + strace + qemu-system + # qemu-efi-aarch64 + # qemu-efi-arm + # qemu-efi + ovmf + # ovmf-ia32 + openbios-ppc + openbios-sparc + ### Used by grub-fs-tester + wamerican + ### needed by grub-mkrescue + xorriso + ### needed by xorriso to handle utf8 chars + locales + language-pack-en + ### needed by grub-mkrescue for efi platforms + mtools + ### Requirements for specific tests + tar + cpio + gzip + lzop + xz-utils + parted + util-linux + squashfs-tools + zfsutils-linux + dosfstools + exfat-utils + ntfs-3g + e2fsprogs + btrfs-progs + xfsprogs + hfsprogs + jfsutils + reiserfsprogs + udftools + nilfs-tools + f2fs-tools + genromfs + attr + ### Needed for mac-roman module for hfs_test + # linux-image-extra + before_script: + - mkdir -pv ${BUILDDIR} ${INSTALLDIR}; + - export > ${BUILDDIR}/environment.log + - . ${SRCDIR}/.ci/functions.gitlab.sh + - *setup-loopback-before_script + - *install-packages-before_script + - *install-non-repo-packages-before_script + - *install-extra-qemu-packages-before_script + - *cross_compilers-setup_script + - *build-ccache-before_script + - *print_debug_info-before_script + # Need to keep evaling PATH until there are no more variables in it + # because gitlab CI does not do recursive variable expansion. + - while [ -z ${PATH%%*\$*} ]; do + eval "export PATH=$PATH"; + done; + - pwd + - ls -la + script: + # Build all selected GRUB targets. + - test "x${SHELL_TRACE}" = "xy" && set -x; + - export TARGET=${GRUB_TARGETS}; + - export ARCH=${TARGET%-*}; + - export PLATFORM=${TARGET#*-}; + - if [ "x${BUILD_ALL_TARGETS}" != "xy" ] && ( echo "${DISABLED_BUILDS}" | grep -q "^${TARGET}$" ); then + echo -e "${TXT_CLEAR}${TXT_YELLOW}Building $TARGET has been disabled, skipping.${TXT_CLEAR}"; + exit 0; + fi; + - ( + case "$ARCH" in + arm64) ARCH=aarch64-linux;; + arm) ARCH=arm-linux-gnueabi;; + mips) ARCH=mips64-linux;; + powerpc) ARCH=powerpc64-linux;; + ia64|riscv32|riscv64|sparc64) ARCH=$ARCH-linux;; + i386|x86_64) ;; + *) echo "Unknown architecture":" $ARCH"; exit 1;; + esac; + case "$PLATFORM" in + coreboot) + export GRUB_CBFSTOOL="/where/is/cbfstool"; + export GRUB_COREBOOT_ROM="/where/is/coreboot/rom";; + esac; + + echo -e "#!$SHELL\nREAL_SHELL=\${REAL_SHELL:-\$(readlink /proc/\$\$/exe)}\nexec \$REAL_SHELL \$SHELL_OPTS \"\$@\"" >${BUILDDIR}/shell-wrapper.sh; + chmod +x ${BUILDDIR}/shell-wrapper.sh; + export CONFIG_SHELL=${BUILDDIR}/shell-wrapper.sh; + export CONFIGURE_OPTS+=" CONFIG_SHELL=${BUILDDIR}/shell-wrapper.sh"; + + $SRCDIR/.ci/build.sh 2>&1 | tee ${BUILDDIR}/build.log; + + if [ "x${DISABLE_ALL_TESTS}" = "xy" ]; then + echo "All tests are disabled"; + exit 0; + fi; + + export TESTTMPDIR="${TMPDIR:-/tmp}/grub-test-$TARGET-${CI_COMMIT_SHORT_SHA}"; + $SRCDIR/.ci/test.sh 2>&1 | tee ${BUILDDIR}/test.log || :; + ) 2>&1 | tee "${BUILDDIR}/${CI_JOB_NAME}.log"; + # Do processing of testing output + #- *process-tests-script + - TARGET=${GRUB_TARGETS} $SRCDIR/.ci/process-tests.sh + # Make images for all formats + - TARGET=${GRUB_TARGETS} $SRCDIR/.ci/make-images.sh + after_script: + - *build-ccache-after_script + +.test:allinone: + stage: test + # Run always, even if build fails so we can see pass/fail of the + # individual matrice runs. + when: always + artifacts: + name: "${CI_JOB_NAME}-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA}-${GRUB_TARGETS}" +# when: on_failure + when: always + paths: + - $CI_PROJECT_DIR/build/obj-$GRUB_TARGETS/*.{log,trs} + - $CI_PROJECT_DIR/build/*.log +# expire_in: 1 hour + dependencies: +# needs: + - build:allinone + script: &test_allinone-script + - *process-tests-script + parallel: + <<: *target_matrix + +package:allinone: + stage: .post + when: always + artifacts: + name: "${CI_JOB_NAME}-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA}" + when: always + expire_in: 1 month + paths: + # - *target-archive-paths + # - $CI_PROJECT_DIR/build/obj-*/*.{log,trs} + # - $CI_PROJECT_DIR/build/obj-*/*.strace* + # - $CI_PROJECT_DIR/build/obj-*/test-data.tar.* + # - $CI_PROJECT_DIR/build/obj-*/test.success + - $CI_PROJECT_DIR/build/obj-* + - $CI_PROJECT_DIR/build/*.log + exclude: + - $CI_PROJECT_DIR/build/**/*.o + - $CI_PROJECT_DIR/build/**/.deps-util + needs: + - build:allinone + # - test:allinone + before_script: + - . ${SRCDIR}/.ci/functions.gitlab.sh + - SUMLOG=$CI_PROJECT_DIR/build/test-summary.log + # - TXT_RED="\e[31m"; TXT_YELLOW="\e[33m"; TXT_CLEAR="\e[0m"; + - . $CI_PROJECT_DIR/.ci/functions.gitlab.sh + - find $CI_PROJECT_DIR/build > $CI_PROJECT_DIR/build/build-paths.log + script: + - for F in $CI_PROJECT_DIR/build/obj-*/test-suite.log; do + [ ! -e "$F" ] && continue; + TARGETDIR=$(dirname "$F"); + TARGET=$(cut -d- -f2- <<<$(basename "${TARGETDIR}")); + echo 'Ran test suite for target:' "${TARGET}"; + start_log -n "test-$TARGET" "Ran test suite for target:"" ${TARGET}" >>${SUMLOG}; + if [ -f "${TARGETDIR}/test.failures.log" ]; then + grep "^${TARGET}" "${TARGETDIR}/test.failures.log" | sort | + ( IFS=:; while read TARGET FTYPE TESTNAME; do + case "$FTYPE" in + skip) + echo -e "${TXT_CLEAR}${TXT_YELLOW}"' Skipped test:' "${TESTNAME}${TXT_CLEAR}" >>${SUMLOG};; + error) + echo -e "${TXT_CLEAR}${TXT_YELLOW}"' Hard error in test:' "${TESTNAME}${TXT_CLEAR}" >>${SUMLOG};; + expected) + echo -e "${TXT_CLEAR}${TXT_YELLOW}"' Expected failed test:' "${TESTNAME}${TXT_CLEAR}" >>${SUMLOG};; + timeout) + echo -e "${TXT_CLEAR}${TXT_YELLOW}"' Ignoring timed-out test:' "${TESTNAME}${TXT_CLEAR}" >>${SUMLOG};; + unexpected) + echo -e "${TXT_CLEAR}${TXT_RED}"' Failed test:' "${TESTNAME}${TXT_CLEAR}" >>${SUMLOG};; + *) + echo -e "${TXT_CLEAR}${TXT_RED}"' Unhandled test type for '"${TESTNAME} (type=$FTYPE)${TXT_CLEAR}" >>${SUMLOG};; + esac; + done ); + echo -e -n "${TXT_CLEAR}" >>${SUMLOG}; + fi; + end_log -n "test-$TARGET" >>${SUMLOG}; + done + - if [ -f "${SUMLOG}" ]; then + cat "${SUMLOG}"; + fi + +# mergereq:package:allinone: +# extends: +# - package:allinone +# rules: +# - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' +# - if: $CI_MERGE_REQUEST_IID +# - when: always + +#deploy-prod: +# stage: deploy +# script: +# - echo "This job deploys something from the $CI_COMMIT_BRANCH branch." +# - echo "Hello, $GITLAB_USER_LOGIN!" + +# TODO: put generated docs in gitlab pages +# pages: +# stage: deploy +# needs: build:docs +# script: +# - mkdir .public +# - cp -r * .public +# - mv .public public +# artifacts: +# paths: +# - public +# only: +# - master -- 2.27.0 _______________________________________________ Grub-devel mailing list Grub-devel@gnu.org https://lists.gnu.org/mailman/listinfo/grub-devel