diff --git a/ci/test/00_setup_env.sh b/ci/test/00_setup_env.sh index e6905b6dbe4..4a54f47b03f 100755 --- a/ci/test/00_setup_env.sh +++ b/ci/test/00_setup_env.sh @@ -65,6 +65,7 @@ export BASE_BUILD_DIR=${BASE_BUILD_DIR:-$BASE_SCRATCH_DIR/build} # The folder for previous release binaries. # This folder exists only on the ci guest, and on the ci host as a volume. export PREVIOUS_RELEASES_DIR=${PREVIOUS_RELEASES_DIR:-$BASE_ROOT_DIR/releases/$HOST} +export DIR_IWYU="${BASE_SCRATCH_DIR}/iwyu" export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks} export CI_BASE_PACKAGES=${CI_BASE_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3 rsync git procps bison} export GOAL=${GOAL:-install} diff --git a/ci/test/01_base_install.sh b/ci/test/01_base_install.sh index c2469d7ca92..a050067ae4e 100755 --- a/ci/test/01_base_install.sh +++ b/ci/test/01_base_install.sh @@ -6,8 +6,13 @@ export LC_ALL=C.UTF-8 +# This script is always run as root, and the functions can be removed and +# replaced by a bash -c "command" directly, where needed. +# For now, they are named aliases to aid review with --color-moved=dimmed-zebra. CI_EXEC_ROOT () { bash -c "$*"; } +CI_EXEC() { bash -c "$*"; } export -f CI_EXEC_ROOT +export -f CI_EXEC if [ -n "$DPKG_ADD_ARCH" ]; then CI_EXEC_ROOT dpkg --add-architecture "$DPKG_ADD_ARCH" @@ -31,3 +36,55 @@ elif [ "$CI_USE_APT_INSTALL" != "no" ]; then ${CI_RETRY_EXE} CI_EXEC_ROOT apt-get update ${CI_RETRY_EXE} CI_EXEC_ROOT apt-get install --no-install-recommends --no-upgrade -y "$PACKAGES" "$CI_BASE_PACKAGES" fi + +if [ -n "$PIP_PACKAGES" ]; then + if [ "$CI_OS_NAME" == "macos" ]; then + sudo -H pip3 install --upgrade pip + # shellcheck disable=SC2086 + IN_GETOPT_BIN="$(brew --prefix gnu-getopt)/bin/getopt" ${CI_RETRY_EXE} pip3 install --user $PIP_PACKAGES + else + # shellcheck disable=SC2086 + ${CI_RETRY_EXE} CI_EXEC pip3 install --user $PIP_PACKAGES + fi +fi + +if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then + CI_EXEC_ROOT "update-alternatives --install /usr/bin/clang++ clang++ \$(which clang++-12) 100" + CI_EXEC_ROOT "update-alternatives --install /usr/bin/clang clang \$(which clang-12) 100" + CI_EXEC "mkdir -p ${BASE_SCRATCH_DIR}/msan/build/" + CI_EXEC "git clone --depth=1 https://github.com/llvm/llvm-project -b llvmorg-12.0.0 ${BASE_SCRATCH_DIR}/msan/llvm-project" + CI_EXEC "cd ${BASE_SCRATCH_DIR}/msan/build/ && cmake -DLLVM_ENABLE_PROJECTS='libcxx;libcxxabi' -DCMAKE_BUILD_TYPE=Release -DLLVM_USE_SANITIZER=MemoryWithOrigins -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DLLVM_TARGETS_TO_BUILD=X86 ../llvm-project/llvm/" + CI_EXEC "cd ${BASE_SCRATCH_DIR}/msan/build/ && make $MAKEJOBS cxx" +fi + +if [[ "${RUN_TIDY}" == "true" ]]; then + if [ ! -d "${DIR_IWYU}" ]; then + CI_EXEC "mkdir -p ${DIR_IWYU}/build/" + CI_EXEC "git clone --depth=1 https://github.com/include-what-you-use/include-what-you-use -b clang_15 ${DIR_IWYU}/include-what-you-use" + CI_EXEC "cd ${DIR_IWYU}/build && cmake -G 'Unix Makefiles' -DCMAKE_PREFIX_PATH=/usr/lib/llvm-15 ../include-what-you-use" + CI_EXEC_ROOT "cd ${DIR_IWYU}/build && make install $MAKEJOBS" + fi +fi + +CI_EXEC mkdir -p "${DEPENDS_DIR}/SDKs" "${DEPENDS_DIR}/sdk-sources" + +OSX_SDK_BASENAME="Xcode-${XCODE_VERSION}-${XCODE_BUILD_ID}-extracted-SDK-with-libcxx-headers" + +if [ -n "$XCODE_VERSION" ] && [ ! -d "${DEPENDS_DIR}/SDKs/${OSX_SDK_BASENAME}" ]; then + OSX_SDK_FILENAME="${OSX_SDK_BASENAME}.tar.gz" + OSX_SDK_PATH="${DEPENDS_DIR}/sdk-sources/${OSX_SDK_FILENAME}" + if [ ! -f "$OSX_SDK_PATH" ]; then + CI_EXEC curl --location --fail "${SDK_URL}/${OSX_SDK_FILENAME}" -o "$OSX_SDK_PATH" + fi + CI_EXEC tar -C "${DEPENDS_DIR}/SDKs" -xf "$OSX_SDK_PATH" +fi + +if [ -n "$ANDROID_HOME" ] && [ ! -d "$ANDROID_HOME" ]; then + ANDROID_TOOLS_PATH=${DEPENDS_DIR}/sdk-sources/android-tools.zip + if [ ! -f "$ANDROID_TOOLS_PATH" ]; then + CI_EXEC curl --location --fail "${ANDROID_TOOLS_URL}" -o "$ANDROID_TOOLS_PATH" + fi + CI_EXEC mkdir -p "$ANDROID_HOME" + CI_EXEC unzip -o "$ANDROID_TOOLS_PATH" -d "$ANDROID_HOME" + CI_EXEC "yes | ${ANDROID_HOME}/cmdline-tools/bin/sdkmanager --sdk_root=\"${ANDROID_HOME}\" --install \"build-tools;${ANDROID_BUILD_TOOLS_VERSION}\" \"platform-tools\" \"platforms;android-${ANDROID_API_LEVEL}\" \"ndk;${ANDROID_NDK_VERSION}\"" +fi diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh index 70874297306..b4d507cdbb2 100755 --- a/ci/test/04_install.sh +++ b/ci/test/04_install.sh @@ -59,6 +59,10 @@ if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then --name $CONTAINER_NAME \ $CONTAINER_NAME) export CI_CONTAINER_ID + export CI_EXEC_CMD_PREFIX_ROOT="docker exec -u 0 $CI_CONTAINER_ID" + export CI_EXEC_CMD_PREFIX="docker exec -u $LOCAL_UID $CI_CONTAINER_ID" + $CI_EXEC_CMD_PREFIX_ROOT rsync --archive --stats --human-readable /ci_base_install/ "${BASE_ROOT_DIR}" + $CI_EXEC_CMD_PREFIX_ROOT rsync --archive --stats --human-readable /ro_base/ "$BASE_ROOT_DIR" # Create a non-root user inside the container which matches the local user. # @@ -67,8 +71,6 @@ if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then docker exec "$CI_CONTAINER_ID" useradd -u "$LOCAL_UID" -o -m "$LOCAL_USER" docker exec "$CI_CONTAINER_ID" groupmod -o -g "$LOCAL_GID" "$LOCAL_USER" docker exec "$CI_CONTAINER_ID" chown -R "$LOCAL_USER":"$LOCAL_USER" "${BASE_ROOT_DIR}" - export CI_EXEC_CMD_PREFIX_ROOT="docker exec -u 0 $CI_CONTAINER_ID" - export CI_EXEC_CMD_PREFIX="docker exec -u $LOCAL_UID $CI_CONTAINER_ID" else echo "Running on host system without docker wrapper" "${BASE_ROOT_DIR}/ci/test/01_base_install.sh" @@ -85,17 +87,6 @@ export -f CI_EXEC_ROOT CI_EXEC mkdir -p "${BINS_SCRATCH_DIR}" -if [ -n "$PIP_PACKAGES" ]; then - if [ "$CI_OS_NAME" == "macos" ]; then - sudo -H pip3 install --upgrade pip - # shellcheck disable=SC2086 - IN_GETOPT_BIN="$(brew --prefix gnu-getopt)/bin/getopt" ${CI_RETRY_EXE} pip3 install --user $PIP_PACKAGES - else - # shellcheck disable=SC2086 - ${CI_RETRY_EXE} CI_EXEC pip3 install --user $PIP_PACKAGES - fi -fi - if [ "$CI_OS_NAME" == "macos" ]; then top -l 1 -s 0 | awk ' /PhysMem/ {print}' echo "Number of CPUs: $(sysctl -n hw.logicalcpu)" @@ -122,30 +113,6 @@ fi CI_EXEC mkdir -p "${BASE_SCRATCH_DIR}/sanitizer-output/" -if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then - CI_EXEC_ROOT "update-alternatives --install /usr/bin/clang++ clang++ \$(which clang++-12) 100" - CI_EXEC_ROOT "update-alternatives --install /usr/bin/clang clang \$(which clang-12) 100" - CI_EXEC "mkdir -p ${BASE_SCRATCH_DIR}/msan/build/" - CI_EXEC "git clone --depth=1 https://github.com/llvm/llvm-project -b llvmorg-12.0.0 ${BASE_SCRATCH_DIR}/msan/llvm-project" - CI_EXEC "cd ${BASE_SCRATCH_DIR}/msan/build/ && cmake -DLLVM_ENABLE_PROJECTS='libcxx;libcxxabi' -DCMAKE_BUILD_TYPE=Release -DLLVM_USE_SANITIZER=MemoryWithOrigins -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DLLVM_TARGETS_TO_BUILD=X86 ../llvm-project/llvm/" - CI_EXEC "cd ${BASE_SCRATCH_DIR}/msan/build/ && make $MAKEJOBS cxx" -fi - -if [[ "${RUN_TIDY}" == "true" ]]; then - export DIR_IWYU="${BASE_SCRATCH_DIR}/iwyu" - if [ ! -d "${DIR_IWYU}" ]; then - CI_EXEC "mkdir -p ${DIR_IWYU}/build/" - CI_EXEC "git clone --depth=1 https://github.com/include-what-you-use/include-what-you-use -b clang_15 ${DIR_IWYU}/include-what-you-use" - CI_EXEC "cd ${DIR_IWYU}/build && cmake -G 'Unix Makefiles' -DCMAKE_PREFIX_PATH=/usr/lib/llvm-15 ../include-what-you-use" - CI_EXEC_ROOT "cd ${DIR_IWYU}/build && make install $MAKEJOBS" - fi -fi - -if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then - echo "Create $BASE_ROOT_DIR" - CI_EXEC rsync -a /ro_base/ "$BASE_ROOT_DIR" -fi - if [ "$USE_BUSY_BOX" = "true" ]; then echo "Setup to use BusyBox utils" # tar excluded for now because it requires passing in the exact archive type in ./depends (fixed in later BusyBox version) diff --git a/ci/test/05_before_script.sh b/ci/test/05_before_script.sh index f49305597d4..8e3153a20bb 100755 --- a/ci/test/05_before_script.sh +++ b/ci/test/05_before_script.sh @@ -14,29 +14,6 @@ else CI_EXEC_ROOT echo \> \$HOME/.bitcoin fi -CI_EXEC mkdir -p "${DEPENDS_DIR}/SDKs" "${DEPENDS_DIR}/sdk-sources" - -OSX_SDK_BASENAME="Xcode-${XCODE_VERSION}-${XCODE_BUILD_ID}-extracted-SDK-with-libcxx-headers" - -if [ -n "$XCODE_VERSION" ] && [ ! -d "${DEPENDS_DIR}/SDKs/${OSX_SDK_BASENAME}" ]; then - OSX_SDK_FILENAME="${OSX_SDK_BASENAME}.tar.gz" - OSX_SDK_PATH="${DEPENDS_DIR}/sdk-sources/${OSX_SDK_FILENAME}" - if [ ! -f "$OSX_SDK_PATH" ]; then - CI_EXEC curl --location --fail "${SDK_URL}/${OSX_SDK_FILENAME}" -o "$OSX_SDK_PATH" - fi - CI_EXEC tar -C "${DEPENDS_DIR}/SDKs" -xf "$OSX_SDK_PATH" -fi - -if [ -n "$ANDROID_HOME" ] && [ ! -d "$ANDROID_HOME" ]; then - ANDROID_TOOLS_PATH=${DEPENDS_DIR}/sdk-sources/android-tools.zip - if [ ! -f "$ANDROID_TOOLS_PATH" ]; then - CI_EXEC curl --location --fail "${ANDROID_TOOLS_URL}" -o "$ANDROID_TOOLS_PATH" - fi - CI_EXEC mkdir -p "$ANDROID_HOME" - CI_EXEC unzip -o "$ANDROID_TOOLS_PATH" -d "$ANDROID_HOME" - CI_EXEC "yes | ${ANDROID_HOME}/cmdline-tools/bin/sdkmanager --sdk_root=\"${ANDROID_HOME}\" --install \"build-tools;${ANDROID_BUILD_TOOLS_VERSION}\" \"platform-tools\" \"platforms;android-${ANDROID_API_LEVEL}\" \"ndk;${ANDROID_NDK_VERSION}\"" -fi - if [ -z "$NO_DEPENDS" ]; then if [[ $CI_IMAGE_NAME_TAG == *centos* ]]; then # CentOS has problems building the depends if the config shell is not explicitly set