mirror of https://github.com/bitcoin/bitcoin
ci: Cache more stuff in the ci images: msan, iwyu, pip, sdks
This commit is contained in:
MarcoFalke
_
parent
2305643646
commit
faa0839837
|
|
@ -65,6 +65,7 @@ export BASE_BUILD_DIR=${BASE_BUILD_DIR:-$BASE_SCRATCH_DIR/build}
|
|||
# The folder for previous release binaries.
|
||||
# This folder exists only on the ci guest, and on the ci host as a volume.
|
||||
export PREVIOUS_RELEASES_DIR=${PREVIOUS_RELEASES_DIR:-$BASE_ROOT_DIR/releases/$HOST}
|
||||
export DIR_IWYU="${BASE_SCRATCH_DIR}/iwyu"
|
||||
export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks}
|
||||
export CI_BASE_PACKAGES=${CI_BASE_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3 rsync git procps bison}
|
||||
export GOAL=${GOAL:-install}
|
||||
|
|
|
|||
|
|
@ -6,8 +6,13 @@
|
|||
|
||||
export LC_ALL=C.UTF-8
|
||||
|
||||
# This script is always run as root, and the functions can be removed and
|
||||
# replaced by a bash -c "command" directly, where needed.
|
||||
# For now, they are named aliases to aid review with --color-moved=dimmed-zebra.
|
||||
CI_EXEC_ROOT () { bash -c "$*"; }
|
||||
CI_EXEC() { bash -c "$*"; }
|
||||
export -f CI_EXEC_ROOT
|
||||
export -f CI_EXEC
|
||||
|
||||
if [ -n "$DPKG_ADD_ARCH" ]; then
|
||||
CI_EXEC_ROOT dpkg --add-architecture "$DPKG_ADD_ARCH"
|
||||
|
|
@ -31,3 +36,55 @@ elif [ "$CI_USE_APT_INSTALL" != "no" ]; then
|
|||
${CI_RETRY_EXE} CI_EXEC_ROOT apt-get update
|
||||
${CI_RETRY_EXE} CI_EXEC_ROOT apt-get install --no-install-recommends --no-upgrade -y "$PACKAGES" "$CI_BASE_PACKAGES"
|
||||
fi
|
||||
|
||||
if [ -n "$PIP_PACKAGES" ]; then
|
||||
if [ "$CI_OS_NAME" == "macos" ]; then
|
||||
sudo -H pip3 install --upgrade pip
|
||||
# shellcheck disable=SC2086
|
||||
IN_GETOPT_BIN="$(brew --prefix gnu-getopt)/bin/getopt" ${CI_RETRY_EXE} pip3 install --user $PIP_PACKAGES
|
||||
else
|
||||
# shellcheck disable=SC2086
|
||||
${CI_RETRY_EXE} CI_EXEC pip3 install --user $PIP_PACKAGES
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then
|
||||
CI_EXEC_ROOT "update-alternatives --install /usr/bin/clang++ clang++ \$(which clang++-12) 100"
|
||||
CI_EXEC_ROOT "update-alternatives --install /usr/bin/clang clang \$(which clang-12) 100"
|
||||
CI_EXEC "mkdir -p ${BASE_SCRATCH_DIR}/msan/build/"
|
||||
CI_EXEC "git clone --depth=1 https://github.com/llvm/llvm-project -b llvmorg-12.0.0 ${BASE_SCRATCH_DIR}/msan/llvm-project"
|
||||
CI_EXEC "cd ${BASE_SCRATCH_DIR}/msan/build/ && cmake -DLLVM_ENABLE_PROJECTS='libcxx;libcxxabi' -DCMAKE_BUILD_TYPE=Release -DLLVM_USE_SANITIZER=MemoryWithOrigins -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DLLVM_TARGETS_TO_BUILD=X86 ../llvm-project/llvm/"
|
||||
CI_EXEC "cd ${BASE_SCRATCH_DIR}/msan/build/ && make $MAKEJOBS cxx"
|
||||
fi
|
||||
|
||||
if [[ "${RUN_TIDY}" == "true" ]]; then
|
||||
if [ ! -d "${DIR_IWYU}" ]; then
|
||||
CI_EXEC "mkdir -p ${DIR_IWYU}/build/"
|
||||
CI_EXEC "git clone --depth=1 https://github.com/include-what-you-use/include-what-you-use -b clang_15 ${DIR_IWYU}/include-what-you-use"
|
||||
CI_EXEC "cd ${DIR_IWYU}/build && cmake -G 'Unix Makefiles' -DCMAKE_PREFIX_PATH=/usr/lib/llvm-15 ../include-what-you-use"
|
||||
CI_EXEC_ROOT "cd ${DIR_IWYU}/build && make install $MAKEJOBS"
|
||||
fi
|
||||
fi
|
||||
|
||||
CI_EXEC mkdir -p "${DEPENDS_DIR}/SDKs" "${DEPENDS_DIR}/sdk-sources"
|
||||
|
||||
OSX_SDK_BASENAME="Xcode-${XCODE_VERSION}-${XCODE_BUILD_ID}-extracted-SDK-with-libcxx-headers"
|
||||
|
||||
if [ -n "$XCODE_VERSION" ] && [ ! -d "${DEPENDS_DIR}/SDKs/${OSX_SDK_BASENAME}" ]; then
|
||||
OSX_SDK_FILENAME="${OSX_SDK_BASENAME}.tar.gz"
|
||||
OSX_SDK_PATH="${DEPENDS_DIR}/sdk-sources/${OSX_SDK_FILENAME}"
|
||||
if [ ! -f "$OSX_SDK_PATH" ]; then
|
||||
CI_EXEC curl --location --fail "${SDK_URL}/${OSX_SDK_FILENAME}" -o "$OSX_SDK_PATH"
|
||||
fi
|
||||
CI_EXEC tar -C "${DEPENDS_DIR}/SDKs" -xf "$OSX_SDK_PATH"
|
||||
fi
|
||||
|
||||
if [ -n "$ANDROID_HOME" ] && [ ! -d "$ANDROID_HOME" ]; then
|
||||
ANDROID_TOOLS_PATH=${DEPENDS_DIR}/sdk-sources/android-tools.zip
|
||||
if [ ! -f "$ANDROID_TOOLS_PATH" ]; then
|
||||
CI_EXEC curl --location --fail "${ANDROID_TOOLS_URL}" -o "$ANDROID_TOOLS_PATH"
|
||||
fi
|
||||
CI_EXEC mkdir -p "$ANDROID_HOME"
|
||||
CI_EXEC unzip -o "$ANDROID_TOOLS_PATH" -d "$ANDROID_HOME"
|
||||
CI_EXEC "yes | ${ANDROID_HOME}/cmdline-tools/bin/sdkmanager --sdk_root=\"${ANDROID_HOME}\" --install \"build-tools;${ANDROID_BUILD_TOOLS_VERSION}\" \"platform-tools\" \"platforms;android-${ANDROID_API_LEVEL}\" \"ndk;${ANDROID_NDK_VERSION}\""
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -59,6 +59,10 @@ if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then
|
|||
--name $CONTAINER_NAME \
|
||||
$CONTAINER_NAME)
|
||||
export CI_CONTAINER_ID
|
||||
export CI_EXEC_CMD_PREFIX_ROOT="docker exec -u 0 $CI_CONTAINER_ID"
|
||||
export CI_EXEC_CMD_PREFIX="docker exec -u $LOCAL_UID $CI_CONTAINER_ID"
|
||||
$CI_EXEC_CMD_PREFIX_ROOT rsync --archive --stats --human-readable /ci_base_install/ "${BASE_ROOT_DIR}"
|
||||
$CI_EXEC_CMD_PREFIX_ROOT rsync --archive --stats --human-readable /ro_base/ "$BASE_ROOT_DIR"
|
||||
|
||||
# Create a non-root user inside the container which matches the local user.
|
||||
#
|
||||
|
|
@ -67,8 +71,6 @@ if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then
|
|||
docker exec "$CI_CONTAINER_ID" useradd -u "$LOCAL_UID" -o -m "$LOCAL_USER"
|
||||
docker exec "$CI_CONTAINER_ID" groupmod -o -g "$LOCAL_GID" "$LOCAL_USER"
|
||||
docker exec "$CI_CONTAINER_ID" chown -R "$LOCAL_USER":"$LOCAL_USER" "${BASE_ROOT_DIR}"
|
||||
export CI_EXEC_CMD_PREFIX_ROOT="docker exec -u 0 $CI_CONTAINER_ID"
|
||||
export CI_EXEC_CMD_PREFIX="docker exec -u $LOCAL_UID $CI_CONTAINER_ID"
|
||||
else
|
||||
echo "Running on host system without docker wrapper"
|
||||
"${BASE_ROOT_DIR}/ci/test/01_base_install.sh"
|
||||
|
|
@ -85,17 +87,6 @@ export -f CI_EXEC_ROOT
|
|||
|
||||
CI_EXEC mkdir -p "${BINS_SCRATCH_DIR}"
|
||||
|
||||
if [ -n "$PIP_PACKAGES" ]; then
|
||||
if [ "$CI_OS_NAME" == "macos" ]; then
|
||||
sudo -H pip3 install --upgrade pip
|
||||
# shellcheck disable=SC2086
|
||||
IN_GETOPT_BIN="$(brew --prefix gnu-getopt)/bin/getopt" ${CI_RETRY_EXE} pip3 install --user $PIP_PACKAGES
|
||||
else
|
||||
# shellcheck disable=SC2086
|
||||
${CI_RETRY_EXE} CI_EXEC pip3 install --user $PIP_PACKAGES
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ "$CI_OS_NAME" == "macos" ]; then
|
||||
top -l 1 -s 0 | awk ' /PhysMem/ {print}'
|
||||
echo "Number of CPUs: $(sysctl -n hw.logicalcpu)"
|
||||
|
|
@ -122,30 +113,6 @@ fi
|
|||
|
||||
CI_EXEC mkdir -p "${BASE_SCRATCH_DIR}/sanitizer-output/"
|
||||
|
||||
if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then
|
||||
CI_EXEC_ROOT "update-alternatives --install /usr/bin/clang++ clang++ \$(which clang++-12) 100"
|
||||
CI_EXEC_ROOT "update-alternatives --install /usr/bin/clang clang \$(which clang-12) 100"
|
||||
CI_EXEC "mkdir -p ${BASE_SCRATCH_DIR}/msan/build/"
|
||||
CI_EXEC "git clone --depth=1 https://github.com/llvm/llvm-project -b llvmorg-12.0.0 ${BASE_SCRATCH_DIR}/msan/llvm-project"
|
||||
CI_EXEC "cd ${BASE_SCRATCH_DIR}/msan/build/ && cmake -DLLVM_ENABLE_PROJECTS='libcxx;libcxxabi' -DCMAKE_BUILD_TYPE=Release -DLLVM_USE_SANITIZER=MemoryWithOrigins -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DLLVM_TARGETS_TO_BUILD=X86 ../llvm-project/llvm/"
|
||||
CI_EXEC "cd ${BASE_SCRATCH_DIR}/msan/build/ && make $MAKEJOBS cxx"
|
||||
fi
|
||||
|
||||
if [[ "${RUN_TIDY}" == "true" ]]; then
|
||||
export DIR_IWYU="${BASE_SCRATCH_DIR}/iwyu"
|
||||
if [ ! -d "${DIR_IWYU}" ]; then
|
||||
CI_EXEC "mkdir -p ${DIR_IWYU}/build/"
|
||||
CI_EXEC "git clone --depth=1 https://github.com/include-what-you-use/include-what-you-use -b clang_15 ${DIR_IWYU}/include-what-you-use"
|
||||
CI_EXEC "cd ${DIR_IWYU}/build && cmake -G 'Unix Makefiles' -DCMAKE_PREFIX_PATH=/usr/lib/llvm-15 ../include-what-you-use"
|
||||
CI_EXEC_ROOT "cd ${DIR_IWYU}/build && make install $MAKEJOBS"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then
|
||||
echo "Create $BASE_ROOT_DIR"
|
||||
CI_EXEC rsync -a /ro_base/ "$BASE_ROOT_DIR"
|
||||
fi
|
||||
|
||||
if [ "$USE_BUSY_BOX" = "true" ]; then
|
||||
echo "Setup to use BusyBox utils"
|
||||
# tar excluded for now because it requires passing in the exact archive type in ./depends (fixed in later BusyBox version)
|
||||
|
|
|
|||
|
|
@ -14,29 +14,6 @@ else
|
|||
CI_EXEC_ROOT echo \> \$HOME/.bitcoin
|
||||
fi
|
||||
|
||||
CI_EXEC mkdir -p "${DEPENDS_DIR}/SDKs" "${DEPENDS_DIR}/sdk-sources"
|
||||
|
||||
OSX_SDK_BASENAME="Xcode-${XCODE_VERSION}-${XCODE_BUILD_ID}-extracted-SDK-with-libcxx-headers"
|
||||
|
||||
if [ -n "$XCODE_VERSION" ] && [ ! -d "${DEPENDS_DIR}/SDKs/${OSX_SDK_BASENAME}" ]; then
|
||||
OSX_SDK_FILENAME="${OSX_SDK_BASENAME}.tar.gz"
|
||||
OSX_SDK_PATH="${DEPENDS_DIR}/sdk-sources/${OSX_SDK_FILENAME}"
|
||||
if [ ! -f "$OSX_SDK_PATH" ]; then
|
||||
CI_EXEC curl --location --fail "${SDK_URL}/${OSX_SDK_FILENAME}" -o "$OSX_SDK_PATH"
|
||||
fi
|
||||
CI_EXEC tar -C "${DEPENDS_DIR}/SDKs" -xf "$OSX_SDK_PATH"
|
||||
fi
|
||||
|
||||
if [ -n "$ANDROID_HOME" ] && [ ! -d "$ANDROID_HOME" ]; then
|
||||
ANDROID_TOOLS_PATH=${DEPENDS_DIR}/sdk-sources/android-tools.zip
|
||||
if [ ! -f "$ANDROID_TOOLS_PATH" ]; then
|
||||
CI_EXEC curl --location --fail "${ANDROID_TOOLS_URL}" -o "$ANDROID_TOOLS_PATH"
|
||||
fi
|
||||
CI_EXEC mkdir -p "$ANDROID_HOME"
|
||||
CI_EXEC unzip -o "$ANDROID_TOOLS_PATH" -d "$ANDROID_HOME"
|
||||
CI_EXEC "yes | ${ANDROID_HOME}/cmdline-tools/bin/sdkmanager --sdk_root=\"${ANDROID_HOME}\" --install \"build-tools;${ANDROID_BUILD_TOOLS_VERSION}\" \"platform-tools\" \"platforms;android-${ANDROID_API_LEVEL}\" \"ndk;${ANDROID_NDK_VERSION}\""
|
||||
fi
|
||||
|
||||
if [ -z "$NO_DEPENDS" ]; then
|
||||
if [[ $CI_IMAGE_NAME_TAG == *centos* ]]; then
|
||||
# CentOS has problems building the depends if the config shell is not explicitly set
|
||||
|
|
|
|||
Loading…
Reference in New Issue