Merge #17725: ci: Add valgrind run

facb416ad5 ci: Add valgrind run (MarcoFalke)

Pull request description:

  Fixes #17460

ACKs for top commit:
  practicalswift:
    ACK facb416ad5

Tree-SHA512: 55396e548a76f976d7b7170b68bc5f93cfd44656162267172f66db7eb549699a2a22d3b1bb0d5f180fe0697931939e652c8cdb86b435e81e7ce572485798009d

.travis.yml

@@ -111,7 +111,7 @@ jobs:
         FILE_ENV="./ci/test/00_setup_env_native_centos.sh"
 
     - stage: test
-      name: 'x86_64 Linux  [GOAL: install]  [bionic]  [uses qt5 dev package instead of depends Qt to speed up build and avoid timeout] [unsigned char]'
+      name: 'x86_64 Linux  [GOAL: install]  [bionic]  [uses qt5 dev package and some depends packages] [unsigned char]'
       env: >-
         FILE_ENV="./ci/test/00_setup_env_native_qt5.sh"
 
@@ -126,6 +126,11 @@ jobs:
       env: >-
         FILE_ENV="./ci/test/00_setup_env_native_asan.sh"
 
+    - stage: test
+      name: 'x86_64 Linux  [GOAL: install]  [bionic]  [no depends, only system libs, valgrind]'
+      env: >-
+        FILE_ENV="./ci/test/00_setup_env_native_valgrind.sh"
+
     - stage: test
       name: 'x86_64 Linux  [GOAL: install]  [bionic]  [no depends, only system libs, sanitizers: fuzzer,address,undefined]'
       env: >-

ci/test/00_setup_env_native_valgrind.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2019 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+export PACKAGES="valgrind clang llvm python3-zmq libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev"
+export USE_VALGRIND=1
+export NO_DEPENDS=1
+export TEST_RUNNER_EXTRA="p2p_segwit.py"  # Only run one test for now. TODO enable all and bump timeouts
+export RUN_FUNCTIONAL_TESTS=true
+export GOAL="install"
+export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=no CC=clang CXX=clang++"  # TODO enable GUI

ci/test/04_install.sh

@@ -41,7 +41,7 @@ export ASAN_OPTIONS="detect_stack_use_after_return=1:check_initialization_order=
 export LSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/lsan"
 export TSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/tsan:log_path=${BASE_SCRATCH_DIR}/sanitizer-output/tsan"
 export UBSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1:report_error_type=1"
-env | grep -E '^(BITCOIN_CONFIG|CCACHE_|WINEDEBUG|LC_ALL|BOOST_TEST_RANDOM|CONFIG_SHELL|(ASAN|LSAN|TSAN|UBSAN)_OPTIONS)' | tee /tmp/env
+env | grep -E '^(BITCOIN_CONFIG|BASE_|CCACHE_|WINEDEBUG|LC_ALL|BOOST_TEST_RANDOM|CONFIG_SHELL|(ASAN|LSAN|TSAN|UBSAN)_OPTIONS)' | tee /tmp/env
 if [[ $HOST = *-mingw32 ]]; then
   DOCKER_ADMIN="--cap-add SYS_ADMIN"
 elif [[ $BITCOIN_CONFIG = *--with-sanitizers=*address* ]]; then # If ran with (ASan + LSan), Docker needs access to ptrace (https://github.com/google/sanitizers/issues/764)

ci/test/06_script_b.sh

@@ -8,7 +8,6 @@ export LC_ALL=C.UTF-8
 
 if [ -n "$QEMU_USER_CMD" ]; then
   BEGIN_FOLD wrap-qemu
-  echo "Prepare to run functional tests for HOST=$HOST"
   # Generate all binaries, so that they can be wrapped
   DOCKER_EXEC make $MAKEJOBS -C src/secp256k1 VERBOSE=1
   DOCKER_EXEC make $MAKEJOBS -C src/univalue VERBOSE=1
@@ -25,6 +24,12 @@ if [ -n "$QEMU_USER_CMD" ]; then
   END_FOLD
 fi
 
+if [ -n "$USE_VALGRIND" ]; then
+  BEGIN_FOLD wrap-valgrind
+  DOCKER_EXEC "${BASE_ROOT_DIR}/ci/test/wrap-valgrind.sh"
+  END_FOLD
+fi
+
 if [ "$RUN_UNIT_TESTS" = "true" ]; then
   BEGIN_FOLD unit-tests
   bash -c "${CI_WAIT}" &  # Print dots in case the unit tests take a long time to run

ci/test/wrap-valgrind.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2018 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+for b_name in "${BASE_OUTDIR}/bin"/*; do
+    # shellcheck disable=SC2044
+    for b in $(find "${BASE_ROOT_DIR}" -executable -type f -name $(basename $b_name)); do
+      echo "Wrap $b ..."
+      mv "$b" "${b}_orig"
+      echo '#!/usr/bin/env bash' > "$b"
+      echo "valgrind --gen-suppressions=all --quiet --error-exitcode=1 --suppressions=${BASE_ROOT_DIR}/contrib/valgrind.supp \"${b}_orig\" \"\$@\"" >> "$b"
+      chmod +x "$b"
+    done
+done

contrib/valgrind.supp

@@ -30,8 +30,18 @@
    Memcheck:Cond
    obj:*/libdb_cxx-*.so
    fun:__log_put
-   obj:*/libdb_cxx-*.so
-   fun:__log_put_record
+}
+{
+   Suppress libdb warning
+   Memcheck:Param
+   pwrite64(buf)
+   fun:pwrite
+   fun:__os_io
+}
+{
+   Suppress libdb warning
+   Memcheck:Cond
+   fun:__log_putr.isra.1
 }
 {
    Suppress libdb warning
@@ -41,6 +51,37 @@
    fun:__os_io
    obj:*/libdb_cxx-*.so
 }
+{
+   Suppress uninitialized bytes warning in compat code
+   Memcheck:Param
+   ioctl(TCSET{S,SW,SF})
+   fun:tcsetattr
+}
+{
+   Suppress libdb warning
+   Memcheck:Leak
+   fun:malloc
+   ...
+   obj:*/libdb_cxx-*.so
+}
+{
+   Suppress leaks on init
+   Memcheck:Leak
+   ...
+   fun:_Z11AppInitMainR11NodeContext
+}
+{
+   Suppress leaks on shutdown
+   Memcheck:Leak
+   ...
+   fun:_Z8ShutdownR11NodeContext
+}
+{
+   Ignore GUI warning
+   Memcheck:Leak
+   ...
+   obj:/usr/lib64/libgdk-3.so.0.2404.7
+}
 {
    Suppress leveldb warning (leveldb::InitModule()) - https://github.com/google/leveldb/issues/113
    Memcheck:Leak
@@ -56,17 +97,49 @@
    ...
    fun:_ZN7leveldbL14InitDefaultEnvEv
 }
+{
+   Suppress leveldb leak
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:_Znwm
+   ...
+   fun:_ZN7leveldb6DBImpl14BackgroundCallEv
+}
+{
+   Suppress leveldb leak
+   Memcheck:Leak
+   fun:_Znwm
+   ...
+   fun:GetCoin
+}
 {
    Suppress wcsnrtombs glibc SSE4 warning (could be related: https://stroika.atlassian.net/browse/STK-626)
    Memcheck:Addr16
    fun:__wcsnlen_sse4_1
    fun:wcsnrtombs
 }
+{
+   Suppress wcsnrtombs warning (remove after removing boost::fs)
+   Memcheck:Cond
+   ...
+   fun:_ZN5boost10filesystem6detail11unique_pathERKNS0_4pathEPNS_6system10error_codeE
+   fun:unique_path
+}
+{
+   Suppress boost warning
+   Memcheck:Leak
+   fun:_Znwm
+   ...
+   fun:_ZN5boost9unit_test9framework5state17execute_test_treeEmjPKNS2_23random_generator_helperE
+   fun:_ZN5boost9unit_test9framework3runEmb
+   fun:_ZN5boost9unit_test14unit_test_mainEPFbvEiPPc
+   fun:main
+}
 {
    Suppress boost::filesystem warning (fixed in boost 1.70: https://github.com/boostorg/filesystem/commit/bbe9d1771e5d679b3f10c42a58fc81f7e8c024a9)
    Memcheck:Cond
    fun:_ZN5boost10filesystem6detail28directory_iterator_incrementERNS0_18directory_iteratorEPNS_6system10error_codeE
-   fun:_ZN5boost10filesystem6detail28directory_iterator_constructERNS0_18directory_iteratorERKNS0_4pathEPNS_6system10error_codeE
+   ...
    obj:*/libboost_filesystem.so.*
 }
 {
@@ -74,6 +147,7 @@
    Memcheck:Leak
    match-leak-kinds: reachable
    fun:_Znwm
+   ...
    fun:_ZN5boost10filesystem8absoluteERKNS0_4pathES3_
 }
 {