qa: Add test/fuzz/test_runner.py

2019-01-16 11:49:01 -05:00 · 2019-01-16 11:49:01 -05:00 · fa7ca8ef58
parent ad039aa0d3
commit fa7ca8ef58
8 changed files with 199 additions and 27 deletions
--- a/.travis.yml
+++ b/.travis.yml
@ -15,6 +15,7 @@ env:
    - MAKEJOBS=-j3
    - RUN_UNIT_TESTS=true
    - RUN_FUNCTIONAL_TESTS=true
+    - RUN_FUZZ_TESTS=false
    - DOCKER_NAME_TAG=ubuntu:18.04
    - BOOST_TEST_RANDOM=1$TRAVIS_BUILD_ID
    - CCACHE_SIZE=100M
@ -100,7 +101,7 @@ jobs:
        PACKAGES="python3-zmq qtbase5-dev qttools5-dev-tools protobuf-compiler libdbus-1-dev libharfbuzz-dev libprotobuf-dev"
        DEP_OPTS="NO_QT=1 NO_UPNP=1 DEBUG=1 ALLOW_HOST_PACKAGES=1"
        GOAL="install"
-        BITCOIN_CONFIG="--enable-zmq --with-gui=qt5 --enable-fuzz --enable-glibc-back-compat --enable-reduce-exports --enable-debug CXXFLAGS=\"-g0 -O2\""
+        BITCOIN_CONFIG="--enable-zmq --with-gui=qt5 --enable-glibc-back-compat --enable-reduce-exports --enable-debug CXXFLAGS=\"-g0 -O2\""

    - stage: test
      name: 'x86_64 Linux  [GOAL: install]  [trusty]  [no functional tests, no depends, only system libs]'
@ -132,6 +133,18 @@ jobs:
        GOAL="install"
        BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=qt5 CPPFLAGS=-DDEBUG_LOCKORDER --with-sanitizers=address,integer,undefined CC=clang CXX=clang++"

+    - stage: test
+      name: 'x86_64 Linux  [GOAL: install]  [bionic]  [no depends, only system libs, sanitizers: fuzzer,address]'
+      env: >-
+        HOST=x86_64-unknown-linux-gnu
+        PACKAGES="clang llvm python3 libssl1.0-dev libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev"
+        NO_DEPENDS=1
+        RUN_UNIT_TESTS=false
+        RUN_FUNCTIONAL_TESTS=false
+        RUN_FUZZ_TESTS=true
+        GOAL="install"
+        BITCOIN_CONFIG="--disable-wallet --disable-bench --with-utils=no --with-daemon=no --with-libs=no --with-gui=no --enable-fuzz --with-sanitizers=fuzzer,address CC=clang CXX=clang++"
+
    - stage: test
      name: 'x86_64 Linux  [GOAL: install]  [bionic]  [no wallet]'
      env: >-
--- a/.travis/test_04_install.sh
+++ b/.travis/test_04_install.sh
@ -7,6 +7,11 @@
 export LC_ALL=C.UTF-8

 travis_retry docker pull "$DOCKER_NAME_TAG"
+
+export DIR_FUZZ_IN=${TRAVIS_BUILD_DIR}/qa-assets
+git clone https://github.com/bitcoin-core/qa-assets ${DIR_FUZZ_IN}
+export DIR_FUZZ_IN=${DIR_FUZZ_IN}/fuzz_seed_corpus/
+
 mkdir -p "${TRAVIS_BUILD_DIR}/sanitizer-output/"
 export ASAN_OPTIONS=""
 export LSAN_OPTIONS="suppressions=${TRAVIS_BUILD_DIR}/test/sanitizer_suppressions/lsan"
--- a/.travis/test_06_script_b.sh
+++ b/.travis/test_06_script_b.sh
@ -19,3 +19,9 @@ if [ "$RUN_FUNCTIONAL_TESTS" = "true" ]; then
  DOCKER_EXEC test/functional/test_runner.py --ci --combinedlogslen=4000 --coverage --quiet --failfast
  END_FOLD
 fi
+
+if [ "$RUN_FUZZ_TESTS" = "true" ]; then
+  BEGIN_FOLD fuzz-tests
+  DOCKER_EXEC test/fuzz/test_runner.py -l DEBUG ${DIR_FUZZ_IN}
+  END_FOLD
+fi
--- a/Makefile.am
+++ b/Makefile.am
@ -220,7 +220,11 @@ endif

 dist_noinst_SCRIPTS = autogen.sh

-EXTRA_DIST = $(DIST_SHARE) test/functional/test_runner.py test/functional $(DIST_CONTRIB) $(DIST_DOCS) $(WINDOWS_PACKAGING) $(OSX_PACKAGING) $(BIN_CHECKS)
+EXTRA_DIST = $(DIST_SHARE) $(DIST_CONTRIB) $(DIST_DOCS) $(WINDOWS_PACKAGING) $(OSX_PACKAGING) $(BIN_CHECKS)
+
+EXTRA_DIST += \
+    test/functional \
+    test/fuzz

 EXTRA_DIST += \
    test/util/bitcoin-util-test.py \
--- a/doc/fuzzing.md
+++ b/doc/fuzzing.md
@ -5,6 +5,29 @@ A special test harness in `src/test/fuzz/` is provided for each fuzz target to
 provide an easy entry point for fuzzers and the like. In this document we'll
 describe how to use it with AFL and libFuzzer.

+## Preparing fuzzing
+
+AFL needs an input directory with examples, and an output directory where it
+will place examples that it found. These can be anywhere in the file system,
+we'll define environment variables to make it easy to reference them.
+
+libFuzzer will use the input directory as output directory.
+
+Extract the example seeds (or other starting inputs) into the inputs
+directory before starting fuzzing.
+
+```
+git clone https://github.com/bitcoin-core/qa-assets
+export DIR_FUZZ_IN=$PWD/qa-assets/fuzz_seed_corpus
+```
+
+Only for AFL:
+
+```
+mkdir outputs
+export AFLOUT=$PWD/outputs
+```
+
 ## AFL

 ### Building AFL
@ -23,7 +46,7 @@ export AFLPATH=$PWD
 To build Bitcoin Core using AFL instrumentation (this assumes that the
 `AFLPATH` was set as above):
 ```
-./configure --disable-ccache --disable-shared --enable-tests --enable-fuzz CC=${AFLPATH}/afl-gcc CXX=${AFLPATH}/afl-g++
+./configure --disable-ccache --disable-shared --enable-tests --enable-fuzz --disable-wallet --disable-bench --with-utils=no --with-daemon=no --with-libs=no --with-gui=no CC=${AFLPATH}/afl-gcc CXX=${AFLPATH}/afl-g++
 export AFL_HARDEN=1
 cd src/
 make
@ -39,31 +62,14 @@ binary will be instrumented in such a way that the AFL
 features "persistent mode" and "deferred forkserver" can be used. See
 https://github.com/mcarpenter/afl/tree/master/llvm_mode for details.

-### Preparing fuzzing
-
-AFL needs an input directory with examples, and an output directory where it
-will place examples that it found. These can be anywhere in the file system,
-we'll define environment variables to make it easy to reference them.
-
-```
-mkdir inputs
-AFLIN=$PWD/inputs
-mkdir outputs
-AFLOUT=$PWD/outputs
-```
-
-Example inputs are available from:
-
- https://download.visucore.com/bitcoin/bitcoin_fuzzy_in.tar.xz
- http://strateman.ninja/fuzzing.tar.xz
-
-Extract these (or other starting inputs) into the `inputs` directory before starting fuzzing.
-
 ### Fuzzing

 To start the actual fuzzing use:
+
 ```
-$AFLPATH/afl-fuzz -i ${AFLIN} -o ${AFLOUT} -m52 -- test/fuzz/fuzz_target_foo
+export FUZZ_TARGET=fuzz_target_foo  # Pick a fuzz_target
+mkdir ${AFLOUT}/${FUZZ_TARGET}
+$AFLPATH/afl-fuzz -i ${DIR_FUZZ_IN}/${FUZZ_TARGET} -o ${AFLOUT}/${FUZZ_TARGET} -m52 -- test/fuzz/${FUZZ_TARGET}
 ```

 You may have to change a few kernel parameters to test optimally - `afl-fuzz`
@ -74,10 +80,10 @@ will print an error and suggestion if so.
 A recent version of `clang`, the address sanitizer and libFuzzer is needed (all
 found in the `compiler-rt` runtime libraries package).

-To build the `test/test_bitcoin_fuzzy` executable run
+To build all fuzz targets with libFuzzer, run

 ```
-./configure --disable-ccache --enable-fuzz --with-sanitizers=fuzzer,address CC=clang CXX=clang++
+./configure --disable-ccache --disable-wallet --disable-bench --with-utils=no --with-daemon=no --with-libs=no --with-gui=no --enable-fuzz --with-sanitizers=fuzzer,address CC=clang CXX=clang++
 make
 ```

@ -86,3 +92,6 @@ interchangeably between libFuzzer and AFL.

 See https://llvm.org/docs/LibFuzzer.html#running on how to run the libFuzzer
 instrumented executable.
+
+Alternatively run the script in `./test/fuzz/test_runner.py` and provide it
+with the `${DIR_FUZZ_IN}` created earlier.
--- a/src/Makefile.test.include
+++ b/src/Makefile.test.include
@ -2,7 +2,6 @@
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.

-bin_PROGRAMS += test/test_bitcoin

 FUZZ_TARGETS = \
  test/fuzz/address_deserialize \
@ -28,6 +27,8 @@ FUZZ_TARGETS = \

 if ENABLE_FUZZ
 noinst_PROGRAMS += $(FUZZ_TARGETS:=)
+else
+bin_PROGRAMS += test/test_bitcoin
 endif

 TEST_SRCDIR = test
--- a/test/config.ini.in
+++ b/test/config.ini.in
@ -16,4 +16,5 @@ RPCAUTH=@abs_top_srcdir@/share/rpcauth/rpcauth.py
@ENABLE_WALLET_TRUE@ENABLE_WALLET=true
@BUILD_BITCOIN_CLI_TRUE@ENABLE_CLI=true
@BUILD_BITCOIND_TRUE@ENABLE_BITCOIND=true
+@ENABLE_FUZZ_TRUE@ENABLE_FUZZ=true
@ENABLE_ZMQ_TRUE@ENABLE_ZMQ=true
--- a/test/fuzz/test_runner.py
+++ b/test/fuzz/test_runner.py
@ -0,0 +1,133 @@
+#!/usr/bin/env python3
+# Copyright (c) 2019 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+"""Run fuzz test targets.
+"""
+
+import argparse
+import configparser
+import os
+import sys
+import subprocess
+import logging
+
+
+def main():
+    parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    parser.add_argument(
+        "-l",
+        "--loglevel",
+        dest="loglevel",
+        default="INFO",
+        help="log events at this level and higher to the console. Can be set to DEBUG, INFO, WARNING, ERROR or CRITICAL. Passing --loglevel DEBUG will output all logs to console.",
+    )
+    parser.add_argument(
+        '--export_coverage',
+        action='store_true',
+        help='If true, export coverage information to files in the seed corpus',
+    )
+    parser.add_argument(
+        'seed_dir',
+        help='The seed corpus to run on (must contain subfolders for each fuzz target).',
+    )
+    parser.add_argument(
+        'target',
+        nargs='*',
+        help='The target(s) to run. Default is to run all targets.',
+    )
+
+    args = parser.parse_args()
+
+    # Set up logging
+    logging.basicConfig(
+        format='%(message)s',
+        level=int(args.loglevel) if args.loglevel.isdigit() else args.loglevel.upper(),
+    )
+
+    # Read config generated by configure.
+    config = configparser.ConfigParser()
+    configfile = os.path.abspath(os.path.dirname(__file__)) + "/../config.ini"
+    config.read_file(open(configfile, encoding="utf8"))
+
+    if not config["components"].getboolean("ENABLE_FUZZ"):
+        logging.error("Must have fuzz targets built")
+        sys.exit(1)
+
+    # Build list of tests
+    test_list_all = parse_test_list(makefile=os.path.join(config["environment"]["SRCDIR"], 'src', 'Makefile.test.include'))
+
+    if not test_list_all:
+        logging.error("No fuzz targets found")
+        sys.exit(1)
+
+    logging.info("Fuzz targets found: {}".format(test_list_all))
+
+    args.target = args.target or test_list_all  # By default run all
+    test_list_error = list(set(args.target).difference(set(test_list_all)))
+    if test_list_error:
+        logging.error("Unknown fuzz targets selected: {}".format(test_list_error))
+    test_list_selection = list(set(test_list_all).intersection(set(args.target)))
+    if not test_list_selection:
+        logging.error("No fuzz targets selected")
+    logging.info("Fuzz targets selected: {}".format(test_list_selection))
+
+    help_output = subprocess.run(
+        args=[
+            os.path.join(config["environment"]["BUILDDIR"], 'src', 'test', 'fuzz', test_list_selection[0]),
+            '-help=1',
+        ],
+        check=True,
+        stderr=subprocess.PIPE,
+        universal_newlines=True,
+    ).stderr
+    if "libFuzzer" not in help_output:
+        logging.error("Must be built with libFuzzer")
+        sys.exit(1)
+
+    run_once(
+        corpus=args.seed_dir,
+        test_list=test_list_selection,
+        build_dir=config["environment"]["BUILDDIR"],
+        export_coverage=args.export_coverage,
+    )
+
+
+def run_once(*, corpus, test_list, build_dir, export_coverage):
+    for t in test_list:
+        args = [
+            os.path.join(build_dir, 'src', 'test', 'fuzz', t),
+            '-runs=1',
+            os.path.join(corpus, t),
+        ]
+        logging.debug('Run {} with args {}'.format(t, args))
+        output = subprocess.run(args, check=True, stderr=subprocess.PIPE, universal_newlines=True).stderr
+        logging.debug('Output: {}'.format(output))
+        if not export_coverage:
+            continue
+        for l in output.splitlines():
+            if 'INITED' in l:
+                with open(os.path.join(corpus, t + '_coverage'), 'w', encoding='utf-8') as cov_file:
+                    cov_file.write(l)
+                    break
+
+
+def parse_test_list(makefile):
+    with open(makefile, encoding='utf-8') as makefile_test:
+        test_list_all = []
+        read_targets = False
+        for line in makefile_test.readlines():
+            line = line.strip().replace('test/fuzz/', '').replace(' \\', '')
+            if read_targets:
+                if not line:
+                    break
+                test_list_all.append(line)
+                continue
+
+            if line == 'FUZZ_TARGETS =':
+                read_targets = True
+    return test_list_all
+
+
+if __name__ == '__main__':
+    main()