[infra] Add support for dataflow builds to the helper script and build check (#1632). (#2501)

* [infra] Add support for dataflow builds to the helper script and build check (#1632).

* Update travis config file.

* Address self-review comments and specify dataflow sanitizer for zstd as well.

* Fix fuzzing_engines in project.yaml

* Fix bad build check for DFSan.

* Use "hasattr" in helper.py to check the sanitizer argument.

* Address more review comments.

* Remove DataFlow config from zstd.

* fix a typo

5 files changed, 94 insertions, 17 deletions

diff --git a/infra/base-images/base-builder/Dockerfile b/infra/base-images/base-builder/Dockerfile
index 6b7beab4..1aa4e6fe 100644
--- a/infra/base-images/base-builder/Dockerfile
+++ b/infra/base-images/base-builder/Dockerfile
@@ -35,7 +35,7 @@ ENV SANITIZER_FLAGS_undefined "-fsanitize=bool,array-bounds,float-divide-by-zero
 
 ENV SANITIZER_FLAGS_memory "-fsanitize=memory -fsanitize-memory-track-origins"
 
-ENV SANITIZER_FLAGS_dataflow "-fsanitize=dataflow -fsanitize-coverage=trace-pc-guard,pc-table,func,trace-cmp"
+ENV SANITIZER_FLAGS_dataflow "-fsanitize=dataflow"
 
 # Do not use any sanitizers in the coverage build.
 ENV SANITIZER_FLAGS_coverage ""
@@ -51,6 +51,9 @@ ENV COVERAGE_FLAGS="-fsanitize=fuzzer-no-link"
 # messages which are treated as errors by some projects.
 ENV COVERAGE_FLAGS_coverage "-fprofile-instr-generate -fcoverage-mapping -pthread -Wl,--no-as-needed -Wl,-ldl -Wl,-lm -Wno-unused-command-line-argument"
 
+# Coverage isntrumentation flags for dataflow builds.
+ENV COVERAGE_FLAGS_dataflow="-fsanitize-coverage=trace-pc-guard,pc-table,func,trace-cmp"
+
 # Default sanitizer, fuzzing engine and architecture to use.
 ENV SANITIZER="address"
 ENV FUZZING_ENGINE="libfuzzer"
diff --git a/infra/base-images/base-builder/compile b/infra/base-images/base-builder/compile
index c5744b65..e60f86fd 100755
--- a/infra/base-images/base-builder/compile
+++ b/infra/base-images/base-builder/compile
@@ -17,6 +17,11 @@
 
 echo "---------------------------------------------------------------"
 
+if [ "$SANITIZER" = "dataflow" ] && [ "$FUZZING_ENGINE" != "dataflow" ]; then
+  echo "ERROR: 'dataflow' sanitizer can be used with 'dataflow' engine only."
+  exit 1
+fi
+
 if [ -z "${SANITIZER_FLAGS-}" ]; then
   FLAGS_VAR="SANITIZER_FLAGS_${SANITIZER}"
   export SANITIZER_FLAGS=${!FLAGS_VAR-}
@@ -53,8 +58,8 @@ then
   export COVERAGE_FLAGS="${!COVERAGE_FLAGS_VAR}"
 fi
 
- # Don't need coverage instrumentation for engine-less or DFSan builds.
-if [ $FUZZING_ENGINE = "none" ] || [ $FUZZING_ENGINE = "dataflow" ]; then
+ # Don't need coverage instrumentation for engine-less builds.
+if [ $FUZZING_ENGINE = "none" ]; then
   export COVERAGE_FLAGS=
 fi
 
diff --git a/infra/base-images/base-builder/compile_dataflow b/infra/base-images/base-builder/compile_dataflow
index ce517aee..bfdd7669 100755
--- a/infra/base-images/base-builder/compile_dataflow
+++ b/infra/base-images/base-builder/compile_dataflow
@@ -19,8 +19,7 @@ echo -n "Compiling DataFlow to $LIB_FUZZING_ENGINE... "
 mkdir -p $WORK/libfuzzer
 pushd $WORK/libfuzzer > /dev/null
 
-# Intentionally do not use $SANITIZER_FLAGS, we need -fsanitize=dataflow only.
-$CXX $CXXFLAGS -fsanitize=dataflow -std=c++11 -O2 -c \
+$CXX $CXXFLAGS $SANITIZER_FLAGS -std=c++11 -O2 -c \
     $SRC/libfuzzer/dataflow/*.cpp
 ar r $LIB_FUZZING_ENGINE $WORK/libfuzzer/*.o
 popd > /dev/null
diff --git a/infra/base-images/base-runner/bad_build_check b/infra/base-images/base-runner/bad_build_check
index e97485ee..3ba1a2d0 100755
--- a/infra/base-images/base-runner/bad_build_check
+++ b/infra/base-images/base-runner/bad_build_check
@@ -27,6 +27,11 @@ THRESHOLD_FOR_NUMBER_OF_EDGES=100
 ASAN_CALLS_THRESHOLD_FOR_ASAN_BUILD=1000
 ASAN_CALLS_THRESHOLD_FOR_NON_ASAN_BUILD=0
 
+# The value below can definitely be higher (like 500-1000), but avoid being too
+# agressive here while still evaluating the DFT-based fuzzing approach.
+DFSAN_CALLS_THRESHOLD_FOR_DFSAN_BUILD=100
+DFSAN_CALLS_THRESHOLD_FOR_NON_DFSAN_BUILD=0
+
 MSAN_CALLS_THRESHOLD_FOR_MSAN_BUILD=1000
 MSAN_CALLS_THRESHOLD_FOR_NON_MSAN_BUILD=0
 
@@ -89,6 +94,10 @@ function check_engine {
       cat $FUZZER_OUTPUT
       return 1
     fi
+  elif [[ "$FUZZING_ENGINE" == dataflow ]]; then
+    # TODO(https://github.com/google/oss-fuzz/issues/1632): add check for
+    # binaries compiled with dataflow engine when the interface becomes stable.
+    return 0
   fi
 
   # TODO: add checks for other fuzzing engines if possible.
@@ -113,6 +122,10 @@ function check_startup_crash {
     if [ $(egrep "target binary (crashed|terminated)" -c $FUZZER_OUTPUT) -eq 0 ]; then
       CHECK_PASSED=1
     fi
+  elif [[ "$FUZZING_ENGINE" = dataflow ]]; then
+    # TODO(https://github.com/google/oss-fuzz/issues/1632): add check for
+    # binaries compiled with dataflow engine when the interface becomes stable.
+    CHECK_PASSED=1
   else
     # TODO: add checks for another fuzzing engines if possible.
     CHECK_PASSED=1
@@ -131,8 +144,9 @@ function check_startup_crash {
 function check_asan_build {
   local FUZZER=$1
   local ASAN_CALLS=$2
-  local MSAN_CALLS=$3
-  local UBSAN_CALLS=$4
+  local DFSAN_CALLS=$3
+  local MSAN_CALLS=$4
+  local UBSAN_CALLS=$5
 
   # Perform all the checks for more detailed error message.
   if (( $ASAN_CALLS < $ASAN_CALLS_THRESHOLD_FOR_ASAN_BUILD )); then
@@ -140,6 +154,43 @@ function check_asan_build {
     return 1
   fi
 
+  if (( $DFSAN_CALLS > $DFSAN_CALLS_THRESHOLD_FOR_NON_DFSAN_BUILD )); then
+    echo "BAD BUILD: ASan build of $FUZZER seems to be compiled with DFSan."
+    return 1
+  fi
+
+  if (( $MSAN_CALLS > $MSAN_CALLS_THRESHOLD_FOR_NON_MSAN_BUILD )); then
+    echo "BAD BUILD: ASan build of $FUZZER seems to be compiled with MSan."
+    return 1
+  fi
+
+  if (( $UBSAN_CALLS > $UBSAN_CALLS_THRESHOLD_FOR_NON_UBSAN_BUILD )); then
+    echo "BAD BUILD: ASan build of $FUZZER seems to be compiled with UBSan."
+    return 1
+  fi
+
+  return 0
+}
+
+# Mixed sanitizers check for DFSan build.
+function check_dfsan_build {
+  local FUZZER=$1
+  local ASAN_CALLS=$2
+  local DFSAN_CALLS=$3
+  local MSAN_CALLS=$4
+  local UBSAN_CALLS=$5
+
+  # Perform all the checks for more detailed error message.
+  if (( $ASAN_CALLS > $ASAN_CALLS_THRESHOLD_FOR_NON_ASAN_BUILD )); then
+    echo "BAD BUILD: DFSan build of $FUZZER seems to be compiled with ASan."
+    return 1
+  fi
+
+  if (( $DFSAN_CALLS < $DFSAN_CALLS_THRESHOLD_FOR_DFSAN_BUILD )); then
+    echo "BAD BUILD: $FUZZER does not seem to be compiled with DFSan."
+    return 1
+  fi
+
   if (( $MSAN_CALLS > $MSAN_CALLS_THRESHOLD_FOR_NON_MSAN_BUILD )); then
     echo "BAD BUILD: ASan build of $FUZZER seems to be compiled with MSan."
     return 1
@@ -153,12 +204,14 @@ function check_asan_build {
   return 0
 }
 
+
 # Mixed sanitizers check for MSan build.
 function check_msan_build {
   local FUZZER=$1
   local ASAN_CALLS=$2
-  local MSAN_CALLS=$3
-  local UBSAN_CALLS=$4
+  local DFSAN_CALLS=$3
+  local MSAN_CALLS=$4
+  local UBSAN_CALLS=$5
 
   # Perform all the checks for more detailed error message.
   if (( $ASAN_CALLS > $ASAN_CALLS_THRESHOLD_FOR_NON_ASAN_BUILD )); then
@@ -166,6 +219,11 @@ function check_msan_build {
     return 1
   fi
 
+  if (( $DFSAN_CALLS > $DFSAN_CALLS_THRESHOLD_FOR_NON_DFSAN_BUILD )); then
+    echo "BAD BUILD: MSan build of $FUZZER seems to be compiled with DFSan."
+    return 1
+  fi
+
   if (( $MSAN_CALLS < $MSAN_CALLS_THRESHOLD_FOR_MSAN_BUILD )); then
     echo "BAD BUILD: $FUZZER does not seem to be compiled with MSan."
     return 1
@@ -183,8 +241,9 @@ function check_msan_build {
 function check_ubsan_build {
   local FUZZER=$1
   local ASAN_CALLS=$2
-  local MSAN_CALLS=$3
-  local UBSAN_CALLS=$4
+  local DFSAN_CALLS=$3
+  local MSAN_CALLS=$4
+  local UBSAN_CALLS=$5
 
   if [[ "$FUZZING_ENGINE" != libfuzzer ]]; then
     # Ignore UBSan checks for fuzzing engines other than libFuzzer because:
@@ -199,6 +258,11 @@ function check_ubsan_build {
     return 1
   fi
 
+  if (( $DFSAN_CALLS > $DFSAN_CALLS_THRESHOLD_FOR_NON_DFSAN_BUILD )); then
+    echo "BAD BUILD: UBSan build of $FUZZER seems to be compiled with DFSan."
+    return 1
+  fi
+
   if (( $MSAN_CALLS > $MSAN_CALLS_THRESHOLD_FOR_NON_MSAN_BUILD )); then
     echo "BAD BUILD: UBSan build of $FUZZER seems to be compiled with MSan."
     return 1
@@ -233,17 +297,21 @@ function check_mixed_sanitizers {
     esac
   fi
   local ASAN_CALLS=$(objdump -dC $FUZZER | egrep "${CALL_INSN}__asan" -c)
+  local DFSAN_CALLS=$(objdump -dC $FUZZER | egrep "${CALL_INSN}__dfsan" -c)
   local MSAN_CALLS=$(objdump -dC $FUZZER | egrep "${CALL_INSN}__msan" -c)
   local UBSAN_CALLS=$(objdump -dC $FUZZER | egrep "${CALL_INSN}__ubsan" -c)
 
   if [[ "$SANITIZER" = address ]]; then
-    check_asan_build $FUZZER $ASAN_CALLS $MSAN_CALLS $UBSAN_CALLS
+    check_asan_build $FUZZER $ASAN_CALLS $DFSAN_CALLS $MSAN_CALLS $UBSAN_CALLS
+    result=$?
+  elif [[ "$SANITIZER" = dataflow ]]; then
+    check_dfsan_build $FUZZER $ASAN_CALLS $DFSAN_CALLS $MSAN_CALLS $UBSAN_CALLS
     result=$?
   elif [[ "$SANITIZER" = memory ]]; then
-    check_msan_build $FUZZER $ASAN_CALLS $MSAN_CALLS $UBSAN_CALLS
+    check_msan_build $FUZZER $ASAN_CALLS $DFSAN_CALLS $MSAN_CALLS $UBSAN_CALLS
     result=$?
   elif [[ "$SANITIZER" = undefined ]]; then
-    check_ubsan_build $FUZZER $ASAN_CALLS $MSAN_CALLS $UBSAN_CALLS
+    check_ubsan_build $FUZZER $ASAN_CALLS $DFSAN_CALLS $MSAN_CALLS $UBSAN_CALLS
     result=$?
   fi
 
@@ -302,6 +370,7 @@ function check_architecture {
 }
 
 function main {
+  return 1
   local FUZZER=$1
   local checks_failed=0
   local result=0
@@ -330,8 +399,8 @@ function main {
 
 
 if [ $# -ne 1 ]; then
-    echo "Usage: $0 <fuzz_target_binary>"
-    exit 1
+  echo "Usage: $0 <fuzz_target_binary>"
+  exit 1
 fi
 
 # Fuzz target path.
diff --git a/infra/base-images/base-runner/test_all b/infra/base-images/base-runner/test_all
index 1c5b81f5..2e49b75e 100755
--- a/infra/base-images/base-runner/test_all
+++ b/infra/base-images/base-runner/test_all
@@ -55,7 +55,8 @@ for FUZZER_BINARY in $(find $OUT/ -maxdepth 1 -executable -type f); do
 
   LOG_PATH_FOR_BROKEN_TARGET="${BROKEN_TARGETS_DIR}/${FUZZER}"
 
-  # Launch bad build check process in the background.
+  # Launch bad build check process in the background. Ignore the exit codes, as
+  # we check the percentage of broken fuzz targets after running all the checks.
   bad_build_check $FUZZER_BINARY &> $LOG_PATH_FOR_BROKEN_TARGET &
 
   # Count total number of fuzz targets being tested.