[fuzzing] Run fuzzers a little bit as a test (grpc#29919)

* [fuzzer] Add a script to sample fuzzers

* remember the script

* add ci

* bleh

* fix

* Update sample_fuzzers.sh

* tweak

* tweak

* tweak

* tweak

* tweak

* fix fuzzer found bug

* add explainer

* make it bold af

* limit max fuzzing time in addition to runs

Author: ctiller

bazel/grpc_build_system.bzl

@@ -378,14 +378,15 @@ def grpc_cc_test(name, srcs = [], deps = [], external_deps = [], args = [], data
         "linkstatic": linkstatic,
     }
 
-    ios_cc_test(
-        name = name,
-        srcs = srcs,
-        tags = tags,
-        deps = core_deps,
-        args = args,
-        **test_args
-    )
+    if "grpc-fuzzer" not in tags:
+        ios_cc_test(
+            name = name,
+            srcs = srcs,
+            tags = tags,
+            deps = core_deps,
+            args = args,
+            **test_args
+        )
     if not uses_polling:
         # the test behavior doesn't depend on polling, just generate the test
         native.cc_test(

src/core/lib/channel/promise_based_filter.cc

@@ -591,7 +591,13 @@ void ClientCallData::StartBatch(grpc_transport_stream_op_batch* b) {
     batch.CancelWith(GRPC_ERROR_REF(cancelled_error_), &flusher);
   }
 
-  if (batch.is_captured()) batch.ResumeWith(&flusher);
+  if (batch.is_captured()) {
+    if (!is_last()) {
+      batch.ResumeWith(&flusher);
+    } else {
+      batch.CancelWith(GRPC_ERROR_CANCELLED, &flusher);
+    }
+  }
 }
 
 // Handle cancellation.

src/core/lib/channel/promise_based_filter.h

@@ -171,6 +171,7 @@ class BaseCallData : public Activity, private Wakeable {
     ~Flusher();
 
     void Resume(grpc_transport_stream_op_batch* batch) {
+      GPR_ASSERT(!call_->is_last());
       release_.push_back(batch);
     }
 

test/core/end2end/fuzzers/api_fuzzer_corpus/crash-a807143b6ea93ba41d3b8e76bb99d06022c82565

@@ -0,0 +1,21 @@
+actions {
+  create_channel {
+    channel_actions {
+    }
+  }
+}
+actions {
+  create_call {
+    method {
+      value: "contenttype"
+    }
+  }
+}
+actions {
+  queue_batch {
+    operations {
+      receive_initial_metadata {
+      }
+    }
+  }
+}

test/core/slice/BUILD

@@ -27,6 +27,7 @@ grpc_fuzzer(
     tags = ["no_windows"],
     deps = [
         "//:grpc",
+        "//test/core/util:grpc_suppressions",
     ],
 )
 
@@ -38,6 +39,7 @@ grpc_fuzzer(
     tags = ["no_windows"],
     deps = [
         "//:grpc",
+        "//test/core/util:grpc_suppressions",
     ],
 )
 

test/core/util/grpc_fuzzer.bzl

@@ -20,7 +20,7 @@ load("//bazel:grpc_build_system.bzl", "grpc_cc_test")
 load("@rules_proto//proto:defs.bzl", "proto_library")
 load("@rules_cc//cc:defs.bzl", "cc_proto_library")
 
-def grpc_fuzzer(name, corpus, srcs = [], deps = [], data = [], size = "large", **kwargs):
+def grpc_fuzzer(name, corpus, srcs = [], tags = [], deps = [], data = [], size = "large", **kwargs):
     """Instantiates a fuzzer test.
 
     Args:
@@ -30,12 +30,14 @@ def grpc_fuzzer(name, corpus, srcs = [], deps = [], data = [], size = "large", *
         deps: The dependencies of the test.
         data: The data dependencies of the test.
         size: The size of the test.
+        tags: The tags for the test.
         **kwargs: Other arguments to supply to the test.
     """
     CORPUS_DIR = native.package_name() + "/" + corpus
     grpc_cc_test(
         name = name,
         srcs = srcs,
+        tags = tags + ["grpc-fuzzer", "no-cache"],
         deps = deps + select({
             "//:grpc_build_fuzzers": [],
             "//conditions:default": ["//test/core/util:fuzzer_corpus_test"],
@@ -46,13 +48,13 @@ def grpc_fuzzer(name, corpus, srcs = [], deps = [], data = [], size = "large", *
         ],
         size = size,
         args = select({
-            "//:grpc_build_fuzzers": [CORPUS_DIR],
+            "//:grpc_build_fuzzers": [CORPUS_DIR, "-runs=5000", "-max_total_time=300"],
             "//conditions:default": ["--directory=" + CORPUS_DIR],
         }),
         **kwargs
     )
 
-def grpc_proto_fuzzer(name, corpus, proto, srcs = [], deps = [], data = [], size = "large", **kwargs):
+def grpc_proto_fuzzer(name, corpus, proto, srcs = [], tags = [], deps = [], data = [], size = "large", **kwargs):
     """Instantiates a protobuf mutator fuzzer test.
 
     Args:
@@ -63,6 +65,7 @@ def grpc_proto_fuzzer(name, corpus, proto, srcs = [], deps = [], data = [], size
         deps: The dependencies of the test.
         data: The data dependencies of the test.
         size: The size of the test.
+        tags: The tags for the test.
         **kwargs: Other arguments to supply to the test.
     """
     PROTO_LIBRARY = "_%s_proto" % name
@@ -82,6 +85,7 @@ def grpc_proto_fuzzer(name, corpus, proto, srcs = [], deps = [], data = [], size
     grpc_cc_test(
         name = name,
         srcs = srcs,
+        tags = tags + ["grpc-fuzzer", "no-cache"],
         deps = deps + [
             "@com_google_libprotobuf_mutator//:libprotobuf_mutator",
             CC_PROTO_LIBRARY,
@@ -95,7 +99,7 @@ def grpc_proto_fuzzer(name, corpus, proto, srcs = [], deps = [], data = [], size
         ],
         size = size,
         args = select({
-            "//:grpc_build_fuzzers": [CORPUS_DIR],
+            "//:grpc_build_fuzzers": [CORPUS_DIR, "-runs=5000", "-max_total_time=300"],
             "//conditions:default": ["--directory=" + CORPUS_DIR],
         }),
         **kwargs

tools/fuzzing/sample_fuzzers.sh

@@ -0,0 +1,19 @@
+#! /bin/bash -ex
+# Copyright 2022 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+dir=$(dirname "${0}")
+cd "${dir}/../.."
+tools/bazel test `tools/bazel query "attr(tags, '\\bgrpc-fuzzer\\b', //test/...)"` \
+  -c dbg --config fuzzer_asan --test_output=errors

tools/internal_ci/linux/grpc_sample_fuzzers.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+# Enter the gRPC repo root
+cd $(dirname $0)/../../..
+
+cat tools/internal_ci/linux/grpc_sample_fuzzers_failure_explanation.txt
+
+source tools/internal_ci/helper_scripts/prepare_build_linux_rc
+
+export DOCKERFILE_DIR=tools/dockerfile/test/cxx_debian11_x64
+export DOCKER_RUN_SCRIPT=tools/internal_ci/linux/grpc_sample_fuzzers_in_docker.sh
+exec tools/run_tests/dockerize/build_and_run_docker.sh

tools/internal_ci/linux/grpc_sample_fuzzers_failure_explanation.txt

@@ -0,0 +1,31 @@
+*******************************************************************************
+*******************************************************************************
+*******************************************************************************
+*******************************************************************************
+*******************************************************************************
+
+
+HEY THIS TEST IS FAILING AND I DONT KNOW HOW TO REPRODUCE IT??
+
+At the point when this file was authored, I had no idea how to get the crash
+artifacts out of bazel/the fuzzer/kokoro/somewhere and onto your machine.
+
+Without that this seemed like a problem - why check whether fuzzers pass if
+we can't debug them?
+
+My thinking is this: if we can find an example that fails within a few minutes
+in CI, we can probably do likewise on your machine.
+
+Try:
+
+bazel build path/to/fuzzer --config=fuzzer_asan -c dbg
+bazel-bin/path/to/fuzzer path/to/fuzzer_corpus -jobs=$CPUS -workers=$CPUS
+
+You should see whatever failure this CI job found before terribly long.
+
+
+*******************************************************************************
+*******************************************************************************
+*******************************************************************************
+*******************************************************************************
+*******************************************************************************

tools/internal_ci/linux/grpc_sample_fuzzers_in_docker.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+# Copyright 2022 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+# Enter the gRPC repo root
+cd $(dirname $0)/../../..
+
+# some extra pip packages are needed for the check_on_pr.py script to work
+# TODO(jtattermusch): avoid needing to install these pip packages each time
+time python3 -m pip install --user -r tools/internal_ci/helper_scripts/requirements.linux_perf.txt
+
+tools/run_tests/start_port_server.py
+
+tools/internal_ci/linux/run_if_c_cpp_modified.sh tools/fuzzing/sample_fuzzers.sh \
+  -d "origin/$KOKORO_GITHUB_PULL_REQUEST_TARGET_BRANCH"

tools/internal_ci/linux/pull_request/grpc_sample_fuzzers.cfg

@@ -0,0 +1,35 @@
+# Copyright 2022 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Config file for the internal CI (in protobuf text format)
+
+# Location of the continuous shell script in repository.
+build_file: "grpc/tools/internal_ci/linux/grpc_sample_fuzzers.sh"
+timeout_mins: 120
+before_action {
+  fetch_keystore {
+    keystore_resource {
+      keystore_config_id: 73836
+      keyname: "grpc_checks_private_key"
+    }
+  }
+}
+action {
+  define_artifacts {
+    regex: "**/*sponge_log.*"
+    regex: "**/test.log"
+    regex: "github/grpc/reports/**"
+    regex: "**/crash-*"
+  }
+}