Dictionary support for fuzzers

Author: ctiller

build.yaml

@@ -1185,6 +1185,7 @@ targets:
   - gpr
   corpus_dirs:
   - test/core/end2end/fuzzers/client_fuzzer_corpus
+  dict: test/core/end2end/fuzzers/hpack.dictionary
   maxlen: 2048
 - name: compression_test
   build: test
@@ -1674,6 +1675,7 @@ targets:
   - gpr
   corpus_dirs:
   - test/core/transport/chttp2/hpack_parser_corpus
+  dict: test/core/end2end/fuzzers/hpack.dictionary
   maxlen: 512
 - name: hpack_parser_test
   build: test
@@ -2025,6 +2027,7 @@ targets:
   - gpr
   corpus_dirs:
   - test/core/end2end/fuzzers/server_fuzzer_corpus
+  dict: test/core/end2end/fuzzers/hpack.dictionary
   maxlen: 2048
 - name: server_test
   build: test

templates/tools/fuzzer/runners.template

@@ -37,6 +37,10 @@ template: |
 
   flags="-max_total_time=$runtime -artifact_prefix=fuzzer_output/ -max_len=${selected.maxlen}"
 
+  %if selected.get('dict'):
+  flags="$flags -dict=${selected.dict}"
+  %endif
+
   if [ "$jobs" != "1" ]
   then
     flags="-jobs=$jobs -workers=$jobs $flags"

test/core/end2end/fuzzers/hpack.dictionary

@@ -0,0 +1,91 @@
+# hpack fuzzing dictionary
+kw0="\x01""0"
+kw1="\x01""1"
+kw2="\x01""2"
+kw3="\x03""200"
+kw4="\x03""204"
+kw5="\x03""206"
+kw6="\x03""304"
+kw7="\x03""400"
+kw8="\x03""404"
+kw9="\x03""500"
+kw10="\x06""accept"
+kw11="\x0e""accept-charset"
+kw12="\x0f""accept-encoding"
+kw13="\x0f""accept-language"
+kw14="\x0d""accept-ranges"
+kw15="\x1b""access-control-allow-origin"
+kw16="\x03""age"
+kw17="\x05""allow"
+kw18="\x10""application/grpc"
+kw19="\x0a:authority"
+kw20="\x0d""authorization"
+kw21="\x0d""cache-control"
+kw22="\x0a""census-bin"
+kw23="\x11""census-binary-bin"
+kw24="\x13""content-disposition"
+kw25="\x10""content-encoding"
+kw26="\x10""content-language"
+kw27="\x0e""content-length"
+kw28="\x10""content-location"
+kw29="\x0d""content-range"
+kw30="\x0c""content-type"
+kw31="\x06""cookie"
+kw32="\x04""date"
+kw33="\x07""deflate"
+kw34="\x0c""deflate,gzip"
+kw35="\x00"
+kw36="\x04""etag"
+kw37="\x06""expect"
+kw38="\x07""expires"
+kw39="\x04""from"
+kw40="\x03GET"
+kw41="\x04grpc"
+kw42="\x14grpc-accept-encoding"
+kw43="\x0dgrpc-encoding"
+kw44="\x1egrpc-internal-encoding-request"
+kw45="\x0cgrpc-message"
+kw46="\x0bgrpc-status"
+kw47="\x0cgrpc-timeout"
+kw48="\x04gzip"
+kw49="\x0dgzip, deflate"
+kw50="\x04host"
+kw51="\x04http"
+kw52="\x05https"
+kw53="\x08identity"
+kw54="\x10identity,deflate"
+kw55="\x15identity,deflate,gzip"
+kw56="\x0didentity,gzip"
+kw57="\x08if-match"
+kw58="\x11if-modified-since"
+kw59="\x0dif-none-match"
+kw60="\x08if-range"
+kw61="\x13if-unmodified-since"
+kw62="\x0dlast-modified"
+kw63="\x04link"
+kw64="\x08location"
+kw65="\x0cmax-forwards"
+kw66="\x07:method"
+kw67="\x05:path"
+kw68="\x04POST"
+kw69="\x12proxy-authenticate"
+kw70="\x13proxy-authorization"
+kw71="\x03PUT"
+kw72="\x05range"
+kw73="\x07referer"
+kw74="\x07refresh"
+kw75="\x0bretry-after"
+kw76="\x07:scheme"
+kw77="\x06server"
+kw78="\x0aset-cookie"
+kw79="\x01/"
+kw80="\x0b/index.html"
+kw81="\x07:status"
+kw82="\x19strict-transport-security"
+kw83="\x02te"
+kw84="\x08trailers"
+kw85="\x11transfer-encoding"
+kw86="\x0auser-agent"
+kw87="\x04vary"
+kw88="\x03via"
+kw89="\x10www-authenticate"

tools/codegen/core/gen_static_metadata.py

@@ -205,6 +205,7 @@ def put_banner(files, banner):
 args = sys.argv[1:]
 H = None
 C = None
+D = None
 if args:
   if 'header' in args:
     H = sys.stdout
@@ -214,11 +215,17 @@ def put_banner(files, banner):
     C = sys.stdout
   else:
     C = open('/dev/null', 'w')
+  if 'dictionary' in args:
+    D = sys.stdout
+  else:
+    D = open('/dev/null', 'w')
 else:
   H = open(os.path.join(
       os.path.dirname(sys.argv[0]), '../../../src/core/lib/transport/static_metadata.h'), 'w')
   C = open(os.path.join(
       os.path.dirname(sys.argv[0]), '../../../src/core/lib/transport/static_metadata.c'), 'w')
+  D = open(os.path.join(
+      os.path.dirname(sys.argv[0]), '../../../test/core/end2end/fuzzers/hpack.dictionary'), 'w')
 
 # copy-paste copyright notice from this file
 with open(sys.argv[0]) as my_source:
@@ -235,6 +242,27 @@ def put_banner(files, banner):
     copyright.append(line)
   put_banner([H,C], [line[2:].rstrip() for line in copyright])
 
+
+hex_bytes = [ord(c) for c in "abcdefABCDEF0123456789"]
+
+
+def esc_c(line):
+  out = "\""
+  last_was_hex = False
+  for c in line:
+    if 32 <= c < 127:
+      if c in hex_bytes and last_was_hex:
+        out += "\"\""
+      if c != ord('"'):
+        out += chr(c)
+      else:
+        out += "\\\""
+      last_was_hex = False
+    else:
+      out += "\\x%02x" % c
+      last_was_hex = True
+  return out + "\""
+
 put_banner([H,C],
 """WARNING: Auto-generated code.
 
@@ -263,6 +291,10 @@ def put_banner(files, banner):
 print >>C, 'grpc_mdstr grpc_static_mdstr_table[GRPC_STATIC_MDSTR_COUNT];'
 print >>C
 
+print >>D, '# hpack fuzzing dictionary'
+for i, elem in enumerate(all_strs):
+  print >>D, 'kw%d=%s' % (i, esc_c([len(elem)] + [ord(c) for c in elem]))
+
 print >>H, '#define GRPC_STATIC_MDELEM_COUNT %d' % len(all_elems)
 print >>H, 'extern grpc_mdelem grpc_static_mdelem_table[GRPC_STATIC_MDELEM_COUNT];'
 print >>H, 'extern uintptr_t grpc_static_mdelem_user_data[GRPC_STATIC_MDELEM_COUNT];'

tools/fuzzer/runners/client_fuzzer.sh

@@ -31,6 +31,8 @@
 
 flags="-max_total_time=$runtime -artifact_prefix=fuzzer_output/ -max_len=2048"
 
+flags="$flags -dict=test/core/end2end/fuzzers/hpack.dictionary"
+
 if [ "$jobs" != "1" ]
 then
   flags="-jobs=$jobs -workers=$jobs $flags"

tools/fuzzer/runners/hpack_parser_fuzzer_test.sh

@@ -31,6 +31,8 @@
 
 flags="-max_total_time=$runtime -artifact_prefix=fuzzer_output/ -max_len=512"
 
+flags="$flags -dict=test/core/end2end/fuzzers/hpack.dictionary"
+
 if [ "$jobs" != "1" ]
 then
   flags="-jobs=$jobs -workers=$jobs $flags"

tools/fuzzer/runners/http_fuzzer_test.sh

@@ -31,6 +31,7 @@
 
 flags="-max_total_time=$runtime -artifact_prefix=fuzzer_output/ -max_len=2048"
 
+
 if [ "$jobs" != "1" ]
 then
   flags="-jobs=$jobs -workers=$jobs $flags"

tools/fuzzer/runners/json_fuzzer_test.sh

@@ -31,6 +31,7 @@
 
 flags="-max_total_time=$runtime -artifact_prefix=fuzzer_output/ -max_len=512"
 
+
 if [ "$jobs" != "1" ]
 then
   flags="-jobs=$jobs -workers=$jobs $flags"

tools/fuzzer/runners/nanopb_fuzzer_response_test.sh

@@ -31,6 +31,7 @@
 
 flags="-max_total_time=$runtime -artifact_prefix=fuzzer_output/ -max_len=128"
 
+
 if [ "$jobs" != "1" ]
 then
   flags="-jobs=$jobs -workers=$jobs $flags"

tools/fuzzer/runners/nanopb_fuzzer_serverlist_test.sh

@@ -31,6 +31,7 @@
 
 flags="-max_total_time=$runtime -artifact_prefix=fuzzer_output/ -max_len=128"
 
+
 if [ "$jobs" != "1" ]
 then
   flags="-jobs=$jobs -workers=$jobs $flags"

tools/fuzzer/runners/server_fuzzer.sh

@@ -31,6 +31,8 @@
 
 flags="-max_total_time=$runtime -artifact_prefix=fuzzer_output/ -max_len=2048"
 
+flags="$flags -dict=test/core/end2end/fuzzers/hpack.dictionary"
+
 if [ "$jobs" != "1" ]
 then
   flags="-jobs=$jobs -workers=$jobs $flags"

tools/fuzzer/runners/uri_fuzzer_test.sh

@@ -31,6 +31,7 @@
 
 flags="-max_total_time=$runtime -artifact_prefix=fuzzer_output/ -max_len=128"
 
+
 if [ "$jobs" != "1" ]
 then
   flags="-jobs=$jobs -workers=$jobs $flags"