Skip to content

Commit cc6621c

Browse files
Stranger6667Stranger6667
committed
chore: New Schemathesis variants
1 parent 8452d88 commit cc6621c

File tree

8 files changed

+802
-94
lines changed

8 files changed

+802
-94
lines changed

‎postprocessing/src/fuzzers/mod.rs

+16
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,11 @@ pub enum Fuzzer {
4545
pub enum SchemathesisKind {
4646
Default,
4747
AllChecks,
48+
LessPreProcessing,
4849
Negative,
50+
NegativeNoSwarm,
51+
NoFormats,
52+
NoMutations,
4953
StatefulOld,
5054
StatefulNew,
5155
}
@@ -55,7 +59,11 @@ impl fmt::Display for SchemathesisKind {
5559
match self {
5660
SchemathesisKind::Default => f.write_str("Default"),
5761
SchemathesisKind::AllChecks => f.write_str("AllChecks"),
62+
SchemathesisKind::LessPreProcessing => f.write_str("LessPreProcessing"),
5863
SchemathesisKind::Negative => f.write_str("Negative"),
64+
SchemathesisKind::NegativeNoSwarm => f.write_str("NegativeNoSwarm"),
65+
SchemathesisKind::NoFormats => f.write_str("NoFormats"),
66+
SchemathesisKind::NoMutations => f.write_str("NoMutations"),
5967
SchemathesisKind::StatefulOld => f.write_str("StatefulOld"),
6068
SchemathesisKind::StatefulNew => f.write_str("StatefulNew"),
6169
}
@@ -82,7 +90,15 @@ impl FromStr for Fuzzer {
8290
"tnt_fuzzer" => Ok(Fuzzer::TntFuzzer),
8391
"schemathesis:Default" => Ok(Fuzzer::Schemathesis(SchemathesisKind::Default)),
8492
"schemathesis:AllChecks" => Ok(Fuzzer::Schemathesis(SchemathesisKind::AllChecks)),
93+
"schemathesis:LessPreProcessing" => {
94+
Ok(Fuzzer::Schemathesis(SchemathesisKind::LessPreProcessing))
95+
}
8596
"schemathesis:Negative" => Ok(Fuzzer::Schemathesis(SchemathesisKind::Negative)),
97+
"schemathesis:NegativeNoSwarm" => {
98+
Ok(Fuzzer::Schemathesis(SchemathesisKind::NegativeNoSwarm))
99+
}
100+
"schemathesis:NoFormats" => Ok(Fuzzer::Schemathesis(SchemathesisKind::NoFormats)),
101+
"schemathesis:NoMutations" => Ok(Fuzzer::Schemathesis(SchemathesisKind::NoMutations)),
86102
"schemathesis:StatefulOld" => Ok(Fuzzer::Schemathesis(SchemathesisKind::StatefulOld)),
87103
"schemathesis:StatefulNew" => Ok(Fuzzer::Schemathesis(SchemathesisKind::StatefulNew)),
88104
"restler" => Ok(Fuzzer::Restler),

‎run.py

+61-92
Original file line numberDiff line numberDiff line change
@@ -13,136 +13,105 @@
1313
logger = structlog.get_logger()
1414
load_dotenv()
1515

16+
BASIC_SCHEMATHESIS = (
17+
"schemathesis:AllChecks",
18+
"schemathesis:Default",
19+
"schemathesis:LessPreProcessing",
20+
"schemathesis:Negative",
21+
"schemathesis:NegativeNoSwarm",
22+
"schemathesis:NoFormats",
23+
"schemathesis:NoMutations",
24+
)
25+
1626
COMBINATIONS = {
1727
"age_of_empires_2_api:Default": {
1828
"fuzzers": [
1929
"api_fuzzer",
2030
"got_swag",
2131
"restler",
22-
"schemathesis:AllChecks",
23-
"schemathesis:Default",
24-
"schemathesis:Negative",
32+
*BASIC_SCHEMATHESIS,
2533
],
2634
},
27-
"age_of_empires_2_api:Linked": {
28-
"fuzzers": ["schemathesis:StatefulNew", "schemathesis:StatefulOld"],
29-
},
35+
# "age_of_empires_2_api:Linked": {
36+
# "fuzzers": ["schemathesis:StatefulNew", "schemathesis:StatefulOld"],
37+
# },
3038
"cccatalog_api:Default": {
31-
"fuzzers": ["got_swag", "restler", "schemathesis:AllChecks", "schemathesis:Default", "schemathesis:Negative"],
32-
},
33-
"cccatalog_api:Linked": {
34-
"fuzzers": ["schemathesis:StatefulNew"],
39+
"fuzzers": ["got_swag", "restler", *BASIC_SCHEMATHESIS],
3540
},
41+
# "cccatalog_api:Linked": {
42+
# "fuzzers": ["schemathesis:StatefulNew"],
43+
# },
3644
"covid19_japan_web_api:Default": {
3745
"fuzzers": [
3846
"api_fuzzer",
3947
"cats",
4048
"restler",
41-
"schemathesis:AllChecks",
42-
"schemathesis:Default",
43-
"schemathesis:Negative",
49+
*BASIC_SCHEMATHESIS,
4450
],
4551
},
46-
"covid19_japan_web_api:Linked": {
47-
"fuzzers": ["schemathesis:StatefulNew", "schemathesis:StatefulOld"],
48-
},
49-
"disease_sh:Default": {
50-
"fuzzers": ["api_fuzzer", "cats", "schemathesis:AllChecks", "schemathesis:Default", "schemathesis:Negative"]
51-
},
52-
"disease_sh:Linked": {"fuzzers": ["schemathesis:StatefulNew", "schemathesis:StatefulOld"]},
53-
"httpbin": {
54-
"fuzzers": ["api_fuzzer", "schemathesis:AllChecks", "schemathesis:Default", "schemathesis:Negative"],
55-
},
52+
# "covid19_japan_web_api:Linked": {
53+
# "fuzzers": ["schemathesis:StatefulNew", "schemathesis:StatefulOld"],
54+
# },
55+
"disease_sh:Default": {"fuzzers": ["api_fuzzer", "cats", *BASIC_SCHEMATHESIS]},
56+
# "disease_sh:Linked": {"fuzzers": ["schemathesis:StatefulNew", "schemathesis:StatefulOld"]},
57+
# "httpbin": {
58+
# "fuzzers": ["api_fuzzer", "schemathesis:AllChecks", "schemathesis:Default", "schemathesis:LessPreProcessing","schemathesis:Negative", "schemathesis:NegativeNoSwarm","schemathesis:NoFormats","schemathesis:NoMutations"],
59+
# },
5660
"jupyter_server:Default": {
57-
"fuzzers": ["cats", "restler", "schemathesis:AllChecks", "schemathesis:Default", "schemathesis:Negative"],
58-
},
59-
"jupyter_server:Linked": {
60-
"fuzzers": ["schemathesis:StatefulNew", "schemathesis:StatefulOld"],
61+
"fuzzers": ["cats", "restler", *BASIC_SCHEMATHESIS],
6162
},
63+
# "jupyter_server:Linked": {
64+
# "fuzzers": ["schemathesis:StatefulNew", "schemathesis:StatefulOld"],
65+
# },
6266
"jupyterhub:Default": {
63-
"fuzzers": ["schemathesis:AllChecks", "schemathesis:Default", "schemathesis:Negative"],
64-
},
65-
"jupyterhub:Linked": {
66-
"fuzzers": ["schemathesis:StatefulNew", "schemathesis:StatefulOld"],
67-
},
68-
"mailhog": {
69-
"fuzzers": [
70-
"api_fuzzer",
71-
"cats",
72-
"restler",
73-
"schemathesis:AllChecks",
74-
"schemathesis:Default",
75-
"schemathesis:Negative",
76-
]
77-
},
78-
"open_fec:Default": {
79-
"fuzzers": [
80-
"api_fuzzer",
81-
"cats",
82-
"fuzz_lightyear",
83-
"got_swag",
84-
"restler",
85-
"schemathesis:AllChecks",
86-
"schemathesis:Default",
87-
"schemathesis:Negative",
88-
"swagger_fuzzer",
89-
],
90-
},
91-
"open_fec:Linked": {
92-
"fuzzers": ["schemathesis:StatefulNew", "schemathesis:StatefulOld"],
93-
},
67+
"fuzzers": [*BASIC_SCHEMATHESIS],
68+
},
69+
# "jupyterhub:Linked": {
70+
# "fuzzers": ["schemathesis:StatefulNew", "schemathesis:StatefulOld"],
71+
# },
72+
"mailhog": {"fuzzers": ["api_fuzzer", "cats", "restler", *BASIC_SCHEMATHESIS]},
73+
# "open_fec:Default": {
74+
# "fuzzers": [
75+
# "api_fuzzer",
76+
# "cats",
77+
# "fuzz_lightyear",
78+
# "got_swag",
79+
# "restler",
80+
# "schemathesis:AllChecks",
81+
# "schemathesis:Default",
82+
# "schemathesis:Negative",
83+
# "swagger_fuzzer",
84+
# ],
85+
# },
86+
# "open_fec:Linked": {
87+
# "fuzzers": ["schemathesis:StatefulNew", "schemathesis:StatefulOld"],
88+
# },
9489
"opentopodata": {
95-
"fuzzers": [
96-
"api_fuzzer",
97-
"cats",
98-
"restler",
99-
"schemathesis:AllChecks",
100-
"schemathesis:Default",
101-
"schemathesis:Negative",
102-
],
90+
"fuzzers": ["api_fuzzer", "cats", "restler", *BASIC_SCHEMATHESIS],
10391
},
104-
"otto_parser": {"fuzzers": ["schemathesis:AllChecks", "schemathesis:Default", "schemathesis:Negative"]},
92+
"otto_parser": {"fuzzers": [*BASIC_SCHEMATHESIS]},
10593
"pslab_webapp": {
106-
"fuzzers": [
107-
"api_fuzzer",
108-
"cats",
109-
"fuzz_lightyear",
110-
"restler",
111-
"schemathesis:AllChecks",
112-
"schemathesis:Default",
113-
"schemathesis:Negative",
114-
],
94+
"fuzzers": ["api_fuzzer", "cats", "fuzz_lightyear", "restler", *BASIC_SCHEMATHESIS],
11595
},
11696
"pulpcore": {
11797
"fuzzers": [
11898
"api_fuzzer",
11999
"cats",
120100
"got_swag",
121-
"schemathesis:AllChecks",
122-
"schemathesis:Default",
123-
"schemathesis:Negative",
101+
*BASIC_SCHEMATHESIS,
124102
"tnt_fuzzer",
125103
],
126104
},
127-
"request_baskets:Default": {
128-
"fuzzers": [
129-
"api_fuzzer",
130-
"cats",
131-
"restler",
132-
"schemathesis:AllChecks",
133-
"schemathesis:Default",
134-
"schemathesis:Negative",
135-
]
136-
},
137-
"request_baskets:Linked": {"fuzzers": ["schemathesis:StatefulNew", "schemathesis:StatefulOld"]},
105+
"request_baskets:Default": {"fuzzers": ["api_fuzzer", "cats", "restler", *BASIC_SCHEMATHESIS]},
106+
# "request_baskets:Linked": {"fuzzers": ["schemathesis:StatefulNew", "schemathesis:StatefulOld"]},
138107
"restler_demo:Default": {
139-
"fuzzers": ["got_swag", "restler", "schemathesis:AllChecks", "schemathesis:Default", "schemathesis:Negative"],
108+
"fuzzers": ["got_swag", "restler", *BASIC_SCHEMATHESIS],
140109
},
141110
"restler_demo:Linked": {
142111
"fuzzers": ["schemathesis:StatefulNew", "schemathesis:StatefulOld"],
143112
},
144113
"worklog:Default": {
145-
"fuzzers": ["api_fuzzer", "restler", "schemathesis:AllChecks", "schemathesis:Default", "schemathesis:Negative"],
114+
"fuzzers": ["api_fuzzer", "restler", *BASIC_SCHEMATHESIS],
146115
},
147116
"worklog:Linked": {
148117
"fuzzers": ["schemathesis:StatefulNew", "schemathesis:StatefulOld"],

‎src/wafp/fuzzers/catalog/schemathesis/Dockerfile

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
FROM schemathesis/schemathesis:v3.19.5
2+
3+
COPY hooks.py hooks.py
4+
5+
USER root
6+
RUN apk add --no-cache git
7+
8+
ARG EXTRA_REQUIREMENTS=empty-requirements.txt
9+
COPY $EXTRA_REQUIREMENTS requirements.txt
10+
11+
RUN pip install --no-cache-dir -r requirements.txt
12+
13+
USER schemathesis

‎src/wafp/fuzzers/catalog/schemathesis/__init__.py

+29
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ def get_environment_variables(self) -> Dict[str, str]:
6262
if self.api_name is not None:
6363
# This is a bit more convenient, as `API_NAME` is an optional positional argument to Schemathesis
6464
env["SCHEMATHESIS_API_NAME"] = self.api_name
65+
env["EXTRA_REQUIREMENTS"] = "empty-requirements.txt"
6566
return env
6667

6768

@@ -82,6 +83,13 @@ def get_entrypoint_args(
8283
return args
8384

8485

86+
class NoMutations(Default):
87+
def get_environment_variables(self) -> Dict[str, str]:
88+
env = super().get_environment_variables()
89+
env["EXTRA_REQUIREMENTS"] = "ablation-mutation-requirements.txt"
90+
return env
91+
92+
8593
class Negative(Default):
8694
def get_entrypoint_args(
8795
self, context: FuzzerContext, schema: str, base_url: str, headers: Dict[str, str], ssl_insecure: bool = False
@@ -91,6 +99,27 @@ def get_entrypoint_args(
9199
return args
92100

93101

102+
class NegativeNoSwarm(Negative):
103+
def get_environment_variables(self) -> Dict[str, str]:
104+
env = super().get_environment_variables()
105+
env["SCHEMATHESIS_DISABLE_SWARM_TESTING"] = "true"
106+
return env
107+
108+
109+
class NoFormats(Default):
110+
def get_environment_variables(self) -> Dict[str, str]:
111+
env = super().get_environment_variables()
112+
env["SCHEMATHESIS_DISABLE_FORMAT_STRATEGIES"] = "true"
113+
return env
114+
115+
116+
class LessPreProcessing(Default):
117+
def get_environment_variables(self) -> Dict[str, str]:
118+
env = super().get_environment_variables()
119+
env["SCHEMATHESIS_USE_LESS_SCHEMA_PRE_PROCESSING"] = "true"
120+
return env
121+
122+
94123
class Fast(Default):
95124
@property
96125
def max_examples(self) -> int:

‎src/wafp/fuzzers/catalog/schemathesis/ablation-mutation-requirements.txt

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
git+https://github.com/Zac-HD/hypothesis.git@ablation-mutation#subdirectory=hypothesis-python

‎src/wafp/fuzzers/catalog/schemathesis/docker-compose.yml

+9-2
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,15 @@
11
version: '3'
22
services:
33
fuzzer:
4-
image: schemathesis/schemathesis:v3.19.5
5-
init: true
4+
build:
5+
context: .
6+
args:
7+
- EXTRA_REQUIREMENTS=${EXTRA_REQUIREMENTS}
68
network_mode: host
79
environment:
10+
- SCHEMATHESIS_HOOKS=hooks
811
- SCHEMATHESIS_API_NAME
12+
- SCHEMATHESIS_DISABLE_SWARM_TESTING
13+
- SCHEMATHESIS_DISABLE_FORMAT_STRATEGIES
14+
- SCHEMATHESIS_USE_LESS_SCHEMA_PRE_PROCESSING
15+
- EXTRA_REQUIREMENTS

‎src/wafp/fuzzers/catalog/schemathesis/empty-requirements.txt

Whitespace-only changes.

0 commit comments

Comments
 (0)