Skip to content

Commit

Permalink
Adding better fuzzing for both XlsInt and XlsFixed.
Browse files Browse the repository at this point in the history 
PiperOrigin-RevId: 608060184
  • Loading branch information
@JoshVarga @copybara-github
JoshVarga authored and copybara-github committed Feb 18, 2024
1 parent 7b21a91 commit 2d427bb
Show file tree
Hide file tree
Showing 6 changed files with 235 additions and 186 deletions.
113 changes: 60 additions & 53 deletions xls/contrib/xlscc/build_rules/cc_generator.bzl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,78 +27,85 @@ def xls_ac_fuzz_binaries(name, deps, seed_start, seed_count, test_ac_fixed, test
All of the outputs of the fuzzing process.
Including generated source code and binaries.
"""
all_outputs = []
cc_files = []
test_list = []
for x in range(seed_count):
test_outputs = []
seed = seed_start + x
srcfile = "{}_{}.cc".format(name, str(seed))
srcfile = "{}_{}.cc".format(name, seed)
cc_files.append(srcfile)
cmd = "./$(location cc_generate_test) -seed=" + str(seed) + " --cc_filepath=$(OUTS) \
--test_ac_int=" + str(test_ac_int) + " --test_ac_fixed=" + str(test_ac_fixed)
native.genrule(
name = "fuzzfiles_{}_{}".format(name, str(seed)),
name = "fuzzfiles_{}_{}".format(name, seed),
outs = [srcfile],
cmd = cmd,
tools = ["cc_generate_test"],
)

native.cc_binary(
name = "{}_{}".format(name, str(seed)),
name = "{}_{}".format(name, seed),
srcs = [srcfile],
deps = [
"@com_github_hlslibs_ac_types//:ac_int",
"@com_github_hlslibs_ac_types//:ac_fixed",
],
)
all_outputs.append("{}_{}".format(name, str(seed)))
all_outputs.append(srcfile)
test_outputs.append("{}_{}".format(name, seed))
test_outputs.append(srcfile)

native.filegroup(
name = "{}_group".format(name),
data = all_outputs,
)
native.filegroup(
name = "{}_group_{}".format(name, seed),
data = test_outputs,
)

all_outputs.extend([
"//xls/contrib/xlscc:synth_only_headers",
"//xls/contrib/xlscc:xlscc",
"@com_github_hlslibs_ac_types//:ac_types_as_data",
])
test_outputs.extend([
"//xls/contrib/xlscc:synth_only_headers",
"//xls/contrib/xlscc:xlscc",
"@com_github_hlslibs_ac_types//:ac_types_as_data",
])

native.cc_test(
native.cc_test(
name = "{}_{}_test".format(name, seed),
testonly = 1,
srcs = ["cc_fuzz_tester.cc"],
data = test_outputs,
args = [
"--seed={}".format(seed),
"--input_path=xls/contrib/xlscc/unit_tests/{}_".format(name),
],
deps = [
":cc_generator",
":unit_test",
"@com_google_absl//absl/container:inlined_vector",
"@com_google_absl//absl/flags:flag",
"@com_google_absl//absl/status:statusor",
"@com_google_absl//absl/strings",
"@com_google_absl//absl/strings:str_format",
"//xls/codegen:combinational_generator",
"//xls/codegen:module_signature",
"//xls/common:init_xls",
"//xls/common:subprocess",
"//xls/common/file:filesystem",
"//xls/common/file:get_runfile_path",
"//xls/common/file:temp_directory",
"//xls/common/logging",
"//xls/common/status:matchers",
"//xls/common/status:status_macros",
"//xls/interpreter:ir_interpreter",
"//xls/ir",
"//xls/ir:events",
"//xls/ir:function_builder",
"//xls/ir:ir_test_base",
"//xls/ir:value",
"//xls/passes:optimization_pass_pipeline",
"//xls/simulation:module_simulator",
"//xls/simulation:verilog_simulators",
] + deps,
)
test_list.append(":{}_{}_test".format(name, seed))
native.test_suite(
name = name,
testonly = 1,
srcs = ["cc_fuzz_tester.cc"],
data = all_outputs,
args = [
"--seed={}".format(seed_start),
"--sample_count={}".format(seed_count),
"--input_path=xls/contrib/xlscc/unit_tests/{}_".format(name),
],
deps = [
":cc_generator",
":unit_test",
"@com_google_absl//absl/container:inlined_vector",
"@com_google_absl//absl/flags:flag",
"@com_google_absl//absl/status:statusor",
"@com_google_absl//absl/strings",
"@com_google_absl//absl/strings:str_format",
"//xls/codegen:combinational_generator",
"//xls/codegen:module_signature",
"//xls/common:init_xls",
"//xls/common:subprocess",
"//xls/common/file:filesystem",
"//xls/common/file:get_runfile_path",
"//xls/common/file:temp_directory",
"//xls/common/logging",
"//xls/common/status:matchers",
"//xls/common/status:status_macros",
"//xls/interpreter:ir_interpreter",
"//xls/ir",
"//xls/ir:events",
"//xls/ir:function_builder",
"//xls/ir:ir_test_base",
"//xls/ir:value",
"//xls/passes:optimization_pass_pipeline",
"//xls/simulation:module_simulator",
"//xls/simulation:verilog_simulators",
] + deps,
tests = test_list,
)
return [DefaultInfo(files = depset(all_outputs))]
return [DefaultInfo(files = depset(cc_files))]
4 changes: 2 additions & 2 deletions xls/contrib/xlscc/unit_tests/BUILD
View file
Original file line number Diff line number Diff line change
Expand Up @@ -389,7 +389,7 @@ cc_binary(

xls_ac_fuzz_binaries(
name = "fuzz_int",
seed_count = 10,
seed_count = 1,
seed_start = 10,
test_ac_fixed = False,
test_ac_int = True,
Expand All @@ -406,7 +406,7 @@ xls_ac_fuzz_binaries(

xls_ac_fuzz_binaries(
name = "fuzz_fixed",
seed_count = 10,
seed_count = 1,
seed_start = 10,
test_ac_fixed = True,
test_ac_int = False,
Expand Down
22 changes: 11 additions & 11 deletions xls/contrib/xlscc/unit_tests/cc_fuzz_tester.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.

#include <algorithm>
#include <cstdlib>
#include <filesystem> // NOLINT
#include <iostream>
Expand Down Expand Up @@ -74,7 +75,8 @@ class GeneratedTester : public XlsccTestBase {
xls::InvokeSubprocess({std::string(exec_filename)}));

std::string stripped_result =
absl::StrReplaceAll(result_value.stdout, {{"0b", ""}});
absl::StrReplaceAll(result_value.stdout, {{"0b", ""}, {".", ""}});
std::reverse(stripped_result.begin(), stripped_result.end());
absl::InlinedVector<bool, 1> expected_in;
expected_in.reserve(stripped_result.size());
for (char& ch : stripped_result) {
Expand All @@ -83,7 +85,7 @@ class GeneratedTester : public XlsccTestBase {
auto expected = xls::Value(xls::Bits(expected_in));
XLS_ASSIGN_OR_RETURN(auto calc_result, RunIntTest(expected, test_content));
if (calc_result != expected) {
std::cout << expected << " " << calc_result << '\n';
std::cout << "ac: " << expected << " xls: " << calc_result << '\n';
return absl::InternalError("test failed");
}
return absl::OkStatus();
Expand Down Expand Up @@ -202,7 +204,6 @@ class GeneratedTester : public XlsccTestBase {
TEST_F(GeneratedTester, Simple) {
xlscc::GeneratedTester tester;
int seed = absl::GetFlag(FLAGS_seed);
int sample_count = absl::GetFlag(FLAGS_sample_count);
std::string input_path = absl::GetFlag(FLAGS_input_path);
std::string crash_path = absl::GetFlag(FLAGS_crash_path);
bool run_failed = absl::GetFlag(FLAGS_run_failed);
Expand All @@ -227,15 +228,14 @@ TEST_F(GeneratedTester, Simple) {
} else {
crash_path = temp_dir;
}
for (int i = seed; i < sample_count; i++) {
XLS_ASSERT_OK_AND_ASSIGN(
std::filesystem::path exec_filename,
xls::GetXlsRunfilePath(input_path + std::to_string(i)));
XLS_ASSERT_OK_AND_ASSIGN(
std::filesystem::path exec_filename,
xls::GetXlsRunfilePath(input_path + std::to_string(seed)));

std::filesystem::path cc_filepath = exec_filename.string() + ".cc";
absl::Status status = RunExisting(cc_filepath, exec_filename);
XLS_EXPECT_OK(status) << "failed test case: " << std::to_string(i);
}
std::filesystem::path cc_filepath = exec_filename.string() + ".cc";
absl::Status status = RunExisting(cc_filepath, exec_filename);
XLS_EXPECT_OK(status)
<< "failed test case: " << std::to_string(seed) << " " << cc_filepath;
} else {
absl::StatusOr<std::vector<std::filesystem::path>> files =
xls::GetDirectoryEntries(crash_path);
Expand Down
12 changes: 5 additions & 7 deletions xls/contrib/xlscc/unit_tests/cc_generate_test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,11 +38,9 @@ ABSL_FLAG(std::string, cc_filepath, "", "output file name");

namespace {

enum class TestType: uint8_t { kAcInt, kAcFixed };

static bool GenerateTest(int seed, const std::string& filename, TestType type) {
std::string content = type == TestType::kAcInt ? xlscc::GenerateIntTest(seed)
: xlscc::GenerateFixedTest(seed);
static bool GenerateTest(int seed, const std::string& filename,
xlscc::VariableType type) {
std::string content = xlscc::GenerateTest(seed, type);
std::cout << filename << '\n';
absl::Status contents_set = xls::SetFileContents(filename, content);
return contents_set.ok();
Expand All @@ -63,10 +61,10 @@ int main(int argc, char** argv) {

bool success = true;
if (absl::GetFlag(FLAGS_test_ac_fixed)) {
success &= GenerateTest(seed, cc_filepath, TestType::kAcFixed);
success &= GenerateTest(seed, cc_filepath, xlscc::VariableType::kAcFixed);
}
if (absl::GetFlag(FLAGS_test_ac_int)) {
success &= GenerateTest(seed, cc_filepath, TestType::kAcInt);
success &= GenerateTest(seed, cc_filepath, xlscc::VariableType::kAcInt);
}
return success ? EXIT_SUCCESS : EXIT_FAILURE;
}
Loading

0 comments on commit 2d427bb

Please sign in to comment.