Skip to content

Commit

Permalink
Switch to --testvector_textproto as input to simulation tools.
Browse files Browse the repository at this point in the history 
Instead of the various ad-hoc serializations and different flags
such as --args_file/--input_file and --channel_values, use testvector proto
universally.

Issues: #1645
PiperOrigin-RevId: 694676298
  • Loading branch information
@hzeller @copybara-github
hzeller authored and copybara-github committed Nov 9, 2024
1 parent abc11e1 commit 887923d
Show file tree
Hide file tree
Showing 5 changed files with 108 additions and 78 deletions.
7 changes: 4 additions & 3 deletions xls/fuzzer/cpp_run_fuzz.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,8 +75,8 @@ absl::StatusOr<std::optional<std::filesystem::path>> MinimizeIr(
{
std::vector<std::string> args = {
std::string{sample_runner_main_path}, "--logtostderr",
"--options_file=ir_minimizer.options.pbtxt", "--args_file=args.txt",
"--input_file=$1"};
"--options_file=ir_minimizer.options.pbtxt",
"--testvector_textproto=testvector.pbtxt", "--input_file=$1"};
const std::filesystem::path test_script = run_dir / "ir_minimizer_test.sh";
XLS_RETURN_IF_ERROR(SetFileContents(
test_script, absl::StrCat("#!/bin/sh\n! ", absl::StrJoin(args, " "))));
Expand Down Expand Up @@ -131,7 +131,8 @@ absl::StatusOr<std::optional<std::filesystem::path>> MinimizeIr(
run_dir / absl::StrCat(basename, ".stderr");

std::vector<std::string> args = {find_failing_input_main_path,
"--input_file=args.txt", "sample.ir"};
"--testvector_textproto=testvector.pbtxt",
"sample.ir"};
args.insert(args.end(), extra_args.begin(), extra_args.end());
XLS_ASSIGN_OR_RETURN(
find_failing_input_result,
Expand Down
12 changes: 4 additions & 8 deletions xls/fuzzer/run_fuzz.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -210,14 +210,11 @@ absl::Status RunSample(const Sample& smp, const std::filesystem::path& run_dir,
testvector::SampleInputsProto testvector;
XLS_RETURN_IF_ERROR(smp.FillSampleInputs(&testvector));
XLS_RETURN_IF_ERROR(SetTextProtoFile(testvector_path, testvector));
// TODO(hzeller): This is a preparation, but testvector.pbtxt is not yet
// passed to tools. This is the egg, chicken follows in next change.
// argv.push_back("--testvector_textproto=testvector.pbtxt");
argv.push_back("--testvector_textproto=testvector.pbtxt");

std::filesystem::path args_file_name = run_dir / "args.txt";
XLS_RETURN_IF_ERROR(
SetFileContents(args_file_name, ArgsBatchToText(smp.args_batch())));
argv.push_back("--args_file=args.txt");

std::optional<std::filesystem::path> ir_channel_names_file_name =
std::nullopt;
Expand All @@ -226,7 +223,6 @@ absl::Status RunSample(const Sample& smp, const std::filesystem::path& run_dir,
XLS_RETURN_IF_ERROR(
SetFileContents(*ir_channel_names_file_name,
IrChannelNamesToText(smp.ir_channel_names())));
argv.push_back("--ir_channel_names_file=ir_channel_names.txt");
}

argv.push_back("\"$RUNDIR\"");
Expand All @@ -246,9 +242,9 @@ cd "$RUNDIR"
VLOG(1) << "Starting to run sample";
VLOG(2) << smp.input_text();
SampleRunner runner(run_dir);
XLS_RETURN_IF_ERROR(runner.RunFromFiles(sample_file_name, options_file_name,
args_file_name,
ir_channel_names_file_name));
XLS_RETURN_IF_ERROR(
runner.RunFromFiles(sample_file_name, options_file_name, args_file_name,
ir_channel_names_file_name, testvector_path));

fuzzer::SampleTimingProto timing = runner.timing();

Expand Down
140 changes: 80 additions & 60 deletions xls/fuzzer/sample_runner.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -321,7 +321,7 @@ absl::StatusOr<std::vector<dslx::InterpValue>> ParseValues(std::string_view s) {
// Evaluate the IR file with a function as its top and return the result Values.
absl::StatusOr<std::vector<dslx::InterpValue>> EvaluateIrFunction(
const std::filesystem::path& ir_path,
const std::filesystem::path& args_path, bool use_jit,
const std::filesystem::path& testvector_path, bool use_jit,
const SampleOptions& options, const std::filesystem::path& run_dir,
const SampleRunner::Commands& commands) {
std::optional<SampleRunner::Commands::Callable> command =
Expand All @@ -332,15 +332,16 @@ absl::StatusOr<std::vector<dslx::InterpValue>> EvaluateIrFunction(

XLS_ASSIGN_OR_RETURN(
std::string results_text,
RunCommand(absl::StrFormat("Evaluating IR file (%s): %s",
(use_jit ? "JIT" : "interpreter"), ir_path),
*command,
{
absl::StrCat("--input_file=", args_path.string()),
absl::StrFormat("--%suse_llvm_jit", use_jit ? "" : "no"),
ir_path,
},
run_dir, options));
RunCommand(
absl::StrFormat("Evaluating IR file (%s): %s",
(use_jit ? "JIT" : "interpreter"), ir_path),
*command,
{
absl::StrCat("--testvector_textproto=", testvector_path.string()),
absl::StrFormat("--%suse_llvm_jit", use_jit ? "" : "no"),
ir_path,
},
run_dir, options));
XLS_RETURN_IF_ERROR(SetFileContents(
absl::StrCat(ir_path.string(), ".results"), results_text));
return ParseValues(results_text);
Expand Down Expand Up @@ -397,7 +398,7 @@ absl::StatusOr<std::filesystem::path> OptimizeIr(
absl::StatusOr<std::vector<dslx::InterpValue>> SimulateFunction(
const std::filesystem::path& verilog_path,
const std::filesystem::path& module_sig_path,
const std::filesystem::path& args_path, const SampleOptions& options,
const std::filesystem::path& testvector_path, const SampleOptions& options,
const std::filesystem::path& run_dir,
const SampleRunner::Commands& commands) {
std::optional<SampleRunner::Commands::Callable> command =
Expand All @@ -408,7 +409,7 @@ absl::StatusOr<std::vector<dslx::InterpValue>> SimulateFunction(

std::vector<std::string> simulator_args = {
absl::StrCat("--signature_file=", module_sig_path.string()),
absl::StrCat("--args_file=", args_path.string()),
absl::StrCat("--testvector_textproto=", testvector_path.string()),
};
XLS_RETURN_IF_ERROR(CheckSimulator(options.simulator()));
if (!options.simulator().empty()) {
Expand Down Expand Up @@ -706,8 +707,8 @@ absl::StatusOr<std::filesystem::path> DslxToIrProc(

absl::StatusOr<absl::flat_hash_map<std::string, std::vector<Value>>>
EvaluateIrProc(const std::filesystem::path& ir_path, int64_t tick_count,
const std::filesystem::path& ir_channel_values_path,
bool use_jit, const SampleOptions& options,
const std::filesystem::path& testvector_proto_path, bool use_jit,
const SampleOptions& options,
const std::filesystem::path& run_dir,
const SampleRunner::Commands& commands) {
std::optional<SampleRunner::Commands::Callable> command =
Expand All @@ -721,8 +722,7 @@ EvaluateIrProc(const std::filesystem::path& ir_path, int64_t tick_count,
absl::StrFormat("Evaluating IR file (%s): %s", evaluation_type, ir_path);
std::string_view backend_type = use_jit ? "serial_jit" : "ir_interpreter";
std::vector<std::string> args = {
absl::StrCat("--inputs_for_all_channels=",
ir_channel_values_path.string()),
absl::StrCat("--testvector_textproto=", testvector_proto_path.string()),
absl::StrCat("--ticks=", tick_count),
absl::StrCat("--backend=", backend_type),
ir_path,
Expand Down Expand Up @@ -780,7 +780,7 @@ std::string GetOutputChannelToString(
absl::StatusOr<absl::flat_hash_map<std::string, std::vector<Value>>>
SimulateProc(const std::filesystem::path& verilog_path,
const std::filesystem::path& module_sig_path,
const std::filesystem::path& ir_channel_values_path,
const std::filesystem::path& testvector_path,
std::string_view output_channel_counts,
const SampleOptions& options, const std::filesystem::path& run_dir,
const SampleRunner::Commands& commands) {
Expand All @@ -792,7 +792,7 @@ SimulateProc(const std::filesystem::path& verilog_path,

std::vector<std::string> simulator_args = {
absl::StrCat("--signature_file=", module_sig_path.string()),
absl::StrCat("--channel_values_file=", ir_channel_values_path.string()),
absl::StrCat("--testvector_textproto=", testvector_path.string()),
absl::StrCat("--output_channel_counts=", output_channel_counts),
};
if (!options.simulator().empty()) {
Expand Down Expand Up @@ -853,14 +853,15 @@ absl::Status SampleRunner::Run(const Sample& sample) {
}

return RunFromFiles(input_path, options_path, args_path,
ir_channel_names_path);
ir_channel_names_path, testvector_path);
}

absl::Status SampleRunner::RunFromFiles(
const std::filesystem::path& input_path,
const std::filesystem::path& options_path,
const std::optional<std::filesystem::path>& args_path,
const std::optional<std::filesystem::path>& ir_channel_names_path) {
const std::optional<std::filesystem::path>& args_path, // deprecated
const std::optional<std::filesystem::path>& ir_channel_names_path, // ditto
const std::optional<std::filesystem::path>& testvector_path) {
VLOG(1) << "Running sample in directory " << run_dir_;
VLOG(1) << "Reading sample files.";

Expand All @@ -874,10 +875,11 @@ absl::Status SampleRunner::RunFromFiles(
absl::Status status;
switch (options.sample_type()) {
case fuzzer::SampleType::SAMPLE_TYPE_FUNCTION:
status = RunFunction(input_path, options, args_path);
status = RunFunction(input_path, options, std::nullopt, testvector_path);
break;
case fuzzer::SampleType::SAMPLE_TYPE_PROC:
status = RunProc(input_path, options, args_path, ir_channel_names_path);
status = RunProc(input_path, options, std::nullopt, std::nullopt,
testvector_path);
break;
default:
status = absl::InvalidArgumentError(
Expand All @@ -901,13 +903,21 @@ absl::Status SampleRunner::RunFromFiles(

absl::Status SampleRunner::RunFunction(
const std::filesystem::path& input_path, const SampleOptions& options,
const std::optional<std::filesystem::path>& args_path) {
const std::optional<std::filesystem::path>& args_path,
const std::optional<std::filesystem::path>& testvector_path) {
XLS_ASSIGN_OR_RETURN(std::string input_text, GetFileContents(input_path));

std::optional<ArgsBatch> args_batch = std::nullopt;
if (args_path.has_value()) {
XLS_ASSIGN_OR_RETURN(std::string args_text, GetFileContents(*args_path));
XLS_ASSIGN_OR_RETURN(args_batch, dslx::ParseArgsBatch(args_text));
} else if (testvector_path.has_value()) {
testvector::SampleInputsProto sample_inputs;
XLS_RETURN_IF_ERROR(ParseTextProtoFile(*testvector_path, &sample_inputs));
ArgsBatch extracted;
XLS_RETURN_IF_ERROR(Sample::ExtractArgsBatch(/* is_proc_samples= */ false,
sample_inputs, extracted));
args_batch = std::move(extracted);
}

absl::flat_hash_map<std::string, std::vector<dslx::InterpValue>> results;
Expand Down Expand Up @@ -938,20 +948,20 @@ absl::Status SampleRunner::RunFunction(
XLS_RETURN_IF_ERROR(SetFileContents(ir_path, input_text));
}

if (args_path.has_value()) {
if (args_batch.has_value()) {
Stopwatch t;

// Unconditionally evaluate with the interpreter even if using the JIT. This
// exercises the interpreter and serves as a reference.
XLS_ASSIGN_OR_RETURN(results["evaluated unopt IR (interpreter)"],
EvaluateIrFunction(ir_path, *args_path, false, options,
run_dir_, commands_));
EvaluateIrFunction(ir_path, *testvector_path, false,
options, run_dir_, commands_));
timing_.set_unoptimized_interpret_ir_ns(
absl::ToInt64Nanoseconds(t.GetElapsedTime()));

if (options.use_jit()) {
XLS_ASSIGN_OR_RETURN(results["evaluated unopt IR (JIT)"],
EvaluateIrFunction(ir_path, *args_path, true,
EvaluateIrFunction(ir_path, *testvector_path, true,
options, run_dir_, commands_));
timing_.set_unoptimized_jit_ns(
absl::ToInt64Nanoseconds(t.GetElapsedTime()));
Expand All @@ -964,19 +974,21 @@ absl::Status SampleRunner::RunFunction(
OptimizeIr(ir_path, options, run_dir_, commands_));
timing_.set_optimize_ns(absl::ToInt64Nanoseconds(t.GetElapsedTime()));

if (args_path.has_value()) {
if (args_batch.has_value()) {
if (options.use_jit()) {
t.Reset();
XLS_ASSIGN_OR_RETURN(results["evaluated opt IR (JIT)"],
EvaluateIrFunction(opt_ir_path, *args_path, true,
options, run_dir_, commands_));
XLS_ASSIGN_OR_RETURN(
results["evaluated opt IR (JIT)"],
EvaluateIrFunction(opt_ir_path, *testvector_path, true, options,
run_dir_, commands_));
timing_.set_optimized_jit_ns(
absl::ToInt64Nanoseconds(t.GetElapsedTime()));
}
t.Reset();
XLS_ASSIGN_OR_RETURN(results["evaluated opt IR (interpreter)"],
EvaluateIrFunction(opt_ir_path, *args_path, false,
options, run_dir_, commands_));
XLS_ASSIGN_OR_RETURN(
results["evaluated opt IR (interpreter)"],
EvaluateIrFunction(opt_ir_path, *testvector_path, false, options,
run_dir_, commands_));
timing_.set_optimized_interpret_ir_ns(
absl::ToInt64Nanoseconds(t.GetElapsedTime()));
}
Expand All @@ -989,12 +1001,11 @@ absl::Status SampleRunner::RunFunction(
timing_.set_codegen_ns(absl::ToInt64Nanoseconds(t.GetElapsedTime()));

if (options.simulate()) {
XLS_RET_CHECK(args_path.has_value());
t.Reset();
XLS_ASSIGN_OR_RETURN(
results["simulated"],
SimulateFunction(verilog_path, "module_sig.textproto", *args_path,
options, run_dir_, commands_));
SimulateFunction(verilog_path, "module_sig.textproto",
*testvector_path, options, run_dir_, commands_));
timing_.set_simulate_ns(absl::ToInt64Nanoseconds(t.GetElapsedTime()));
}
}
Expand All @@ -1009,20 +1020,31 @@ absl::Status SampleRunner::RunFunction(
absl::Status SampleRunner::RunProc(
const std::filesystem::path& input_path, const SampleOptions& options,
const std::optional<std::filesystem::path>& args_path,
const std::optional<std::filesystem::path>& ir_channel_names_path) {
const std::optional<std::filesystem::path>& ir_channel_names_path,
const std::optional<std::filesystem::path>& testvector_path) {
XLS_ASSIGN_OR_RETURN(std::string input_text, GetFileContents(input_path));

std::optional<ArgsBatch> args_batch = std::nullopt;
std::optional<std::vector<std::string>> ir_channel_names = std::nullopt;

if (args_path.has_value()) {
XLS_ASSIGN_OR_RETURN(std::string args_text, GetFileContents(*args_path));
XLS_ASSIGN_OR_RETURN(args_batch, dslx::ParseArgsBatch(args_text));
}

std::optional<std::vector<std::string>> ir_channel_names = std::nullopt;
if (ir_channel_names_path.has_value()) {
XLS_ASSIGN_OR_RETURN(std::string ir_channel_names_text,
GetFileContents(*ir_channel_names_path));
ir_channel_names = ParseIrChannelNames(ir_channel_names_text);
if (ir_channel_names_path.has_value()) {
XLS_ASSIGN_OR_RETURN(std::string ir_channel_names_text,
GetFileContents(*ir_channel_names_path));
ir_channel_names = ParseIrChannelNames(ir_channel_names_text);
}
} else if (testvector_path.has_value()) {
testvector::SampleInputsProto sample_inputs;
XLS_RETURN_IF_ERROR(ParseTextProtoFile(*testvector_path, &sample_inputs));
ArgsBatch extracted_args;
std::vector<std::string> extracted_channel_names;
XLS_RETURN_IF_ERROR(Sample::ExtractArgsBatch(/* is_proc_samples= */ true,
sample_inputs, extracted_args,
&extracted_channel_names));
args_batch = std::move(extracted_args);
ir_channel_names = std::move(extracted_channel_names);
}

std::string ir_channel_values_file_content;
Expand Down Expand Up @@ -1102,10 +1124,9 @@ absl::Status SampleRunner::RunProc(
// Unconditionally evaluate with the interpreter even if using the JIT. This
// exercises the interpreter and serves as a reference.
Stopwatch t;
XLS_ASSIGN_OR_RETURN(
results["evaluated unopt IR (interpreter)"],
EvaluateIrProc(ir_path, tick_count, ir_channel_values_path, false,
options, run_dir_, commands_));
XLS_ASSIGN_OR_RETURN(results["evaluated unopt IR (interpreter)"],
EvaluateIrProc(ir_path, tick_count, *testvector_path,
false, options, run_dir_, commands_));
if (!reference.has_value()) {
reference = results["evaluated unopt IR (interpreter)"];
}
Expand All @@ -1114,10 +1135,9 @@ absl::Status SampleRunner::RunProc(

if (options.use_jit()) {
t.Reset();
XLS_ASSIGN_OR_RETURN(
results["evaluated unopt IR (JIT)"],
EvaluateIrProc(ir_path, tick_count, ir_channel_values_path, true,
options, run_dir_, commands_));
XLS_ASSIGN_OR_RETURN(results["evaluated unopt IR (JIT)"],
EvaluateIrProc(ir_path, tick_count, *testvector_path,
true, options, run_dir_, commands_));
timing_.set_unoptimized_jit_ns(
absl::ToInt64Nanoseconds(t.GetElapsedTime()));
}
Expand All @@ -1135,17 +1155,17 @@ absl::Status SampleRunner::RunProc(
t.Reset();
XLS_ASSIGN_OR_RETURN(
results["evaluated opt IR (JIT)"],
EvaluateIrProc(*opt_ir_path, tick_count, ir_channel_values_path,
true, options, run_dir_, commands_));
EvaluateIrProc(*opt_ir_path, tick_count, *testvector_path, true,
options, run_dir_, commands_));
timing_.set_optimized_jit_ns(
absl::ToInt64Nanoseconds(t.GetElapsedTime()));
}

t.Reset();
XLS_ASSIGN_OR_RETURN(
results["evaluated opt IR (interpreter)"],
EvaluateIrProc(*opt_ir_path, tick_count, ir_channel_values_path,
false, options, run_dir_, commands_));
EvaluateIrProc(*opt_ir_path, tick_count, *testvector_path, false,
options, run_dir_, commands_));
timing_.set_optimized_interpret_ir_ns(
absl::ToInt64Nanoseconds(t.GetElapsedTime()));

Expand All @@ -1166,8 +1186,8 @@ absl::Status SampleRunner::RunProc(
XLS_ASSIGN_OR_RETURN(
results["simulated"],
SimulateProc(verilog_path, "module_sig.textproto",
ir_channel_values_path, output_channel_counts_str,
options, run_dir_, commands_));
*testvector_path, output_channel_counts_str, options,
run_dir_, commands_));
timing_.set_simulate_ns(absl::ToInt64Nanoseconds(t.GetElapsedTime()));
}
}
Expand Down
Loading

0 comments on commit 887923d

Please sign in to comment.