eval.py

from ranx import Qrels, Run, compare
import argparse
import glob

parser = argparse.ArgumentParser()
parser.add_argument('--path', type=str)
parser.add_argument('--qrel', type=str)
args = parser.parse_args()

run_files = [f for f in glob.glob(f"{args.path}/*.txt")]
run_files = sorted(run_files)
report_strings = []
qrel = Qrels.from_file(args.qrel)
runs=[]
for run_file in run_files:
    run = Run.from_file(run_file)
    run.name = run_file.split('/')[-1]
    runs.append(run)


report = compare(
    qrel,
    runs=runs,
    metrics=["ndcg@10", "precision@10-l2", "recall@1000-l2"],
    max_p=0.05,  # P-value threshold
    stat_test="student"
)
report.rounding_digits = 3
# report.show_percentages = True
table = report.to_table()
print(report.to_latex())
print(report.to_table())
report_strings.append(report.to_latex().split("\n"))
# print(report_strings)
# for i in range(24, 100, 3):
#     print(report_strings[0][i].replace("\\\\", '&'), report_strings[1][i].replace("\\\\", '&'), report_strings[2][i].replace("\\\\", '&'), report_strings[3][i])