From 925984e124567dab3d78dc18b6c50c3773b2782c Mon Sep 17 00:00:00 2001 From: thorunna Date: Tue, 20 Aug 2024 14:51:20 +0000 Subject: [PATCH] Added Icelandic QA evaluation data from Wikipedia --- evals/registry/data/icelandic-qa-wiki/samples.jsonl | 3 +++ evals/registry/evals/icelandic-qa-wiki.yaml | 9 +++++++++ 2 files changed, 12 insertions(+) create mode 100644 evals/registry/data/icelandic-qa-wiki/samples.jsonl create mode 100644 evals/registry/evals/icelandic-qa-wiki.yaml diff --git a/evals/registry/data/icelandic-qa-wiki/samples.jsonl b/evals/registry/data/icelandic-qa-wiki/samples.jsonl new file mode 100644 index 0000000000..996057df4e --- /dev/null +++ b/evals/registry/data/icelandic-qa-wiki/samples.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e91ae91f162e33782627e78d316b34b4ae71ed672f73e91e8f5879bd38150102 +size 532374 diff --git a/evals/registry/evals/icelandic-qa-wiki.yaml b/evals/registry/evals/icelandic-qa-wiki.yaml new file mode 100644 index 0000000000..9247039c8d --- /dev/null +++ b/evals/registry/evals/icelandic-qa-wiki.yaml @@ -0,0 +1,9 @@ +icelandic-qa-wiki: + id: icelandic-qa-wiki.dev.v0 + metrics: [accuracy] + description: Test the model's ability to answer Icelandic questions on Icelandic culture and history, created from the Icelandic Wikipedia. + +icelandic-qa-wiki.dev.v0: + class: evals.elsuite.basic.fuzzy_match:FuzzyMatch + args: + samples_jsonl: icelandic-qa-wiki/samples.jsonl \ No newline at end of file