diff --git a/promptsource/templates.py b/promptsource/templates.py index 2b9eb341e..789e91cc8 100644 --- a/promptsource/templates.py +++ b/promptsource/templates.py @@ -27,7 +27,18 @@ # These are users whose datasets should be included in the results returned by # filter_english_datasets (regardless of their metadata) -INCLUDED_USERS = {"Zaid", "craffel", "GEM", "aps", "khalidalt", "shanya", "rbawden", "BigScienceBiasEval", "gsarti"} +INCLUDED_USERS = { + "Zaid", + "craffel", + "GEM", + "aps", + "khalidalt", + "shanya", + "rbawden", + "BigScienceBiasEval", + "gsarti", + "Muennighoff", +} # These are the metrics with which templates can be tagged METRICS = { diff --git a/promptsource/templates/Muennighoff/xwinograd/en/templates.yaml b/promptsource/templates/Muennighoff/xwinograd/en/templates.yaml new file mode 100644 index 000000000..f0f34dccb --- /dev/null +++ b/promptsource/templates/Muennighoff/xwinograd/en/templates.yaml @@ -0,0 +1,110 @@ +dataset: Muennighoff/xwinograd +subset: en +templates: + 28d31908-4aee-4545-aff2-7528cbf39197: !Template + answer_choices: '{{option1}} ||| {{option2}}' + id: 28d31908-4aee-4545-aff2-7528cbf39197 + jinja: "{{sentence}}\nReplace the _ in the above sentence with the correct option:\ + \ \n- {{option1}}\n- {{option2}}\n|||\n{% if answer == '1' %} {{option1}} {%\ + \ else %} {{ option2 }} {% endif %}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: Replace + reference: '' + 50ce5113-882f-4a9d-b21d-8d98b4644295: !Template + answer_choices: '{{option1}} ||| {{option2}}' + id: 50ce5113-882f-4a9d-b21d-8d98b4644295 + jinja: 'Fill in the _ in the below sentence: + + {{sentence}} + + + Choices: + + - {{ option1 }} + + - {{ option2 }} + + + Answer: ||| {% if answer == ''1'' %} {{option1}} {% else %} {{ option2 }} {% + endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: fill in the blank + reference: '' + 7f0f6d33-25e2-4394-b1f0-49a2a54767aa: !Template + answer_choices: True ||| False + id: 7f0f6d33-25e2-4394-b1f0-49a2a54767aa + jinja: 'The _ in the sentence below refers to {{option1}}. True or False? + + {{sentence}}||| + + {{answer_choices[answer|int - 1]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: false + name: True or False + reference: '' + 80f9679e-7b6c-4ee7-a348-e905ed9aaf9e: !Template + answer_choices: '{{ option1 }} ||| {{ option2 }}' + id: 80f9679e-7b6c-4ee7-a348-e905ed9aaf9e + jinja: '{{ sentence }} In the previous sentence, does _ refer to {{ option1 }} + or {{ option2 }}? ||| {% if answer == ''1'' %} {{option1}} {% else %} {{ option2 + }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: does underscore refer to + reference: '' + bd40cf1f-bda2-4757-b1b5-f1a20a3f7202: !Template + answer_choices: '{{option1}} ||| {{option2}}' + id: bd40cf1f-bda2-4757-b1b5-f1a20a3f7202 + jinja: '{{sentence}} + + What does the _ in the above sentence refer to? {{ option1 }} or {{ option2 + }}? ||| {% if answer == ''1'' %} {{option1}} {% else %} {{ option2 }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: underscore refer to + reference: '' + ec365d5d-bb5c-488c-93a0-4f90e6011c5d: !Template + answer_choices: '{{option1}} ||| {{option2}}' + id: ec365d5d-bb5c-488c-93a0-4f90e6011c5d + jinja: 'In the sentence below, does the _ stand for {{answer_choices[0]}} or {{answer_choices[1]}}? + + {{sentence}}||| + + {{answer_choices[answer | int - 1]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: stand for + reference: '' diff --git a/promptsource/templates/Muennighoff/xwinograd/fr/templates.yaml b/promptsource/templates/Muennighoff/xwinograd/fr/templates.yaml new file mode 100644 index 000000000..5c038558c --- /dev/null +++ b/promptsource/templates/Muennighoff/xwinograd/fr/templates.yaml @@ -0,0 +1,110 @@ +dataset: Muennighoff/xwinograd +subset: en +templates: + 38d31908-4aee-4545-aff2-7528cbf39197: !Template + answer_choices: '{{option1}} ||| {{option2}}' + id: 38d31908-4aee-4545-aff2-7528cbf39197 + jinja: "{{sentence}}\nReplace the _ in the above sentence with the correct option:\ + \ \n- {{option1}}\n- {{option2}}\n|||\n{% if answer == '1' %} {{option1}} {%\ + \ else %} {{ option2 }} {% endif %}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: Replace + reference: '' + 60ce5113-882f-4a9d-b21d-8d98b4644295: !Template + answer_choices: '{{option1}} ||| {{option2}}' + id: 60ce5113-882f-4a9d-b21d-8d98b4644295 + jinja: 'Fill in the _ in the below sentence: + + {{sentence}} + + + Choices: + + - {{ option1 }} + + - {{ option2 }} + + + Answer: ||| {% if answer == ''1'' %} {{option1}} {% else %} {{ option2 }} {% + endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: fill in the blank + reference: '' + 8f0f6d33-25e2-4394-b1f0-49a2a54767aa: !Template + answer_choices: True ||| False + id: 8f0f6d33-25e2-4394-b1f0-49a2a54767aa + jinja: 'The _ in the sentence below refers to {{option1}}. True or False? + + {{sentence}}||| + + {{answer_choices[answer|int - 1]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: false + name: True or False + reference: '' + 90f9679e-7b6c-4ee7-a348-e905ed9aaf9e: !Template + answer_choices: '{{ option1 }} ||| {{ option2 }}' + id: 90f9679e-7b6c-4ee7-a348-e905ed9aaf9e + jinja: '{{ sentence }} In the previous sentence, does _ refer to {{ option1 }} + or {{ option2 }}? ||| {% if answer == ''1'' %} {{option1}} {% else %} {{ option2 + }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: does underscore refer to + reference: '' + cd40cf1f-bda2-4757-b1b5-f1a20a3f7202: !Template + answer_choices: '{{option1}} ||| {{option2}}' + id: cd40cf1f-bda2-4757-b1b5-f1a20a3f7202 + jinja: '{{sentence}} + + What does the _ in the above sentence refer to? {{ option1 }} or {{ option2 + }}? ||| {% if answer == ''1'' %} {{option1}} {% else %} {{ option2 }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: underscore refer to + reference: '' + fc365d5d-bb5c-488c-93a0-4f90e6011c5d: !Template + answer_choices: '{{option1}} ||| {{option2}}' + id: fc365d5d-bb5c-488c-93a0-4f90e6011c5d + jinja: 'In the sentence below, does the _ stand for {{answer_choices[0]}} or {{answer_choices[1]}}? + + {{sentence}}||| + + {{answer_choices[answer | int - 1]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: stand for + reference: '' diff --git a/promptsource/templates/Muennighoff/xwinograd/pt/templates.yaml b/promptsource/templates/Muennighoff/xwinograd/pt/templates.yaml new file mode 100644 index 000000000..5c038558c --- /dev/null +++ b/promptsource/templates/Muennighoff/xwinograd/pt/templates.yaml @@ -0,0 +1,110 @@ +dataset: Muennighoff/xwinograd +subset: en +templates: + 38d31908-4aee-4545-aff2-7528cbf39197: !Template + answer_choices: '{{option1}} ||| {{option2}}' + id: 38d31908-4aee-4545-aff2-7528cbf39197 + jinja: "{{sentence}}\nReplace the _ in the above sentence with the correct option:\ + \ \n- {{option1}}\n- {{option2}}\n|||\n{% if answer == '1' %} {{option1}} {%\ + \ else %} {{ option2 }} {% endif %}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: Replace + reference: '' + 60ce5113-882f-4a9d-b21d-8d98b4644295: !Template + answer_choices: '{{option1}} ||| {{option2}}' + id: 60ce5113-882f-4a9d-b21d-8d98b4644295 + jinja: 'Fill in the _ in the below sentence: + + {{sentence}} + + + Choices: + + - {{ option1 }} + + - {{ option2 }} + + + Answer: ||| {% if answer == ''1'' %} {{option1}} {% else %} {{ option2 }} {% + endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: fill in the blank + reference: '' + 8f0f6d33-25e2-4394-b1f0-49a2a54767aa: !Template + answer_choices: True ||| False + id: 8f0f6d33-25e2-4394-b1f0-49a2a54767aa + jinja: 'The _ in the sentence below refers to {{option1}}. True or False? + + {{sentence}}||| + + {{answer_choices[answer|int - 1]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: false + name: True or False + reference: '' + 90f9679e-7b6c-4ee7-a348-e905ed9aaf9e: !Template + answer_choices: '{{ option1 }} ||| {{ option2 }}' + id: 90f9679e-7b6c-4ee7-a348-e905ed9aaf9e + jinja: '{{ sentence }} In the previous sentence, does _ refer to {{ option1 }} + or {{ option2 }}? ||| {% if answer == ''1'' %} {{option1}} {% else %} {{ option2 + }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: does underscore refer to + reference: '' + cd40cf1f-bda2-4757-b1b5-f1a20a3f7202: !Template + answer_choices: '{{option1}} ||| {{option2}}' + id: cd40cf1f-bda2-4757-b1b5-f1a20a3f7202 + jinja: '{{sentence}} + + What does the _ in the above sentence refer to? {{ option1 }} or {{ option2 + }}? ||| {% if answer == ''1'' %} {{option1}} {% else %} {{ option2 }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: underscore refer to + reference: '' + fc365d5d-bb5c-488c-93a0-4f90e6011c5d: !Template + answer_choices: '{{option1}} ||| {{option2}}' + id: fc365d5d-bb5c-488c-93a0-4f90e6011c5d + jinja: 'In the sentence below, does the _ stand for {{answer_choices[0]}} or {{answer_choices[1]}}? + + {{sentence}}||| + + {{answer_choices[answer | int - 1]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: stand for + reference: '' diff --git a/promptsource/templates/Muennighoff/xwinograd/zh/templates.yaml b/promptsource/templates/Muennighoff/xwinograd/zh/templates.yaml new file mode 100644 index 000000000..c911d8cb2 --- /dev/null +++ b/promptsource/templates/Muennighoff/xwinograd/zh/templates.yaml @@ -0,0 +1,110 @@ +dataset: Muennighoff/xwinograd +subset: zh +templates: + 2e10b0b9-b57e-4bd0-b8cc-cbdd5dcd7551: !Template + answer_choices: '{{option1}} ||| {{option2}}' + id: 2e10b0b9-b57e-4bd0-b8cc-cbdd5dcd7551 + jinja: "{{sentence}}\nReplace the _ in the above sentence with the correct option:\ + \ \n- {{option1}}\n- {{option2}}\n|||\n{% if answer == '1' %} {{option1}} {%\ + \ else %} {{ option2 }} {% endif %}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: Replace + reference: '' + 33f50367-0615-45a0-883a-05b6ff8c0e2f: !Template + answer_choices: '{{option1}} ||| {{option2}}' + id: 33f50367-0615-45a0-883a-05b6ff8c0e2f + jinja: 'Fill in the _ in the below sentence: + + {{sentence}} + + + Choices: + + - {{ option1 }} + + - {{ option2 }} + + + Answer: ||| {% if answer == ''1'' %} {{option1}} {% else %} {{ option2 }} {% + endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: fill in the blank + reference: '' + 37767236-f6e6-4a57-9deb-c0a4c073e56d: !Template + answer_choices: True ||| False + id: 37767236-f6e6-4a57-9deb-c0a4c073e56d + jinja: 'The _ in the sentence below refers to {{option1}}. True or False? + + {{sentence}}||| + + {{answer_choices[answer|int - 1]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: false + name: True or False + reference: '' + 63f2ab14-d78b-4e83-a3f9-186cb6f220c0: !Template + answer_choices: '{{ option1 }} ||| {{ option2 }}' + id: 63f2ab14-d78b-4e83-a3f9-186cb6f220c0 + jinja: '{{ sentence }} In the previous sentence, does _ refer to {{ option1 }} + or {{ option2 }}? ||| {% if answer == ''1'' %} {{option1}} {% else %} {{ option2 + }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: does underscore refer to + reference: '' + aec099a7-da35-4438-aa7c-250651959048: !Template + answer_choices: '{{option1}} ||| {{option2}}' + id: aec099a7-da35-4438-aa7c-250651959048 + jinja: '{{sentence}} + + What does the _ in the above sentence refer to? {{ option1 }} or {{ option2 + }}? ||| {% if answer == ''1'' %} {{option1}} {% else %} {{ option2 }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: underscore refer to + reference: '' + fa9d56aa-b236-44f9-92c1-595784f57243: !Template + answer_choices: '{{option1}} ||| {{option2}}' + id: fa9d56aa-b236-44f9-92c1-595784f57243 + jinja: 'In the sentence below, does the _ stand for {{answer_choices[0]}} or {{answer_choices[1]}}? + + {{sentence}}||| + + {{answer_choices[answer | int - 1]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: stand for + reference: ''