diff --git a/promptsource/templates.py b/promptsource/templates.py index 2b9eb341e..2c9785c40 100644 --- a/promptsource/templates.py +++ b/promptsource/templates.py @@ -27,7 +27,20 @@ # These are users whose datasets should be included in the results returned by # filter_english_datasets (regardless of their metadata) -INCLUDED_USERS = {"Zaid", "craffel", "GEM", "aps", "khalidalt", "shanya", "rbawden", "BigScienceBiasEval", "gsarti"} +INCLUDED_USERS = { + "Zaid", + "craffel", + "GEM", + "aps", + "khalidalt", + "shanya", + "rbawden", + "BigScienceBiasEval", + "gsarti", + "Helsinki-NLP", + "Muennighoff", + "facebook", +} # These are the metrics with which templates can be tagged METRICS = { diff --git a/promptsource/templates/Muennighoff/mbpp/sanitized/templates.yaml b/promptsource/templates/Muennighoff/mbpp/sanitized/templates.yaml new file mode 100644 index 000000000..3e912de08 --- /dev/null +++ b/promptsource/templates/Muennighoff/mbpp/sanitized/templates.yaml @@ -0,0 +1,32 @@ +dataset: Muennighoff/mbpp +subset: sanitized +templates: + 4b108b1c-7514-488f-99ed-3ca5da70e103: !Template + answer_choices: null + id: 4b108b1c-7514-488f-99ed-3ca5da70e103 + jinja: '{{ prompt }} + Here is a solution in Python: + ||| + {{ code }}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Other + original_task: true + name: function solution + reference: '' + 9d85c898-70fe-4a51-be37-5111be357762: !Template + answer_choices: null + id: 9d85c898-70fe-4a51-be37-5111be357762 + jinja: "{{ prompt }} This can be solved in Python with the following code: |||{{ code }}" + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Other + original_task: false + name: function solved + reference: '' diff --git a/promptsource/templates/super_glue/copa/templates.yaml b/promptsource/templates/super_glue/copa/templates.yaml index 9e9c0a30e..20e6c7122 100644 --- a/promptsource/templates/super_glue/copa/templates.yaml +++ b/promptsource/templates/super_glue/copa/templates.yaml @@ -22,21 +22,6 @@ templates: original_task: true name: exercise reference: '' - 150789fe-e309-47a1-82c9-0a4dc2c6b12b: !Template - answer_choices: '{{choice1}} ||| {{choice2}}' - id: 150789fe-e309-47a1-82c9-0a4dc2c6b12b - jinja: "{% if question == \"effect\" %} \n{{ premise }} What could happen next,\ - \ \"{{ answer_choices[0] }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label\ - \ != -1 %}{{ answer_choices[label] }}{%endif%}\n{% endif %}" - metadata: !TemplateMetadata - choices_in_prompt: true - languages: - - en - metrics: - - Accuracy - original_task: true - name: "\u2026What could happen next, C1 or C2?" - reference: '' 4d879cbe-2fd7-424a-9d78-3f5200313fba: !Template answer_choices: '{{choice1}} ||| {{choice2}}' id: 4d879cbe-2fd7-424a-9d78-3f5200313fba @@ -88,21 +73,6 @@ templates: original_task: true name: "C1 or C2? premise, so/because\u2026" reference: "Adapted from Perez et al. 2021 and Schick & Sch\xFCtz 2021." - 84da62c2-9440-4cfc-bdd4-d70c65e33a82: !Template - answer_choices: '{{choice1}} ||| {{choice2}}' - id: 84da62c2-9440-4cfc-bdd4-d70c65e33a82 - jinja: "{% if question == \"effect\" %} \n{{ premise }} As a result, \"{{ answer_choices[0]\ - \ }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label != -1 %}{{ answer_choices[label]\ - \ }}{%endif%}\n{% endif %}" - metadata: !TemplateMetadata - choices_in_prompt: true - languages: - - en - metrics: - - Accuracy - original_task: true - name: "\u2026As a result, C1 or C2?" - reference: '' 8ce80f8a-239e-4393-892c-f63dbb0d9929: !Template answer_choices: '{{choice1}} ||| {{choice2}}' id: 8ce80f8a-239e-4393-892c-f63dbb0d9929 @@ -118,21 +88,6 @@ templates: original_task: true name: best_option reference: '' - 8cf2ba73-aee5-4651-b5d4-b1b88afe4abb: !Template - answer_choices: '{{choice1}} ||| {{choice2}}' - id: 8cf2ba73-aee5-4651-b5d4-b1b88afe4abb - jinja: "{% if question == \"cause\" %} \n{{ premise }} Which may be caused by\ - \ \"{{ answer_choices[0] }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label\ - \ != -1 %}{{ answer_choices[label] }}{%endif%}\n{% endif %}" - metadata: !TemplateMetadata - choices_in_prompt: true - languages: - - en - metrics: - - Accuracy - original_task: true - name: "\u2026which may be caused by" - reference: '' a1f9951e-2b6b-4530-9636-9cdf4c1658c5: !Template answer_choices: '{{choice1}} ||| {{choice2}}' id: a1f9951e-2b6b-4530-9636-9cdf4c1658c5 @@ -174,21 +129,6 @@ templates: original_task: true name: cause_effect reference: '' - a8bf11c3-bea2-45ba-a533-957d8bee5e2e: !Template - answer_choices: '{{choice1}} ||| {{choice2}}' - id: a8bf11c3-bea2-45ba-a533-957d8bee5e2e - jinja: "{% if question == \"cause\" %} \n{{ premise }} Why? \"{{ answer_choices[0]\ - \ }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label != -1 %}{{ answer_choices[label]\ - \ }}{%endif%}\n{% endif %}" - metadata: !TemplateMetadata - choices_in_prompt: true - languages: - - en - metrics: - - Accuracy - original_task: true - name: "\u2026why? C1 or C2" - reference: '' f32348cd-d3cb-4619-87b9-e24f99c78567: !Template answer_choices: '{{choice1}} ||| {{choice2}}' id: f32348cd-d3cb-4619-87b9-e24f99c78567