From ba3e8396c3e98478dd2ae30f1a8e5467ad83dcdc Mon Sep 17 00:00:00 2001 From: Muennighoff Date: Sat, 30 Jul 2022 14:14:52 +0200 Subject: [PATCH] Add xlwic --- promptsource/templates.py | 17 +- .../templates/pasinit/xlwic/en/templates.yaml | 238 ++++++++++++++++++ 2 files changed, 254 insertions(+), 1 deletion(-) create mode 100644 promptsource/templates/pasinit/xlwic/en/templates.yaml diff --git a/promptsource/templates.py b/promptsource/templates.py index 2b9eb341e..429c08690 100644 --- a/promptsource/templates.py +++ b/promptsource/templates.py @@ -27,7 +27,22 @@ # These are users whose datasets should be included in the results returned by # filter_english_datasets (regardless of their metadata) -INCLUDED_USERS = {"Zaid", "craffel", "GEM", "aps", "khalidalt", "shanya", "rbawden", "BigScienceBiasEval", "gsarti"} +INCLUDED_USERS = { + "Zaid", + "craffel", + "GEM", + "aps", + "khalidalt", + "shanya", + "rbawden", + "BigScienceBiasEval", + "gsarti", + "Helsinki-NLP", + "Muennighoff", + "facebook", + "codeparrot", + "pasinit", +} # These are the metrics with which templates can be tagged METRICS = { diff --git a/promptsource/templates/pasinit/xlwic/en/templates.yaml b/promptsource/templates/pasinit/xlwic/en/templates.yaml new file mode 100644 index 000000000..c52528482 --- /dev/null +++ b/promptsource/templates/pasinit/xlwic/en/templates.yaml @@ -0,0 +1,238 @@ +dataset: pasinit/xlwic +subset: en +templates: + 24e73f39-a0d1-44c2-b9a4-4e48f9f1608e: !Template + answer_choices: No ||| Yes + id: 24e73f39-a0d1-44c2-b9a4-4e48f9f1608e + jinja: 'Does the word "{{target_word}}" have the same meaning in these two sentences? + Yes, No? + + {{context_1}} + + {{context_2}} + + ||| {% if label != -1%} + + {{answer_choices[label]}} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: question-context-meaning-with-label + reference: Generalized question-context format with label + 4503ead5-4fa5-4f77-95dc-f0c2ed3eecdc: !Template + answer_choices: No ||| Yes + id: 4503ead5-4fa5-4f77-95dc-f0c2ed3eecdc + jinja: 'Does the word "{{target_word}}" have the same meaning in these two sentences? + + {{context_1}} + + {{context_2}} + + ||| {% if label != -1%} + + {{answer_choices[label]}} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: question-context-meaning + reference: Generalized question-context format + 711d13dc-d414-4b9b-9204-e4f325e859e7: !Template + answer_choices: No ||| Yes + id: 711d13dc-d414-4b9b-9204-e4f325e859e7 + jinja: 'Homework + + + Decide whether the word "{{target_word}}" is used with the same meaning in the two + following sentences. Answer by yes or no. + + {{context_1}} + + {{context_2}} + + ||| {% if label != -1%} + + {{answer_choices[label]}} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: grammar_homework + reference: '' + 925b5ed0-7728-4890-95a4-a74cb7ae1bb4: !Template + answer_choices: False ||| True + id: 925b5ed0-7728-4890-95a4-a74cb7ae1bb4 + jinja: 'Sentence A: {{context_1}} + + Sentence B: {{context_2}} + + + "{{target_word}}" has a similar meaning in sentences A and B. True or False? + + ||| {% if label != -1%} + + {{answer_choices[label]}} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: affirmation_true_or_false + reference: '' + d3a0a5d8-cfe9-4a7f-8a3c-3c526e0ad0c6: !Template + answer_choices: No ||| Yes + id: d3a0a5d8-cfe9-4a7f-8a3c-3c526e0ad0c6 + jinja: '{{context_1}} + + {{context_2}} + + Question: Is the word ''{{target_word}}'' used in the same sense in the two sentences + above? + + ||| {% if label != -1%} + + {{answer_choices[label]}} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: GPT-3-prompt + reference: Following table G32. https://arxiv.org/pdf/2005.14165.pdf + de8b5a93-1841-4897-84db-b100f1c84f4b: !Template + answer_choices: No ||| Yes + id: de8b5a93-1841-4897-84db-b100f1c84f4b + jinja: 'Sentence 1: {{context_1}} + + Sentence 2: {{context_2}} + + + Determine whether the word "{{target_word}}" is used in the same sense in both sentences. + Yes or no? + + ||| {% if label != -1%} + + {{answer_choices[label]}} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: same_sense + reference: '' + dfbc1637-10b8-4f20-a31c-55292f3cebd0: !Template + answer_choices: No ||| Yes + id: dfbc1637-10b8-4f20-a31c-55292f3cebd0 + jinja: "Determine if the word '{{target_word}}' is used in the same way in the two sentences\ + \ below. \n{{context_1}}\n{{context_2}}\n||| {% if label != -1%}\n{{answer_choices[label]}}\n\ + {% endif %}" + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: question-context + reference: Generalized question-context format + e9e1db2a-ab0b-4621-bb41-01d5788d3873: !Template + answer_choices: No ||| Yes + id: e9e1db2a-ab0b-4621-bb41-01d5788d3873 + jinja: '{{context_1}} + + {{context_2}} + + Question: Is the word ''{{target_word}}'' used in the same sense in the two sentences + above? Yes, No? + + ||| {% if label != -1%} + + {{answer_choices[label]}} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: GPT-3-prompt-with-label + reference: Following table G32. https://arxiv.org/pdf/2005.14165.pdf add additional + label + ed2080cf-3117-49ba-9aff-c988a21fdb69: !Template + answer_choices: No ||| Yes + id: ed2080cf-3117-49ba-9aff-c988a21fdb69 + jinja: 'The word "{{target_word}}" has multiple meanings. Does it have the same meaning + in sentences 1 and 2? Yes or no? + + + Sentence 1: {{context_1}} + + Sentence 2: {{context_2}} + + ||| {% if label != -1%} + + {{answer_choices[label]}} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: polysemous + reference: '' + g934a96d-fe4d-4075-aa47-5595b9a604c7: !Template + answer_choices: No ||| Yes + id: g934a96d-fe4d-4075-aa47-5595b9a604c7 + jinja: '{{context_1}} + + {{context_2}} + + Similar sense of {{target_word}}? + + ||| {% if label != -1%} + + {{answer_choices[label]}} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: similar-sense + reference: Following https://arxiv.org/abs/2105.11447, https://github.com/ethanjperez/true_few_shot/tree/main/templates.super_glue