Skip to content

Commit f906000

Browse files
MuennighoffMuennighoff
authored
Merge pull request #1 from haileyschoelkopf/xcopa
Xcopa
2 parents fa8e6e5 + 670e6ad commit f906000

File tree

6 files changed

+1268
-0
lines changed

6 files changed

+1268
-0
lines changed

‎promptsource/templates/xcopa/id/templates.yaml

Lines changed: 211 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,211 @@
1+
dataset: xcopa
2+
subset: id
3+
templates:
4+
1a87b487-1570-4873-aed9-b84d2fc0476c: !Template
5+
answer_choices: '{{choice1}} ||| {{choice2}}'
6+
id: 1a87b487-1570-4873-aed9-b84d2fc0476c
7+
jinja: "{{ premise }} \n\nI am hesitating between two options. Help me choose\
8+
\ the more likely {% if question == \"cause\" %} cause: {% else %} effect: {%\
9+
\ endif %}\n- {{choice1}}\n- {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label]\
10+
\ }}{%endif%}"
11+
metadata: !TemplateMetadata
12+
choices_in_prompt: true
13+
languages:
14+
- en
15+
metrics:
16+
- Accuracy
17+
original_task: true
18+
name: i_am_hesitating
19+
reference: ''
20+
336c4c72-40e3-4122-881e-8cd7a1881eec: !Template
21+
answer_choices: '{{choice1}} ||| {{choice2}}'
22+
id: 336c4c72-40e3-4122-881e-8cd7a1881eec
23+
jinja: "{% if question == \"cause\" %} \n{{ premise }} Why? \"{{ answer_choices[0]\
24+
\ }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label != -1 %}{{ answer_choices[label]\
25+
\ }}{%endif%}\n{% endif %}"
26+
metadata: !TemplateMetadata
27+
choices_in_prompt: true
28+
languages:
29+
- en
30+
metrics:
31+
- Accuracy
32+
original_task: true
33+
name: "\u2026why? C1 or C2"
34+
reference: ''
35+
482f0b87-e748-4e98-8cc8-a23386bc50c3: !Template
36+
answer_choices: '{{choice1}} ||| {{choice2}}'
37+
id: 482f0b87-e748-4e98-8cc8-a23386bc50c3
38+
jinja: "{{ premise }} \n\nWhat's the best option?\n- {{choice1}}\n- {{choice2}}\n\
39+
\nWe are looking for {% if question == \"cause\" %} a cause {% else %} an effect\
40+
\ {% endif %}\n||| {% if label != -1 %}{{answer_choices[label]}}{%endif%}"
41+
metadata: !TemplateMetadata
42+
choices_in_prompt: true
43+
languages:
44+
- en
45+
metrics:
46+
- Accuracy
47+
original_task: true
48+
name: best_option
49+
reference: ''
50+
4a0640a5-c378-422d-879b-7490bc500c8a: !Template
51+
answer_choices: '{{choice1}} ||| {{choice2}}'
52+
id: 4a0640a5-c378-422d-879b-7490bc500c8a
53+
jinja: '{{ premise }} {% if question == "cause" %} because... {% else %} so...
54+
{% endif %}
55+
56+
Choose between:
57+
58+
- {{choice1}}
59+
60+
- {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}'
61+
metadata: !TemplateMetadata
62+
choices_in_prompt: true
63+
languages:
64+
- en
65+
metrics:
66+
- Accuracy
67+
original_task: true
68+
name: choose
69+
reference: ''
70+
78e28a66-a84c-442c-9bf7-44aa49450412: !Template
71+
answer_choices: '{{choice1}} ||| {{choice2}}'
72+
id: 78e28a66-a84c-442c-9bf7-44aa49450412
73+
jinja: '{{ premise }} {% if question == "cause" %} This happened because... {%
74+
else %} As a consequence... {% endif %}
75+
76+
Help me pick the more plausible option:
77+
78+
- {{choice1}}
79+
80+
- {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}'
81+
metadata: !TemplateMetadata
82+
choices_in_prompt: true
83+
languages:
84+
- en
85+
metrics:
86+
- Accuracy
87+
original_task: true
88+
name: plausible_alternatives
89+
reference: ''
90+
7c0b578c-214f-4dc9-a9b4-252d91691cb0: !Template
91+
answer_choices: '{{choice1}} ||| {{choice2}}'
92+
id: 7c0b578c-214f-4dc9-a9b4-252d91691cb0
93+
jinja: "{% if question == \"effect\" %} \n{{ premise }} As a result, \"{{ answer_choices[0]\
94+
\ }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label != -1 %}{{ answer_choices[label]\
95+
\ }}{%endif%}\n{% endif %}"
96+
metadata: !TemplateMetadata
97+
choices_in_prompt: true
98+
languages:
99+
- en
100+
metrics:
101+
- Accuracy
102+
original_task: true
103+
name: "\u2026As a result, C1 or C2?"
104+
reference: ''
105+
94b5be71-c989-4a62-96d9-a7cb042e83c7: !Template
106+
answer_choices: '{{choice1}} ||| {{choice2}}'
107+
id: 94b5be71-c989-4a62-96d9-a7cb042e83c7
108+
jinja: 'Exercise: choose the most plausible alternative.
109+
110+
111+
{{ premise }} {% if question == "cause" %} because... {% else %} so... {% endif
112+
%}
113+
114+
- {{choice1}}
115+
116+
- {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}'
117+
metadata: !TemplateMetadata
118+
choices_in_prompt: true
119+
languages:
120+
- en
121+
metrics:
122+
- Accuracy
123+
original_task: true
124+
name: exercise
125+
reference: ''
126+
b308f6ce-673c-44c1-b84d-95a3045229ea: !Template
127+
answer_choices: '{{choice1}} ||| {{choice2}}'
128+
id: b308f6ce-673c-44c1-b84d-95a3045229ea
129+
jinja: '"{{ answer_choices[0] }}" or "{{ answer_choices[1] }}"? {{ premise }}
130+
{% if question == "cause" %} because {% else %} so {% endif %} ||| {% if label
131+
!= -1 %}{{ answer_choices[label] }}{% endif %}'
132+
metadata: !TemplateMetadata
133+
choices_in_prompt: true
134+
languages:
135+
- en
136+
metrics:
137+
- Accuracy
138+
original_task: true
139+
name: "C1 or C2? premise, so/because\u2026"
140+
reference: "Adapted from Perez et al. 2021 and Schick & Sch\xFCtz 2021."
141+
cf78cf75-90cc-4fe2-8b78-2bf64c9520b4: !Template
142+
answer_choices: '{{choice1}} ||| {{choice2}}'
143+
id: cf78cf75-90cc-4fe2-8b78-2bf64c9520b4
144+
jinja: '{{ premise }}
145+
146+
147+
Select the most plausible {% if question == "cause" %} cause: {% else %} effect:
148+
{% endif %}
149+
150+
- {{choice1}}
151+
152+
- {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}'
153+
metadata: !TemplateMetadata
154+
choices_in_prompt: true
155+
languages:
156+
- en
157+
metrics:
158+
- Accuracy
159+
original_task: true
160+
name: cause_effect
161+
reference: ''
162+
d8263afb-215f-43c4-83b8-c85744144fdb: !Template
163+
answer_choices: '{{choice1}} ||| {{choice2}}'
164+
id: d8263afb-215f-43c4-83b8-c85744144fdb
165+
jinja: "{% if question == \"cause\" %} \n{{ premise }} Which may be caused by\
166+
\ \"{{ answer_choices[0] }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label\
167+
\ != -1 %}{{ answer_choices[label] }}{%endif%}\n{% endif %}"
168+
metadata: !TemplateMetadata
169+
choices_in_prompt: true
170+
languages:
171+
- en
172+
metrics:
173+
- Accuracy
174+
original_task: true
175+
name: "\u2026which may be caused by"
176+
reference: ''
177+
eaddf2e0-ead4-456b-8e81-00bdcde8c7b0: !Template
178+
answer_choices: '{{choice1}} ||| {{choice2}}'
179+
id: eaddf2e0-ead4-456b-8e81-00bdcde8c7b0
180+
jinja: "{% if question == \"effect\" %} \n{{ premise }} What could happen next,\
181+
\ \"{{ answer_choices[0] }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label\
182+
\ != -1 %}{{ answer_choices[label] }}{%endif%}\n{% endif %}"
183+
metadata: !TemplateMetadata
184+
choices_in_prompt: true
185+
languages:
186+
- en
187+
metrics:
188+
- Accuracy
189+
original_task: true
190+
name: "\u2026What could happen next, C1 or C2?"
191+
reference: ''
192+
ebd4242a-14f2-4aed-a183-dc37a18dfe4b: !Template
193+
answer_choices: '{{choice1}} ||| {{choice2}}'
194+
id: ebd4242a-14f2-4aed-a183-dc37a18dfe4b
195+
jinja: 'Pick the more likely continuation to the following sentence:
196+
197+
{{ premise }} {% if question == "cause" %} as a result of: {% else %} as a consequence:
198+
{% endif %}
199+
200+
- {{choice1}}
201+
202+
- {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}'
203+
metadata: !TemplateMetadata
204+
choices_in_prompt: true
205+
languages:
206+
- en
207+
metrics:
208+
- Accuracy
209+
original_task: true
210+
name: more likely
211+
reference: ''

0 commit comments

Comments
 (0)