add r1 reasoning effort

terryyz · terryyz · commit 57a4a3cbc417 · 2025-01-23T01:21:19.000+08:00
diff --git a/bigcodebench/generate.py b/bigcodebench/generate.py
@@ -189,7 +189,7 @@ def run_codegen(
     )
     
     extra = "-" + subset if subset != "full" else ""
-    if reasoning_effort and model.startswith("o1-") or model.startswith("o3-"):
+    if reasoning_effort and model.startswith("o1-") or model.startswith("o3-") or model.endswith("-reasoner"):
         model = model + f"--{reasoning_effort}"
     identifier = model.replace("/", "--") + f"--{revision}--bigcodebench{extra}-{split}--{backend}-{temperature}-{n_samples}-sanitized_calibrated.jsonl"
     

Original file line number	Diff line number	Diff line change
`@@ -189,7 +189,7 @@ def run_codegen(`
`189`	`189`	`)`
`190`	`190`
`191`	`191`	`extra = "-" + subset if subset != "full" else ""`
`192`		`- if reasoning_effort and model.startswith("o1-") or model.startswith("o3-"):`
	`192`	`+ if reasoning_effort and model.startswith("o1-") or model.startswith("o3-") or model.endswith("-reasoner"):`
`193`	`193`	`model = model + f"--{reasoning_effort}"`
`194`	`194`	`identifier = model.replace("/", "--") + f"--{revision}--bigcodebench{extra}-{split}--{backend}-{temperature}-{n_samples}-sanitized_calibrated.jsonl"`
`195`	`195`