fix(generate): update the identifier for o1 and o3

terryyz · terryyz · commit e5f27a984ac5 · 2024-12-21T17:12:00.000+08:00
diff --git a/bigcodebench/generate.py b/bigcodebench/generate.py
@@ -189,6 +189,8 @@ def run_codegen(
     )
     
     extra = "-" + subset if subset != "full" else ""
+    if reasoning_effort and model.startswith("o1-") or model.startswith("o3-"):
+        model = model + f"--{reasoning_effort}"
     identifier = model.replace("/", "--") + f"--{revision}--bigcodebench{extra}-{split}--{backend}-{temperature}-{n_samples}-sanitized_calibrated.jsonl"
     
     target_path = os.path.join(root, identifier)

Original file line number	Diff line number	Diff line change
`@@ -189,6 +189,8 @@ def run_codegen(`
`189`	`189`	`)`
`190`	`190`
`191`	`191`	`extra = "-" + subset if subset != "full" else ""`
	`192`	`+ if reasoning_effort and model.startswith("o1-") or model.startswith("o3-"):`
	`193`	`+ model = model + f"--{reasoning_effort}"`
`192`	`194`	`identifier = model.replace("/", "--") + f"--{revision}--bigcodebench{extra}-{split}--{backend}-{temperature}-{n_samples}-sanitized_calibrated.jsonl"`
`193`	`195`
`194`	`196`	`target_path = os.path.join(root, identifier)`