update

Author: Elfsong

bigcode_eval/evaluator.py

@@ -104,6 +104,7 @@ def evaluate(self, task_name, intermediate_generations=None):
             if self.allow_code_execution and task.requires_execution:
                 os.environ["HF_ALLOW_CODE_EVAL"] = "1"
             print("Evaluating generations...")
+            del self.model
             results = task.process_results(generations, references)
             return results
 

starcoder2-7b-mercury-result.json

@@ -0,0 +1,93 @@
+{
+  "mercury": {
+    "Easy_pass@1": 0.6522727272727272,
+    "Easy_pass@3": 0.7193181818181817,
+    "Easy_pass@5": 0.7272727272727273,
+    "Easy_beyond@1": 0.497185477591695,
+    "Easy_beyond@3": 0.4990392326594245,
+    "Easy_beyond@5": 0.48808251879601455,
+    "Medium_pass@1": 0.6271604938271604,
+    "Medium_pass@3": 0.7000000000000001,
+    "Medium_pass@5": 0.7160493827160493,
+    "Medium_beyond@1": 0.5068938378577104,
+    "Medium_beyond@3": 0.49677158926321824,
+    "Medium_beyond@5": 0.4976374449174768,
+    "Hard_pass@1": 0.28965517241379307,
+    "Hard_pass@3": 0.39425287356321836,
+    "Hard_pass@5": 0.4367816091954023,
+    "Hard_beyond@1": 0.1938273397041406,
+    "Hard_beyond@3": 0.20802385794546693,
+    "Hard_beyond@5": 0.2020810910984664,
+    "Average_pass@1": 0.5210937499999999,
+    "Average_pass@3": 0.602734375,
+    "Average_pass@5": 0.625,
+    "Average_beyond@1": 0.39716289628439005,
+    "Average_beyond@3": 0.39942197986564726,
+    "Average_beyond@5": 0.3939099594450451,
+    "Easy": {
+      "failed@load": 107,
+      "failed@eval": 14,
+      "failed@cases": 32,
+      "failed@timeout": 0,
+      "failed@error": 0,
+      "passed": 287
+    },
+    "Medium": {
+      "failed@load": 101,
+      "failed@eval": 5,
+      "failed@cases": 45,
+      "failed@timeout": 0,
+      "failed@error": 0,
+      "passed": 254
+    },
+    "Hard": {
+      "failed@load": 22,
+      "failed@eval": 77,
+      "failed@cases": 210,
+      "failed@timeout": 0,
+      "failed@error": 0,
+      "passed": 126
+    }
+  },
+  "config": {
+    "prefix": "",
+    "do_sample": true,
+    "temperature": 0.2,
+    "top_k": 0,
+    "top_p": 0.95,
+    "n_samples": 5,
+    "eos": "<|endoftext|>",
+    "seed": 0,
+    "model": "bigcode/starcoder2-7b",
+    "modeltype": "causal",
+    "peft_model": null,
+    "revision": null,
+    "use_auth_token": false,
+    "trust_remote_code": false,
+    "tasks": "mercury",
+    "instruction_tokens": null,
+    "batch_size": 5,
+    "max_length_generation": 2048,
+    "precision": "fp32",
+    "load_in_8bit": false,
+    "load_in_4bit": true,
+    "left_padding": false,
+    "limit": null,
+    "limit_start": 0,
+    "save_every_k_tasks": -1,
+    "postprocess": true,
+    "allow_code_execution": true,
+    "generation_only": false,
+    "load_generations_path": null,
+    "load_data_path": null,
+    "metric_output_path": "starcoder2-7b-mercury-result.json",
+    "save_generations": true,
+    "load_generations_intermediate_paths": null,
+    "save_generations_path": "generations.json",
+    "save_references": false,
+    "save_references_path": "references.json",
+    "prompt": "prompt",
+    "max_memory_per_gpu": null,
+    "check_references": false
+  }
+}