Merge pull request #143 from iCSawyer/main

Add --save_references_path into args

Author: loubnabnl

bigcode_eval/evaluator.py

@@ -84,9 +84,9 @@ def evaluate(self, task_name):
                             f"generations were saved at {self.args.save_generations_path}"
                         )
                 if self.args.save_references:
-                    with open("references.json", "w") as fp:
+                    with open(self.args.save_references_path, "w") as fp:
                         json.dump(references, fp)
-                        print("references were saved at references.json")
+                        print(f"references were saved at {self.args.save_references_path}")
 
             # make sure tokenizer plays nice with multiprocessing
             os.environ["TOKENIZERS_PARALLELISM"] = "false"

main.py

@@ -169,6 +169,12 @@ def parse_args():
         action="store_true",
         help="Whether to save reference solutions/tests",
     )
+    parser.add_argument(
+        "--save_references_path",
+        type=str,
+        default="references.json",
+        help="Path for saving the references solutions/tests",
+    )
     parser.add_argument(
         "--prompt",
         type=str,
@@ -335,9 +341,9 @@ def main():
                         json.dump(generations, fp)
                         print(f"generations were saved at {args.save_generations_path}")
                     if args.save_references:
-                        with open("references.json", "w") as fp:
+                        with open(args.save_references_path, "w") as fp:
                             json.dump(references, fp)
-                            print("references were saved")
+                            print(f"references were saved at {args.save_references_path}")
             else:
                 results[task] = evaluator.evaluate(task)
 

tests/test_generation_evaluation.py

@@ -31,6 +31,7 @@ def update_args(args):
     args.save_generations = False
     args.save_generations_path = ""
     args.save_references = False
+    args.save_references_path = ""
     args.metric_output_path = TMPDIR
     args.load_generations_path = None
     args.generation_only = False