diff --git a/bigcode_eval/evaluator.py b/bigcode_eval/evaluator.py
index fa70000b9..c9b29a2f0 100644
--- a/bigcode_eval/evaluator.py
+++ b/bigcode_eval/evaluator.py
@@ -114,7 +114,7 @@ def save_json_files(
         save_generations_path: str,
         save_references_path: str,
     ) -> None:
-        if self.args.save_generations:
+        if self.args.save_generations or self.args.generation_only:
             with open(save_generations_path, "w") as fp:
                 json.dump(generations, fp)
                 print(f"generations were saved at {save_generations_path}")