From 8eed6237611fe516b64d323980e56d4b26fa316d Mon Sep 17 00:00:00 2001 From: Arjun Guha Date: Fri, 12 Jul 2024 09:36:37 -0400 Subject: [PATCH] Update MultiPL-E to v3 prompts I've put together a new MultiPL-E release, v3 and this updates the harness to point to the new prompts. There are fixes to old prompts as well as support for new languages, both described on the MultiPL-E dataset page. The nature of fixes is that some PLs have their scores go up a little. I've tested with with StarCoder2-15B (using VLLM) and will continue testing other models. --- bigcode_eval/tasks/multiple.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigcode_eval/tasks/multiple.py b/bigcode_eval/tasks/multiple.py index 7f514426d..d0166b683 100644 --- a/bigcode_eval/tasks/multiple.py +++ b/bigcode_eval/tasks/multiple.py @@ -80,7 +80,7 @@ class GeneralMultiPLE(Task): DATASET_PATH = "nuprl/MultiPL-E" DATASET_NAME = None - DATASET_REVISION = "d23b094346c5dbda1080a74bb2a24c18adbf7409" + DATASET_REVISION = "8a4cb75204eb3d5855a81778db6b95bfc80c9136" def __init__(self, language): self.language = language