fix

huggingface · Feb 1, 2024 · 1e7eb76 · 1e7eb76
1 parent 1221f4e
commit 1e7eb76
Showing 1 changed file with 4 additions and 3 deletions.
diff --git a/src/lighteval/metrics/__init__.py b/src/lighteval/metrics/__init__.py
@@ -8,11 +8,12 @@
 
 def apply_target_perplexity_metric(results: list[ModelReturn], formatted_doc: Doc, metrics: list[str]):
     outputs = {}
-    current_results = [results.pop(0) for _ in range(len(formatted_doc.get_golds()))]
+    current_result = results.pop(0)
+    reference_text = formatted_doc.choices[formatted_doc.gold_index]
 
     for metric in metrics:
-        if Metrics[metric].value.category == MetricCategory.PERPLEXITY:
-            outputs.update(Metrics[metric].value.compute(results=current_results))
+        if Metrics[metric].value.category == MetricCategory.TARGET_PERPLEXITY:
+            outputs.update(Metrics[metric].value.compute(results=current_result, reference_text=reference_text))
 
     return results, outputs