fix(abstractions): deepcopy during transformation

PKU-Alignment · Dec 9, 2024 · 5d40c4d · 5d40c4d
1 parent c33bfc0
commit 5d40c4d
Showing 1 changed file with 3 additions and 0 deletions.
diff --git a/src/abstractions/data.py b/src/abstractions/data.py
@@ -15,6 +15,7 @@
 import os
 import json
 import warnings
+import copy
 from functools import partial
 import src.utils.text_utils as tu
 from tqdm import tqdm
@@ -283,6 +284,7 @@ def inv_map_key_fields_fn(sample_dict: Dict) -> Dict:
             if max_batch_size == 1:
                 with tu.JsonListReader(self.data_path) as reader:
                     for element in reader:
+                        element = copy.deepcopy(element)
                         if map_key_fields:
                             element = map_key_fields_fn(element)
 
@@ -295,6 +297,7 @@ def inv_map_key_fields_fn(sample_dict: Dict) -> Dict:
 
                 with tu.JsonListReader(self.data_path) as reader:
                     for element in reader:
+                        element = copy.deepcopy(element)
                         if map_key_fields:
                             element = map_key_fields_fn(element)