more nits

huggingface · Nov 14, 2023 · 3ad148d · 3ad148d
1 parent 6547fd8
commit 3ad148d
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 3 deletions.
diff --git a/bindings/python/tests/bindings/test_pre_tokenizers.py b/bindings/python/tests/bindings/test_pre_tokenizers.py
@@ -110,8 +110,8 @@ def test_can_modify(self):
         assert pretok.replacement == "%"
         pretok.add_prefix_space = True
         assert pretok.add_prefix_space == True
-        pretok.prepend_scheme = "never"
-        assert pretok.prepend_scheme == "never"
+        pretok.prepend_scheme = "Never"
+        assert pretok.prepend_scheme == "Never"
 
 
 class TestCharDelimiterSplit:

diff --git a/tokenizers/src/pre_tokenizers/mod.rs b/tokenizers/src/pre_tokenizers/mod.rs
@@ -126,7 +126,11 @@ mod tests {
 
         assert_eq!(
             pre_tokenizer,
-            PreTokenizerWrapper::Metaspace(Metaspace::new('▁', true))
+            PreTokenizerWrapper::Metaspace(Metaspace::new_with_prepend_scheme(
+                '▁',
+                true,
+                metaspace::PrependScheme::Always
+            ))
         );
     }