End of list.

huggingface · Nov 7, 2024 · c32a2c2 · c32a2c2
1 parent 18b999c
commit c32a2c2
Showing 1 changed file with 1 addition and 0 deletions.
diff --git a/tokenizers/src/tokenizer/mod.rs b/tokenizers/src/tokenizer/mod.rs
@@ -1020,6 +1020,7 @@ pub struct DecodeStream<'tok, M, N, PT, PP, D> {
     ///  - read
     ///  - prefix
     ///  - rest
+    ///
     /// Read is the bit necessary to surround the prefix
     /// so decoding the whole ids produces a valid prefix.
     /// Prefix is the previously produced string, kept around to trim off of