diff --git a/tokenizers/src/models/bpe/word.rs b/tokenizers/src/models/bpe/word.rs index 06b2db245..93b3d9c37 100644 --- a/tokenizers/src/models/bpe/word.rs +++ b/tokenizers/src/models/bpe/word.rs @@ -200,7 +200,8 @@ impl Word { // Make sure we are not processing an expired queue entry let target_new_pair = (self.symbols[top.pos].c, right.c); if merges - .get(&target_new_pair).is_none_or(|(_, new_id)| *new_id != top.new_id) + .get(&target_new_pair) + .is_none_or(|(_, new_id)| *new_id != top.new_id) { continue; }