Fix vocab encoding by disabling add_special_tokens in tokenizer.encode

2026-08-23 17:36:31 +00:00 · 2026-07-16 05:32:29 +02:00
parent 403efc7b00
commit 85a507c67d
1 changed files with 1 additions and 1 deletions
@@ -278,7 +278,7 @@ class Model(ABC):
            elif reverse_vocab[i] in added_vocab:
                # We need to manually encode and decode the added tokens in case special characters
                # used for `\n` / `\t` have been manually added in the added tokens
-                encoded_decoded_token = tokenizer.decode(tokenizer.encode(reverse_vocab[i]))
+                encoded_decoded_token = tokenizer.decode(tokenizer.encode(reverse_vocab[i], add_special_tokens=False))
                tokens.append(encoded_decoded_token)
                if tokenizer.added_tokens_decoder[i].special:
                    toktypes.append(gguf.TokenType.CONTROL)