From 85a507c67dd2e361f94152c1e8babff2cb4e7daf Mon Sep 17 00:00:00 2001 From: isHuangXin Date: Thu, 16 Jul 2026 05:32:29 +0200 Subject: [PATCH] Fix vocab encoding by disabling add_special_tokens in tokenizer.encode --- utils/convert-hf-to-gguf-bitnet.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/convert-hf-to-gguf-bitnet.py b/utils/convert-hf-to-gguf-bitnet.py index e5b02d2..b11e831 100644 --- a/utils/convert-hf-to-gguf-bitnet.py +++ b/utils/convert-hf-to-gguf-bitnet.py @@ -278,7 +278,7 @@ class Model(ABC): elif reverse_vocab[i] in added_vocab: # We need to manually encode and decode the added tokens in case special characters # used for `\n` / `\t` have been manually added in the added tokens - encoded_decoded_token = tokenizer.decode(tokenizer.encode(reverse_vocab[i])) + encoded_decoded_token = tokenizer.decode(tokenizer.encode(reverse_vocab[i], add_special_tokens=False)) tokens.append(encoded_decoded_token) if tokenizer.added_tokens_decoder[i].special: toktypes.append(gguf.TokenType.CONTROL)