diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp index 31e2055f83611..934f3ed0ddf5d 100644 --- a/src/llama-vocab.cpp +++ b/src/llama-vocab.cpp @@ -2220,14 +2220,12 @@ void llama_vocab::impl::tokenizer_st_partition(std::forward_list length - if (match + text.length() > raw_text_base_offset + raw_text_base_length) break; - #ifdef PRETOKENIZERDEBUG LLAMA_LOG_WARN("FF: (%ld %ld %ld) '%s'\n", raw_text->length(), raw_text_base_offset, raw_text_base_length, raw_text->substr(raw_text_base_offset, raw_text_base_length).c_str()); #endif