From 63c6042118f75e342e614ba010e9a1c88af5fd19 Mon Sep 17 00:00:00 2001
From: yumeyao <yumeyao@gmail.com>
Date: Wed, 2 Apr 2025 14:17:06 +0800
Subject: [PATCH] Use string_view::find() to search for tokenization to avoid
 unnecessary looking up beyond the fragment range

---
 src/llama-vocab.cpp | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp
index 31e2055f83611..934f3ed0ddf5d 100644
--- a/src/llama-vocab.cpp
+++ b/src/llama-vocab.cpp
@@ -2220,14 +2220,12 @@ void llama_vocab::impl::tokenizer_st_partition(std::forward_list<fragment_buffer
                     // find the first occurrence of a given special token in this fragment
                     //  passing offset argument only limit the "search area" but match coordinates
                     //  are still relative to the source full raw_text
-                    auto match = raw_text.find(text, raw_text_base_offset);
+                    //  string_view begins at pos 0 for the same reason
+                    auto match = std::string_view(raw_text.data(), raw_text_base_offset + raw_text_base_length).find(text, raw_text_base_offset);
 
                     // no occurrences found, stop processing this fragment for a given special token
                     if (match == std::string::npos) break;
 
-                    // check if match is within bounds of offset <-> length
-                    if (match + text.length() > raw_text_base_offset + raw_text_base_length) break;
-
 #ifdef PRETOKENIZERDEBUG
                     LLAMA_LOG_WARN("FF: (%ld %ld %ld) '%s'\n", raw_text->length(), raw_text_base_offset, raw_text_base_length, raw_text->substr(raw_text_base_offset, raw_text_base_length).c_str());
 #endif