diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp index 4aaf4c8250ce5..5c9eb87566dde 100644 --- a/src/llama-vocab.cpp +++ b/src/llama-vocab.cpp @@ -3078,7 +3078,7 @@ int32_t llama_vocab::tokenize( LLAMA_LOG_ERROR("%s: tokenization result size %zu exceeds int32_t limit\n", __func__, res.size()); return std::numeric_limits::min(); } - + if (n_tokens_max < (int) res.size()) { // LLAMA_LOG_ERROR("%s: too many tokens\n", __func__); return -((int) res.size());