From 185416884131b7f7330405e0a730197f5ccd3284 Mon Sep 17 00:00:00 2001 From: Concedo <39025047+LostRuins@users.noreply.github.com> Date: Sat, 8 Jul 2023 20:31:49 +0800 Subject: [PATCH 1/3] This allows LLAMA models that were previously incompatible with K quants to function mostly as normal. This happens when a model has a vocab != 32000, e.g 32001 which means it's not divisible by 256 or 64. Since the problematic dimensions only apply for `tok_embeddings.weight` and `output.weight` (dimentions 4096 x n_vocab), we can simply quantize these layers to Q8_0 whereas the majority of the hidden layers are still K-quanted since they have compatible dimensions. --- llama.cpp | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/llama.cpp b/llama.cpp index ee6ec0920fc9c..be48a4e185bc3 100644 --- a/llama.cpp +++ b/llama.cpp @@ -2428,15 +2428,15 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s } else { new_type = quantized_type; #ifdef GGML_USE_K_QUANTS + bool convert_incompatible_tensor = false; if (quantized_type == GGML_TYPE_Q2_K || quantized_type == GGML_TYPE_Q3_K || quantized_type == GGML_TYPE_Q4_K || quantized_type == GGML_TYPE_Q5_K || quantized_type == GGML_TYPE_Q6_K) { int nx = tensor.ne.at(0); int ny = tensor.ne.at(1); if (nx % QK_K != 0 || ny % QK_K != 0) { - fprintf(stderr, "\n\n========================= Tensor sizes %d x %d are not divisible by %d\n",nx,ny,QK_K); - fprintf(stderr, "This is required to be able to use k-quants for now!\n"); - fprintf(stderr, "========================================================================================\n\n"); - throw std::runtime_error("Unsupported tensor size encountered\n"); + fprintf(stderr, "\n\nTensor sizes %d x %d are not divisible by %d, required for k-quants.\n",nx,ny,QK_K); + fprintf(stderr, "Q8_0 will be used for this tensor instead.\n"); + convert_incompatible_tensor = true; } } if (tensor.name == "output.weight") { @@ -2464,6 +2464,10 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M || ftype == LLAMA_FTYPE_MOSTLY_Q2_K) new_type = GGML_TYPE_Q4_K; else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L) new_type = GGML_TYPE_Q5_K; } + if(convert_incompatible_tensor) + { + new_type = GGML_TYPE_Q8_0; //fall back to Q8_0 instead of just failing. + } #endif float * f32_data; From 048dca9809dbfb8acf37a21cf130235c465532bd Mon Sep 17 00:00:00 2001 From: LostRuins <39025047+LostRuins@users.noreply.github.com> Date: Mon, 10 Jul 2023 22:57:15 +0800 Subject: [PATCH 2/3] Fix indentation Co-authored-by: Georgi Gerganov --- llama.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/llama.cpp b/llama.cpp index be48a4e185bc3..4ab2917938c7b 100644 --- a/llama.cpp +++ b/llama.cpp @@ -2464,8 +2464,7 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M || ftype == LLAMA_FTYPE_MOSTLY_Q2_K) new_type = GGML_TYPE_Q4_K; else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L) new_type = GGML_TYPE_Q5_K; } - if(convert_incompatible_tensor) - { + if (convert_incompatible_tensor) { new_type = GGML_TYPE_Q8_0; //fall back to Q8_0 instead of just failing. } #endif From fd9a2fdfe2dd7a54546be2c88b196e193ae7920e Mon Sep 17 00:00:00 2001 From: Concedo <39025047+LostRuins@users.noreply.github.com> Date: Mon, 10 Jul 2023 23:22:45 +0800 Subject: [PATCH 3/3] As an alternative, to avoid failing on Metal due to lack of Q8_0 support, instead quantize tok_embeddings.weight to Q4_0 and retain output.weight as F16. This results in a net gain of about 55mb for a 7B model compared to previous approach, but should minimize adverse impact to model quality. --- llama.cpp | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/llama.cpp b/llama.cpp index 4ab2917938c7b..d770a7c8fdc69 100644 --- a/llama.cpp +++ b/llama.cpp @@ -2435,7 +2435,6 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s int ny = tensor.ne.at(1); if (nx % QK_K != 0 || ny % QK_K != 0) { fprintf(stderr, "\n\nTensor sizes %d x %d are not divisible by %d, required for k-quants.\n",nx,ny,QK_K); - fprintf(stderr, "Q8_0 will be used for this tensor instead.\n"); convert_incompatible_tensor = true; } } @@ -2465,7 +2464,15 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L) new_type = GGML_TYPE_Q5_K; } if (convert_incompatible_tensor) { - new_type = GGML_TYPE_Q8_0; //fall back to Q8_0 instead of just failing. + if (tensor.name == "output.weight") { + new_type = GGML_TYPE_F16; //fall back to F16 instead of just failing. + fprintf(stderr, "F16 will be used for this tensor instead.\n"); + } else if (tensor.name == "tok_embeddings.weight") { + new_type = GGML_TYPE_Q4_0; //fall back to Q4_0 instead of just failing. + fprintf(stderr, "Q4_0 will be used for this tensor instead.\n"); + } else { + throw std::runtime_error("Unsupported tensor size encountered\n"); + } } #endif