From ba210e4bc781d37d437feb51c063c315e9a5720d Mon Sep 17 00:00:00 2001 From: "Wang Haoran(Robin)" Date: Wed, 21 Jun 2023 14:21:35 -0700 Subject: [PATCH 01/10] server: add option to output probabilities for completion --- examples/common.h | 1 + examples/server/server.cpp | 126 +++++++++++++++++++++++++++++-------- 2 files changed, 100 insertions(+), 27 deletions(-) diff --git a/examples/common.h b/examples/common.h index 6c2953cb2a7c6..6393c85636f2a 100644 --- a/examples/common.h +++ b/examples/common.h @@ -31,6 +31,7 @@ struct gpt_params { int32_t main_gpu = 0; // the GPU that is used for scratch and small tensors float tensor_split[LLAMA_MAX_DEVICES] = {0}; // how split tensors should be distributed across GPUs bool low_vram = 0; // if true, reduce VRAM usage at the cost of performance + int32_t n_probs = 0; // if greater than 1, output the probabilities of top n_probs tokens. Max 5 // sampling parameters std::unordered_map logit_bias; // logit bias for specific tokens diff --git a/examples/server/server.cpp b/examples/server/server.cpp index c0984aadb92ba..16f5bac0abf35 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -26,6 +26,28 @@ struct server_params { int32_t write_timeout = 600; }; +// completion string output with probabilities +struct completion_string_output { + struct token_prob { + std::string tok_str; + float prob; + }; + + std::vector probs; + std::string tok_str; +}; + +// completion token output with probabilities +struct completion_token_output { + struct token_prob { + llama_token tok; + float prob; + }; + + std::vector probs; + llama_token tok; +}; + static size_t common_part(const std::vector & a, const std::vector & b) { size_t i; for (i = 0; i < a.size() && i < b.size() && a[i] == b[i]; i++) {} @@ -107,6 +129,7 @@ struct llama_server_context { bool stream = false; bool has_next_token = false; std::string generated_text; + std::vector generated_text_probs; size_t num_tokens_predicted = 0; size_t n_past = 0; @@ -137,6 +160,7 @@ struct llama_server_context { num_tokens_predicted = 0; generated_text = ""; generated_text.reserve(params.n_ctx); + generated_text_probs.clear(); truncated = false; stopped_eos = false; stopped_word = false; @@ -216,8 +240,9 @@ struct llama_server_context { llama_set_rng_seed(ctx, params.seed); } - llama_token nextToken() { - llama_token result = -1; + completion_token_output nextToken() { + completion_token_output result; + result.tok = -1; if (embd.size() >= (size_t)params.n_ctx) { // Reset context @@ -256,7 +281,8 @@ struct llama_server_context { if (params.n_predict == 0) { has_next_token = false; - return llama_token_eos(); + result.tok = llama_token_eos(); + return result; } // out of user input, sample next token @@ -273,7 +299,7 @@ struct llama_server_context { const float mirostat_tau = params.mirostat_tau; const float mirostat_eta = params.mirostat_eta; const bool penalize_nl = params.penalize_nl; - llama_token id = 0; + const int32_t n_probs = params.n_probs; { auto * logits = llama_get_logits(ctx); @@ -307,17 +333,17 @@ struct llama_server_context { if (temp <= 0) { // Greedy sampling - id = llama_sample_token_greedy(ctx, &candidates_p); + result.tok = llama_sample_token_greedy(ctx, &candidates_p); } else { if (mirostat == 1) { static float mirostat_mu = 2.0f * mirostat_tau; const int mirostat_m = 100; llama_sample_temperature(ctx, &candidates_p, temp); - id = llama_sample_token_mirostat(ctx, &candidates_p, mirostat_tau, mirostat_eta, mirostat_m, &mirostat_mu); + result.tok = llama_sample_token_mirostat(ctx, &candidates_p, mirostat_tau, mirostat_eta, mirostat_m, &mirostat_mu); } else if (mirostat == 2) { static float mirostat_mu = 2.0f * mirostat_tau; llama_sample_temperature(ctx, &candidates_p, temp); - id = llama_sample_token_mirostat_v2(ctx, &candidates_p, mirostat_tau, mirostat_eta, &mirostat_mu); + result.tok = llama_sample_token_mirostat_v2(ctx, &candidates_p, mirostat_tau, mirostat_eta, &mirostat_mu); } else { // Temperature sampling llama_sample_tail_free(ctx, &candidates_p, tfs_z, 1); @@ -325,17 +351,19 @@ struct llama_server_context { llama_sample_top_p(ctx, &candidates_p, top_p, 1); llama_sample_top_k(ctx, &candidates_p, top_k, 1); llama_sample_temperature(ctx, &candidates_p, temp); - id = llama_sample_token(ctx, &candidates_p); + result.tok = llama_sample_token(ctx, &candidates_p); } } + for (size_t i = 0; i < std::min(candidates_p.size, std::min((size_t) n_probs, size_t(5))); ++i) { + result.probs.push_back({candidates_p.data[i].id, candidates_p.data[i].p}); + } last_n_tokens.erase(last_n_tokens.begin()); - last_n_tokens.push_back(id); + last_n_tokens.push_back(result.tok); num_tokens_predicted++; } // add it to the context - embd.push_back(id); - result = id; + embd.push_back(result.tok); // decrement remaining sampling budget --n_remain; @@ -377,12 +405,22 @@ struct llama_server_context { return stop_pos; } - std::string doCompletion() { - const llama_token token = nextToken(); + completion_string_output doCompletion() { + const completion_token_output token_with_probs = nextToken(); + completion_string_output result; - const std::string token_text = token == -1 ? "" : llama_token_to_str(ctx, token); + const std::string token_text = token_with_probs.tok == -1 ? "" : llama_token_to_str(ctx, token_with_probs.tok); + result.tok_str = token_text; generated_text += token_text; + // iterate through token_with_probs.probs, if tok is valid, convert it to string and add to result.prob + for (const auto & prob : token_with_probs.probs) { + const std::string prob_text = prob.tok == -1 ? "" : llama_token_to_str(ctx, prob.tok); + result.probs.push_back({prob_text, prob.prob}); + } + + generated_text_probs.push_back(result); + if (multibyte_pending > 0) { multibyte_pending -= token_text.size(); } else if (token_text.size() == 1) { @@ -411,8 +449,8 @@ struct llama_server_context { } LOG_VERBOSE("next token", { - { "token", token }, - { "token_text", llama_token_to_str(ctx, token) }, + { "token", token_with_probs.tok }, + { "token_text", llama_token_to_str(ctx, token_with_probs.tok) }, { "has_next_token", has_next_token }, { "n_remain", n_remain }, { "num_tokens_predicted", num_tokens_predicted }, @@ -422,7 +460,7 @@ struct llama_server_context { { "stopping_word", stopping_word }, }); - return token_text; + return result; } std::vector getEmbedding() { @@ -664,6 +702,7 @@ static json format_generation_settings(llama_server_context & llama) { { "ignore_eos", ignore_eos }, { "stream", llama.stream }, { "logit_bias", llama.params.logit_bias }, + { "n_probs", llama.params.n_probs }, }; } @@ -673,9 +712,26 @@ static json format_embedding_response(llama_server_context & llama) { }; } -static json format_final_response(llama_server_context & llama, const std::string & content) { +static json format_final_response(llama_server_context & llama, const std::string & content, const std::vector & probs) { + + json completion_probabilities_json = json::array(); + for (const auto & prob : probs) { + json probs_for_token = json::array(); + for (const auto & p : prob.probs) { + probs_for_token.push_back(json { + { "tok_str", p.tok_str }, + { "prob", p.prob }, + }); + } + completion_probabilities_json.push_back(json { + {"content", prob.tok_str}, + {"probs", probs_for_token}, + }); + } + return json { { "content", content }, + { "completion_probabilities", completion_probabilities_json}, { "stop", true }, { "model", llama.params.model_alias }, { "tokens_predicted", llama.num_tokens_predicted }, @@ -689,11 +745,25 @@ static json format_final_response(llama_server_context & llama, const std::strin }; } -static json format_partial_response(const std::string & content) { - return json { +static json format_partial_response(const std::string & content, const completion_string_output & probs) { + json res = json { { "content", content }, { "stop", false }, }; + + // iterate through probs.probs, and add to res + json probs_json = json::array(); + for (const auto & prob : probs.probs) { + probs_json.push_back(json { + { "tok_str", prob.tok_str }, + { "prob", prob.prob }, + }); + } + if (probs.probs.size() > 0) { + res["probs"] = probs_json; + } + + return res; } static json format_tokenizer_response(const std::vector & tokens) { @@ -723,6 +793,7 @@ static void parse_options_completion(const json & body, llama_server_context & l llama.params.n_keep = body.value("n_keep", default_params.n_keep); llama.params.seed = body.value("seed", default_params.seed); llama.params.prompt = body.value("prompt", default_params.prompt); + llama.params.n_probs = body.value("n_probs", default_params.n_probs); llama.params.logit_bias.clear(); if (body.value("ignore_eos", false)) { @@ -825,7 +896,8 @@ int main(int argc, char ** argv) { size_t stop_pos = std::string::npos; while (llama.has_next_token) { - const std::string token_text = llama.doCompletion(); + const completion_string_output token_text_with_probs = llama.doCompletion(); + const std::string token_text = token_text_with_probs.tok_str; stop_pos = llama.findStoppingStrings(llama.generated_text, token_text.size(), STOP_FULL); @@ -839,7 +911,7 @@ int main(int argc, char ** argv) { llama.generated_text.end()); } - const json data = format_final_response(llama, llama.generated_text); + const json data = format_final_response(llama, llama.generated_text, llama.generated_text_probs); llama_print_timings(llama.ctx); @@ -850,7 +922,7 @@ int main(int argc, char ** argv) { size_t sent_count = 0; while (llama.has_next_token) { - const std::string token_text = llama.doCompletion(); + const completion_string_output token_text_with_probs = llama.doCompletion(); if (llama.multibyte_pending > 0) { continue; } @@ -859,14 +931,14 @@ int main(int argc, char ** argv) { const std::string str_test = llama.generated_text.substr(pos); size_t stop_pos = - llama.findStoppingStrings(str_test, token_text.size(), STOP_FULL); + llama.findStoppingStrings(str_test, token_text_with_probs.tok_str.size(), STOP_FULL); if (stop_pos != std::string::npos) { llama.generated_text.erase( llama.generated_text.begin() + pos + stop_pos, llama.generated_text.end()); pos = std::min(sent_count, llama.generated_text.size()); } else { - stop_pos = llama.findStoppingStrings(str_test, token_text.size(), + stop_pos = llama.findStoppingStrings(str_test, token_text_with_probs.tok_str.size(), STOP_PARTIAL); } @@ -874,9 +946,9 @@ int main(int argc, char ** argv) { sent_count += to_send.size(); const json data = llama.has_next_token - ? format_partial_response(to_send) + ? format_partial_response(to_send, token_text_with_probs) // Generation is done, send extra information. - : format_final_response(llama, to_send); + : format_final_response(llama, to_send, {token_text_with_probs}); const std::string str = "data: " + From ccf254bd4450049dc623dd3a8cabfda70d452483 Mon Sep 17 00:00:00 2001 From: "Wang Haoran(Robin)" Date: Thu, 22 Jun 2023 08:57:35 -0700 Subject: [PATCH 02/10] server: fix comment about max n_probs --- examples/common.h | 2 +- examples/server/server.cpp | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/common.h b/examples/common.h index 6393c85636f2a..5ceac53c5233d 100644 --- a/examples/common.h +++ b/examples/common.h @@ -31,7 +31,7 @@ struct gpt_params { int32_t main_gpu = 0; // the GPU that is used for scratch and small tensors float tensor_split[LLAMA_MAX_DEVICES] = {0}; // how split tensors should be distributed across GPUs bool low_vram = 0; // if true, reduce VRAM usage at the cost of performance - int32_t n_probs = 0; // if greater than 1, output the probabilities of top n_probs tokens. Max 5 + int32_t n_probs = 0; // if greater than 0, output the probabilities of top n_probs tokens. // sampling parameters std::unordered_map logit_bias; // logit bias for specific tokens diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 16f5bac0abf35..4cad658d3eb3a 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -354,6 +354,7 @@ struct llama_server_context { result.tok = llama_sample_token(ctx, &candidates_p); } } + // Add maximum of 5 most probable tokens to the result for (size_t i = 0; i < std::min(candidates_p.size, std::min((size_t) n_probs, size_t(5))); ++i) { result.probs.push_back({candidates_p.data[i].id, candidates_p.data[i].p}); } From cf7619522335100284f35b94be5104dded20b407 Mon Sep 17 00:00:00 2001 From: "Wang Haoran(Robin)" Date: Thu, 22 Jun 2023 21:35:37 -0700 Subject: [PATCH 03/10] server: fix issue when handling probability output for incomplete tokens for multibyte character generation --- examples/server/server.cpp | 135 ++++++++++++++++++++----------------- 1 file changed, 73 insertions(+), 62 deletions(-) diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 4cad658d3eb3a..d9af1308e3d8a 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -26,17 +26,6 @@ struct server_params { int32_t write_timeout = 600; }; -// completion string output with probabilities -struct completion_string_output { - struct token_prob { - std::string tok_str; - float prob; - }; - - std::vector probs; - std::string tok_str; -}; - // completion token output with probabilities struct completion_token_output { struct token_prob { @@ -108,6 +97,36 @@ static void server_log(const char * level, const char * function, int line, fflush(stdout); } +// format incomplete utf-8 multibyte character for output +static std::string tokens_to_output_formatted_string(const llama_context * ctx, const llama_token token) { + const std::string out = token == -1 ? "" : llama_token_to_str(ctx, token); + if (out[0] > 127) { + out = "byte: \\x" + std::format("{:x}", out[0]); + } + return out; +} + +// convert a vector of completion_token_output to json +static json probs_vector_to_json(const llama_context * ctx, const vector probs) { + json out = json::array(); + for (const auto & prob : probs) { + json probs_for_token = json::array(); + for (const auto & p : prob.probs) { + std::string tok_str = tokens_to_output_formatted_string(ctx, p.tok); + probs_for_token.push_back(json { + { "tok_str", tok_str }, + { "prob", p.prob }, + }); + } + std::string tok_str = tokens_to_output_formatted_string(ctx, prob.tok); + out.push_back(json { + {"content", tok_str}, + {"probs", probs_for_token}, + }); + } + return out; +} + static bool server_verbose = false; #if SERVER_VERBOSE != 1 @@ -129,7 +148,7 @@ struct llama_server_context { bool stream = false; bool has_next_token = false; std::string generated_text; - std::vector generated_text_probs; + std::vector generated_token_probs; size_t num_tokens_predicted = 0; size_t n_past = 0; @@ -160,7 +179,7 @@ struct llama_server_context { num_tokens_predicted = 0; generated_text = ""; generated_text.reserve(params.n_ctx); - generated_text_probs.clear(); + generated_token_probs.clear(); truncated = false; stopped_eos = false; stopped_word = false; @@ -406,22 +425,16 @@ struct llama_server_context { return stop_pos; } - completion_string_output doCompletion() { + completion_token_output doCompletion() { const completion_token_output token_with_probs = nextToken(); - completion_string_output result; const std::string token_text = token_with_probs.tok == -1 ? "" : llama_token_to_str(ctx, token_with_probs.tok); - result.tok_str = token_text; generated_text += token_text; - // iterate through token_with_probs.probs, if tok is valid, convert it to string and add to result.prob - for (const auto & prob : token_with_probs.probs) { - const std::string prob_text = prob.tok == -1 ? "" : llama_token_to_str(ctx, prob.tok); - result.probs.push_back({prob_text, prob.prob}); + if (params.n_probs > 0) { + generated_token_probs.push_back(token_with_probs); } - generated_text_probs.push_back(result); - if (multibyte_pending > 0) { multibyte_pending -= token_text.size(); } else if (token_text.size() == 1) { @@ -451,7 +464,7 @@ struct llama_server_context { LOG_VERBOSE("next token", { { "token", token_with_probs.tok }, - { "token_text", llama_token_to_str(ctx, token_with_probs.tok) }, + { "token_text", tokens_to_output_formatted_string(ctx, token_with_probs.tok) }, { "has_next_token", has_next_token }, { "n_remain", n_remain }, { "num_tokens_predicted", num_tokens_predicted }, @@ -461,7 +474,7 @@ struct llama_server_context { { "stopping_word", stopping_word }, }); - return result; + return token_with_probs; } std::vector getEmbedding() { @@ -713,26 +726,10 @@ static json format_embedding_response(llama_server_context & llama) { }; } -static json format_final_response(llama_server_context & llama, const std::string & content, const std::vector & probs) { +static json format_final_response(llama_server_context & llama, const std::string & content, const std::vector & probs) { - json completion_probabilities_json = json::array(); - for (const auto & prob : probs) { - json probs_for_token = json::array(); - for (const auto & p : prob.probs) { - probs_for_token.push_back(json { - { "tok_str", p.tok_str }, - { "prob", p.prob }, - }); - } - completion_probabilities_json.push_back(json { - {"content", prob.tok_str}, - {"probs", probs_for_token}, - }); - } - - return json { + json res = json { { "content", content }, - { "completion_probabilities", completion_probabilities_json}, { "stop", true }, { "model", llama.params.model_alias }, { "tokens_predicted", llama.num_tokens_predicted }, @@ -743,25 +740,25 @@ static json format_final_response(llama_server_context & llama, const std::strin { "stopped_word", llama.stopped_word }, { "stopped_limit", llama.stopped_limit }, { "stopping_word", llama.stopping_word }, - }; + } + + if (llama.params.n_probs > 0) { + json completion_probabilities_json = probs_vector_to_json(llama.ctx, probs); + res["completion_probabilities"] = completion_probabilities_json; + } + + return res; } -static json format_partial_response(const std::string & content, const completion_string_output & probs) { +static json format_partial_response(llama_server_context & llama, const std::string & content, const std::vector & probs) { json res = json { { "content", content }, { "stop", false }, }; - // iterate through probs.probs, and add to res - json probs_json = json::array(); - for (const auto & prob : probs.probs) { - probs_json.push_back(json { - { "tok_str", prob.tok_str }, - { "prob", prob.prob }, - }); - } - if (probs.probs.size() > 0) { - res["probs"] = probs_json; + if (llama.params.n_probs > 0) { + json completion_probabilities_json = probs_vector_to_json(llama.ctx, probs); + res["completion_probabilities"] = completion_probabilities_json; } return res; @@ -897,8 +894,8 @@ int main(int argc, char ** argv) { size_t stop_pos = std::string::npos; while (llama.has_next_token) { - const completion_string_output token_text_with_probs = llama.doCompletion(); - const std::string token_text = token_text_with_probs.tok_str; + const completion_token_output token_with_probs = llama.doCompletion(); + const std::string token_text = llama_token_to_str(llama.ctx, token_with_probs.tok); stop_pos = llama.findStoppingStrings(llama.generated_text, token_text.size(), STOP_FULL); @@ -912,7 +909,7 @@ int main(int argc, char ** argv) { llama.generated_text.end()); } - const json data = format_final_response(llama, llama.generated_text, llama.generated_text_probs); + const json data = format_final_response(llama, llama.generated_text, llama.generated_token_probs); llama_print_timings(llama.ctx); @@ -921,9 +918,11 @@ int main(int argc, char ** argv) { } else { const auto chunked_content_provider = [&](size_t, DataSink & sink) { size_t sent_count = 0; + size_t sent_token_probs_index = 0; while (llama.has_next_token) { - const completion_string_output token_text_with_probs = llama.doCompletion(); + const completion_token_output token_with_probs = llama.doCompletion(); + const std::string token_text = llama_token_to_str(llama.ctx, token_with_probs.tok); if (llama.multibyte_pending > 0) { continue; } @@ -932,24 +931,36 @@ int main(int argc, char ** argv) { const std::string str_test = llama.generated_text.substr(pos); size_t stop_pos = - llama.findStoppingStrings(str_test, token_text_with_probs.tok_str.size(), STOP_FULL); + llama.findStoppingStrings(str_test, token_text.size(), STOP_FULL); if (stop_pos != std::string::npos) { llama.generated_text.erase( llama.generated_text.begin() + pos + stop_pos, llama.generated_text.end()); pos = std::min(sent_count, llama.generated_text.size()); } else { - stop_pos = llama.findStoppingStrings(str_test, token_text_with_probs.tok_str.size(), + stop_pos = llama.findStoppingStrings(str_test, token_text.size(), STOP_PARTIAL); } const std::string to_send = llama.generated_text.substr(pos, stop_pos); sent_count += to_send.size(); + std::vector probs_output = {}; + + if (llama.params.n_probs > 0) { + const std::vector to_send_toks = llama_tokenize(llama.ctx, to_send, false); + size_t probs_pos = std::min(sent_token_probs_index, llama.generated_token_probs.size()); + size_t probs_stop_pos = std::min(sent_token_probs_index + to_send_toks.size(), llama.generated_token_probs.size()); + if (probs_pos < probs_stop_pos) { + probs_output = std::vector(llama.generated_token_probs.begin() + probs_pos, llama.generated_token_probs.begin() + probs_stop_pos); + } + sent_token_probs_index = probs_stop_pos; + } + const json data = llama.has_next_token - ? format_partial_response(to_send, token_text_with_probs) + ? format_partial_response(llama, to_send, probs_output) // Generation is done, send extra information. - : format_final_response(llama, to_send, {token_text_with_probs}); + : format_final_response(llama, to_send, probs_output); const std::string str = "data: " + From 7b93b248efa828129b14353a2f3a9f0a790b6014 Mon Sep 17 00:00:00 2001 From: "Wang Haoran(Robin)" Date: Thu, 22 Jun 2023 21:59:12 -0700 Subject: [PATCH 04/10] server: fix some beginner mistakes --- examples/server/server.cpp | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/examples/server/server.cpp b/examples/server/server.cpp index d9af1308e3d8a..ea484560eb996 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -99,15 +99,19 @@ static void server_log(const char * level, const char * function, int line, // format incomplete utf-8 multibyte character for output static std::string tokens_to_output_formatted_string(const llama_context * ctx, const llama_token token) { - const std::string out = token == -1 ? "" : llama_token_to_str(ctx, token); - if (out[0] > 127) { - out = "byte: \\x" + std::format("{:x}", out[0]); + std::string out = token == -1 ? "" : llama_token_to_str(ctx, token); + // if first bit is 1, meaning it's a partial character + if ((out[0] & 0x80) == 0x80) { + std::stringstream ss; + ss<< std::hex << (out[0] & 0xff); + std::string res ( ss.str() ); + out = "byte: \\x" + res; } return out; } // convert a vector of completion_token_output to json -static json probs_vector_to_json(const llama_context * ctx, const vector probs) { +static json probs_vector_to_json(const llama_context * ctx, const std::vector probs) { json out = json::array(); for (const auto & prob : probs) { json probs_for_token = json::array(); @@ -740,7 +744,7 @@ static json format_final_response(llama_server_context & llama, const std::strin { "stopped_word", llama.stopped_word }, { "stopped_limit", llama.stopped_limit }, { "stopping_word", llama.stopping_word }, - } + }; if (llama.params.n_probs > 0) { json completion_probabilities_json = probs_vector_to_json(llama.ctx, probs); From 02c96a4cbbfde81f8f0ce67ec8275f1a7990b573 Mon Sep 17 00:00:00 2001 From: "Wang Haoran(Robin)" Date: Sat, 24 Jun 2023 07:54:26 -0700 Subject: [PATCH 05/10] server: remove trailling white space --- examples/server/server.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/server/server.cpp b/examples/server/server.cpp index ea484560eb996..4ac8d6fbdadb5 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -103,7 +103,7 @@ static std::string tokens_to_output_formatted_string(const llama_context * ctx, // if first bit is 1, meaning it's a partial character if ((out[0] & 0x80) == 0x80) { std::stringstream ss; - ss<< std::hex << (out[0] & 0xff); + ss<< std::hex << (out[0] & 0xff); std::string res ( ss.str() ); out = "byte: \\x" + res; } From e815b695792aa8b3e744a8eb825549590aa3e7f3 Mon Sep 17 00:00:00 2001 From: "Wang Haoran(Robin)" Date: Sun, 25 Jun 2023 14:15:14 -0700 Subject: [PATCH 06/10] server: remove n_probs upper limit of 5 --- examples/server/server.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 4ac8d6fbdadb5..3d076060ca747 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -101,7 +101,7 @@ static void server_log(const char * level, const char * function, int line, static std::string tokens_to_output_formatted_string(const llama_context * ctx, const llama_token token) { std::string out = token == -1 ? "" : llama_token_to_str(ctx, token); // if first bit is 1, meaning it's a partial character - if ((out[0] & 0x80) == 0x80) { + if (out.size() > 0 && (out[0] & 0x80) == 0x80) { std::stringstream ss; ss<< std::hex << (out[0] & 0xff); std::string res ( ss.str() ); @@ -377,8 +377,8 @@ struct llama_server_context { result.tok = llama_sample_token(ctx, &candidates_p); } } - // Add maximum of 5 most probable tokens to the result - for (size_t i = 0; i < std::min(candidates_p.size, std::min((size_t) n_probs, size_t(5))); ++i) { + + for (size_t i = 0; i < std::min(candidates_p.size, (size_t) n_probs); ++i) { result.probs.push_back({candidates_p.data[i].id, candidates_p.data[i].p}); } last_n_tokens.erase(last_n_tokens.begin()); From c9e6642cf7eae4223e96f224a6c50d0712f6a2c2 Mon Sep 17 00:00:00 2001 From: "Wang Haoran(Robin)" Date: Sun, 25 Jun 2023 16:29:34 -0700 Subject: [PATCH 07/10] server: handle probs output when temp=0; handle final response probs output --- examples/server/server.cpp | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 3d076060ca747..e349489f80f67 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -357,6 +357,9 @@ struct llama_server_context { if (temp <= 0) { // Greedy sampling result.tok = llama_sample_token_greedy(ctx, &candidates_p); + if (n_probs > 0) { + llama_sample_softmax(ctx, &candidates_p); + } } else { if (mirostat == 1) { static float mirostat_mu = 2.0f * mirostat_tau; @@ -369,10 +372,11 @@ struct llama_server_context { result.tok = llama_sample_token_mirostat_v2(ctx, &candidates_p, mirostat_tau, mirostat_eta, &mirostat_mu); } else { // Temperature sampling - llama_sample_tail_free(ctx, &candidates_p, tfs_z, 1); - llama_sample_typical(ctx, &candidates_p, typical_p, 1); - llama_sample_top_p(ctx, &candidates_p, top_p, 1); - llama_sample_top_k(ctx, &candidates_p, top_k, 1); + size_t min_keep = std::max(1, n_probs); + llama_sample_tail_free(ctx, &candidates_p, tfs_z, min_keep); + llama_sample_typical(ctx, &candidates_p, typical_p, min_keep); + llama_sample_top_p(ctx, &candidates_p, top_p, min_keep); + llama_sample_top_k(ctx, &candidates_p, top_k, min_keep); llama_sample_temperature(ctx, &candidates_p, temp); result.tok = llama_sample_token(ctx, &candidates_p); } @@ -747,8 +751,7 @@ static json format_final_response(llama_server_context & llama, const std::strin }; if (llama.params.n_probs > 0) { - json completion_probabilities_json = probs_vector_to_json(llama.ctx, probs); - res["completion_probabilities"] = completion_probabilities_json; + res["completion_probabilities"] = probs_vector_to_json(llama.ctx, probs); } return res; @@ -761,8 +764,7 @@ static json format_partial_response(llama_server_context & llama, const std::str }; if (llama.params.n_probs > 0) { - json completion_probabilities_json = probs_vector_to_json(llama.ctx, probs); - res["completion_probabilities"] = completion_probabilities_json; + res["completion_probabilities"] = probs_vector_to_json(llama.ctx, probs); } return res; @@ -964,7 +966,7 @@ int main(int argc, char ** argv) { const json data = llama.has_next_token ? format_partial_response(llama, to_send, probs_output) // Generation is done, send extra information. - : format_final_response(llama, to_send, probs_output); + : format_final_response(llama, to_send, llama.generated_token_probs); const std::string str = "data: " + From bc88fece87460e0121942a314aa4de71b97bad32 Mon Sep 17 00:00:00 2001 From: "Wang Haoran(Robin)" Date: Mon, 26 Jun 2023 18:11:27 -0700 Subject: [PATCH 08/10] server: fix llama_sample_top_k order --- examples/server/server.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/server/server.cpp b/examples/server/server.cpp index c0ed4b016c64b..e4ddbe9865506 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -378,10 +378,10 @@ struct llama_server_context { } else { // Temperature sampling size_t min_keep = std::max(1, n_probs); + llama_sample_top_k(ctx, &candidates_p, top_k, min_keep); llama_sample_tail_free(ctx, &candidates_p, tfs_z, min_keep); llama_sample_typical(ctx, &candidates_p, typical_p, min_keep); llama_sample_top_p(ctx, &candidates_p, top_p, min_keep); - llama_sample_top_k(ctx, &candidates_p, top_k, min_keep); llama_sample_temperature(ctx, &candidates_p, temp); result.tok = llama_sample_token(ctx, &candidates_p); } From 1a70a80369d64e61d658e82451634e932caddc29 Mon Sep 17 00:00:00 2001 From: "Wang Haoran(Robin)" Date: Sun, 2 Jul 2023 08:00:13 +0800 Subject: [PATCH 09/10] examples/common.h: put all bool variables in gpt_params together --- examples/common.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/common.h b/examples/common.h index 951511f8a5746..03312365eb27c 100644 --- a/examples/common.h +++ b/examples/common.h @@ -31,7 +31,6 @@ struct gpt_params { int32_t n_gpu_layers = 0; // number of layers to store in VRAM int32_t main_gpu = 0; // the GPU that is used for scratch and small tensors float tensor_split[LLAMA_MAX_DEVICES] = {0}; // how split tensors should be distributed across GPUs - bool low_vram = 0; // if true, reduce VRAM usage at the cost of performance int32_t n_probs = 0; // if greater than 0, output the probabilities of top n_probs tokens. // sampling parameters @@ -60,6 +59,7 @@ struct gpt_params { std::string lora_adapter = ""; // lora adapter path std::string lora_base = ""; // base model path for the lora adapter + bool low_vram = 0; // if true, reduce VRAM usage at the cost of performance bool memory_f16 = true; // use f16 instead of f32 for memory kv bool random_prompt = false; // do not randomize prompt if none provided bool use_color = false; // use color to distinguish generations and inputs From 71f829678aec335dac97cb44f255e5eb26544cf8 Mon Sep 17 00:00:00 2001 From: "Wang Haoran(Robin)" Date: Sun, 2 Jul 2023 08:01:19 +0800 Subject: [PATCH 10/10] examples/common.h: put all bool variables in gpt_params together --- examples/common.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/common.h b/examples/common.h index 03312365eb27c..96f2228f8677b 100644 --- a/examples/common.h +++ b/examples/common.h @@ -59,7 +59,7 @@ struct gpt_params { std::string lora_adapter = ""; // lora adapter path std::string lora_base = ""; // base model path for the lora adapter - bool low_vram = 0; // if true, reduce VRAM usage at the cost of performance + bool low_vram = false; // if true, reduce VRAM usage at the cost of performance bool memory_f16 = true; // use f16 instead of f32 for memory kv bool random_prompt = false; // do not randomize prompt if none provided bool use_color = false; // use color to distinguish generations and inputs