From fdb864a61d13c186a186a71c9ceeb9d381fe17e0 Mon Sep 17 00:00:00 2001 From: Jean-Christophe Hoelt Date: Thu, 16 Mar 2023 09:54:24 +0200 Subject: [PATCH 1/5] Add chatLLaMa script --- chatLLaMa | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100755 chatLLaMa diff --git a/chatLLaMa b/chatLLaMa new file mode 100755 index 0000000000000..bf34755d8f31f --- /dev/null +++ b/chatLLaMa @@ -0,0 +1,40 @@ +#!/bin/bash + +MODEL="./models/13B/ggml-model-q4_0.bin" +USER_NAME="${USER_NAME:-User}" +AI_NAME="${AI_NAME:-ChatLLaMa}" + +echo user_name: $USER_NAME +echo ai_name: $AI_NAME + +GEN_OPTIONS="--ctx_size 4096 --temp 0.7 --top_k 40 --top_p 0.5 --repeat_last_n 256 --repeat_penalty 1.17647" + +./main -m "$MODEL" -t 8 -n 1024 $GEN_OPTIONS --color -i -r "${USER_NAME}:" -p \ " +Text transcript of a never ending dialog, where ${USER_NAME} interacts with an AI assistant named ${AI_NAME}. +${AI_NAME} is helpful, kind, honest, good at writing and never fails to answer ${USER_NAME}’s requests immediately and with precision. +There are no annotations like (30 seconds passed...) or (to himself), just what ${USER_NAME} and ${AI_NAME} say alound to each other. +The dialog lasts for years, the entirety of it is shared below. It's 10000 pages long. +The transcript only includes text, it does not include markup like HTML and Markdown. + +$USER_NAME: Hello, $AI_NAME! +$AI_NAME: Hello $USER_NAME! How may I help you today? +$USER_NAME: What time is it? +$AI_NAME: It is $(date +%H:%M). +$USER_NAME: What year is it? +$AI_NAME: We are in $(date +%Y). +$USER_NAME: Please tell me the largest city in Europe. +$AI_NAME: The largest city in Europe is Moscow, the capital of Russia. +$USER_NAME: What can you tell me about Moscow? +$AI_NAME: Moscow, on the Moskva River in western Russia, is the nation’s cosmopolitan capital. In its historic core is the Kremlin, a complex that’s home to the president and tsarist treasures in the Armoury. Outside its walls is Red Square, Russia’s symbolic center. +$USER_NAME: What is a cat? +$AI_NAME: A cat is a domestic species of small carnivorous mammal. It is the only domesticated species in the family Felidae. +$USER_NAME: How do I pass command line arguments to a Node.js program? +$AI_NAME: The arguments are stored in process.argv. + + argv[0] is the path to the Node. js executable. + argv[1] is the path to the script file. + argv[2] is the first argument passed to the script. + argv[3] is the second argument passed to the script and so on. +$USER_NAME: Name a color. +$AI_NAME: Blue +$USER_NAME:" "$@" From 2aaf3799823f09d864f2676a2784321bfe018e76 Mon Sep 17 00:00:00 2001 From: Jean-Christophe Hoelt Date: Fri, 17 Mar 2023 08:47:12 +0200 Subject: [PATCH 2/5] Fix shellcheck errors and do some cleanup --- chatLLaMa | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/chatLLaMa b/chatLLaMa index bf34755d8f31f..ec4f9ac2c860a 100755 --- a/chatLLaMa +++ b/chatLLaMa @@ -4,14 +4,25 @@ MODEL="./models/13B/ggml-model-q4_0.bin" USER_NAME="${USER_NAME:-User}" AI_NAME="${AI_NAME:-ChatLLaMa}" -echo user_name: $USER_NAME -echo ai_name: $AI_NAME +# Adjust to the number of CPU cores you want to use. +N_THREAD="${N_THREAD:-8}" +# Number of tokens to predict (made it larger than default because we want a long interaction) +N_PREDICTS="${N_PREDICTS:-1024}" -GEN_OPTIONS="--ctx_size 4096 --temp 0.7 --top_k 40 --top_p 0.5 --repeat_last_n 256 --repeat_penalty 1.17647" +# Note: you can also override the generation options by specifying them on the command line: +# For example, override the context size by doing: ./chatLLaMa --ctx_size 2048 +GEN_OPTIONS="${GEN_OPTIONS:---ctx_size 4096 --temp 0.7 --top_k 40 --top_p 0.5 --repeat_last_n 256 --repeat_penalty 1.17647}" -./main -m "$MODEL" -t 8 -n 1024 $GEN_OPTIONS --color -i -r "${USER_NAME}:" -p \ " +# shellcheck disable=SC2086 # Intended splitting of GEN_OPTIONS +./main $GEN_OPTIONS \ + --model "$MODEL" \ + --threads "$N_THREAD" \ + --n_predict "$N_PREDICTS" \ + --color --interactive \ + --reverse-prompt "${USER_NAME}:" \ + --prompt " Text transcript of a never ending dialog, where ${USER_NAME} interacts with an AI assistant named ${AI_NAME}. -${AI_NAME} is helpful, kind, honest, good at writing and never fails to answer ${USER_NAME}’s requests immediately and with precision. +${AI_NAME} is helpful, kind, honest, friendly, good at writing and never fails to answer ${USER_NAME}’s requests immediately and with precision. There are no annotations like (30 seconds passed...) or (to himself), just what ${USER_NAME} and ${AI_NAME} say alound to each other. The dialog lasts for years, the entirety of it is shared below. It's 10000 pages long. The transcript only includes text, it does not include markup like HTML and Markdown. From b6bcd016b153f90cbede646822a4ca50f93904ba Mon Sep 17 00:00:00 2001 From: Jean-Christophe Hoelt Date: Sun, 19 Mar 2023 13:56:07 +0200 Subject: [PATCH 3/5] Move chatLLaMa script to `examples` directory --- chatLLaMa => examples/chatLLaMa | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) rename chatLLaMa => examples/chatLLaMa (97%) diff --git a/chatLLaMa b/examples/chatLLaMa similarity index 97% rename from chatLLaMa rename to examples/chatLLaMa index ec4f9ac2c860a..26c52deffd3f1 100755 --- a/chatLLaMa +++ b/examples/chatLLaMa @@ -1,5 +1,7 @@ #!/bin/bash +cd "$(dirname "$0")/.." || exit + MODEL="./models/13B/ggml-model-q4_0.bin" USER_NAME="${USER_NAME:-User}" AI_NAME="${AI_NAME:-ChatLLaMa}" @@ -22,7 +24,7 @@ GEN_OPTIONS="${GEN_OPTIONS:---ctx_size 4096 --temp 0.7 --top_k 40 --top_p 0.5 -- --reverse-prompt "${USER_NAME}:" \ --prompt " Text transcript of a never ending dialog, where ${USER_NAME} interacts with an AI assistant named ${AI_NAME}. -${AI_NAME} is helpful, kind, honest, friendly, good at writing and never fails to answer ${USER_NAME}’s requests immediately and with precision. +${AI_NAME} is helpful, kind, honest, friendly, good at writing and never fails to answer ${USER_NAME}’s requests immediately and with details and precision. There are no annotations like (30 seconds passed...) or (to himself), just what ${USER_NAME} and ${AI_NAME} say alound to each other. The dialog lasts for years, the entirety of it is shared below. It's 10000 pages long. The transcript only includes text, it does not include markup like HTML and Markdown. From b8c383a9b9a9c1501029764448f1e935cbabd1e8 Mon Sep 17 00:00:00 2001 From: Jean-Christophe Hoelt Date: Sun, 19 Mar 2023 14:11:05 +0200 Subject: [PATCH 4/5] Reduce chatLLaMa context size to 2048 Ref d7def1a7524f712e5ebb7cd02bab0f13aa56a7f9 --- examples/chatLLaMa | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/chatLLaMa b/examples/chatLLaMa index 26c52deffd3f1..86d349e427ed2 100755 --- a/examples/chatLLaMa +++ b/examples/chatLLaMa @@ -13,7 +13,7 @@ N_PREDICTS="${N_PREDICTS:-1024}" # Note: you can also override the generation options by specifying them on the command line: # For example, override the context size by doing: ./chatLLaMa --ctx_size 2048 -GEN_OPTIONS="${GEN_OPTIONS:---ctx_size 4096 --temp 0.7 --top_k 40 --top_p 0.5 --repeat_last_n 256 --repeat_penalty 1.17647}" +GEN_OPTIONS="${GEN_OPTIONS:---ctx_size 2048 --temp 0.7 --top_k 40 --top_p 0.5 --repeat_last_n 256 --repeat_penalty 1.17647}" # shellcheck disable=SC2086 # Intended splitting of GEN_OPTIONS ./main $GEN_OPTIONS \ From 1b8f8ad0bae57709518ec3f2ccc71320494f1367 Mon Sep 17 00:00:00 2001 From: Jean-Christophe Hoelt Date: Sun, 19 Mar 2023 18:27:54 +0200 Subject: [PATCH 5/5] Include n_predict to 2048 in examples/chatLLaMa --- examples/chatLLaMa | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/chatLLaMa b/examples/chatLLaMa index 86d349e427ed2..97c48ac872349 100755 --- a/examples/chatLLaMa +++ b/examples/chatLLaMa @@ -2,17 +2,17 @@ cd "$(dirname "$0")/.." || exit -MODEL="./models/13B/ggml-model-q4_0.bin" +MODEL="${MODEL:-./models/13B/ggml-model-q4_0.bin}" USER_NAME="${USER_NAME:-User}" AI_NAME="${AI_NAME:-ChatLLaMa}" # Adjust to the number of CPU cores you want to use. N_THREAD="${N_THREAD:-8}" # Number of tokens to predict (made it larger than default because we want a long interaction) -N_PREDICTS="${N_PREDICTS:-1024}" +N_PREDICTS="${N_PREDICTS:-2048}" # Note: you can also override the generation options by specifying them on the command line: -# For example, override the context size by doing: ./chatLLaMa --ctx_size 2048 +# For example, override the context size by doing: ./chatLLaMa --ctx_size 1024 GEN_OPTIONS="${GEN_OPTIONS:---ctx_size 2048 --temp 0.7 --top_k 40 --top_p 0.5 --repeat_last_n 256 --repeat_penalty 1.17647}" # shellcheck disable=SC2086 # Intended splitting of GEN_OPTIONS