From d75ceb168bbb741b5a8c52ec05bdebbe65bad964 Mon Sep 17 00:00:00 2001
From: brian khuu <mofosyne@gmail.com>
Date: Tue, 14 May 2024 21:53:33 +1000
Subject: [PATCH 1/6] run-single-test.sh: added a single test function script
 and fix debug-test.sh to be more robust

---
 docs/debugging-tests.md    |  35 ++++++++++
 scripts/debug-test.sh      |  26 ++++++--
 scripts/run-single-test.sh | 128 +++++++++++++++++++++++++++++++++++++
 3 files changed, 182 insertions(+), 7 deletions(-)
 create mode 100755 scripts/run-single-test.sh

diff --git a/docs/debugging-tests.md b/docs/debugging-tests.md
index 51a125e191d84..57d0148f1af44 100644
--- a/docs/debugging-tests.md
+++ b/docs/debugging-tests.md
@@ -1,5 +1,40 @@
 # Debugging Tests Tips
 
+## How to run & execute a specific test without anything else to keep the feedback loop short?
+
+There is a script called run-single-test.sh in the scripts folder whose parameter takes a REGEX and an optional test number.
+
+For example, running the following command will output an interactive list from which you can select a test. It takes this form:
+
+`run-single-test.sh [OPTION]... <test_regex> <test_number>`
+
+It will then build & run in the debugger for you.
+
+```bash
+./scripts/run-single-test.sh test-tokenizer
+```
+
+An example of a single test output is shown below. You will get either a green TEST PASS or a red TEST FAIL if a particular test is working or not. This shorter feedback loop will hopefully make it easier for you to figure out the problem you are trying to solve.
+
+```bash
+$ ./scripts/run-single-test.sh test 24
+~/gitextern/llama.cpp ~/gitextern/llama.cpp
+
+... prepping cmake environment ...
+... building test binaries ...
+... running test ...
+
+Ran Test #24: test-eval-callback
+Command: /home/mofosyne/gitextern/llama.cpp/build-ci-debug/bin/eval-callback "--hf-repo" "ggml-org/models" "--hf-file" "tinyllamas/stories260K.gguf" "--model" "stories260K.gguf" "--prompt" "hello" "--seed" "42" "-ngl" "0"
+TEST PASS
+```
+
+For further reference use `run-single-test.sh -h` to print help.
+
+### How does the script work?
+
+This is similar to `debug-test.sh` so you can follow the similar guide in this page for similar process. Just run the command directly rather than though gdb.
+
 ## How to run & debug a specific test without anything else to keep the feedback loop short?
 
 There is a script called debug-test.sh in the scripts folder whose parameter takes a REGEX and an optional test number.
diff --git a/scripts/debug-test.sh b/scripts/debug-test.sh
index 231a23d694873..806fcb7e2f687 100755
--- a/scripts/debug-test.sh
+++ b/scripts/debug-test.sh
@@ -27,6 +27,7 @@ fi
 function select_test() {
     test_suite=${1:-test}
     test_number=${2:-}
+    repo_root=${3:-}
 
     # Sanity Check If Tests Is Detected
     printf "\n\nGathering tests that fit REGEX: ${test_suite} ...\n"
@@ -54,7 +55,7 @@ function select_test() {
         printf "\nRun test#? "
         read test_number
     else
-        printf "\nUser Already Requested #${test_number}"
+        printf "\nUser Already Requested #${test_number}\n"
     fi
 
     # Start GDB with the requested test binary and arguments
@@ -75,9 +76,20 @@ function select_test() {
         args+=($(echo $x | sed -e 's/.*\/..\//..\//'))
     done
 
+    # Print Command
+    red=$(tput setaf 1)
+    green=$(tput setaf 2)
+    yellow=$(tput setaf 3)
+    blue=$(tput setaf 4)
+    normal=$(tput sgr0)
+    printf "${blue}Ran Test #${test_number}: ${tests[test_number]}${normal}\n"
+    printf "${yellow}GDB args: ${gdb_args[test_number]}${normal}\n"
+    printf "${yellow}GDB args @: ${args[@]}${normal}\n"
+
     # Execute debugger
-    echo "gdb args: ${args[@]}"
+    pushd "$repo_root" || exit 1
     gdb --args ${args[@]}
+    popd > /dev/null || exit 1
 }
 
 # Step 0: Check the args
@@ -105,13 +117,13 @@ pushd "$repo_root" || exit 1
 rm -rf "$build_dir" && mkdir "$build_dir" || exit 1
 
 # Step 2: Setup Build Environment and Compile Test Binaries
-cmake -B "./$build_dir" -DCMAKE_BUILD_TYPE=Debug -DLLAMA_CUDA=1 -DLLAMA_FATAL_WARNINGS=ON || exit 1
+# Note: test-eval-callback requires -DLLAMA_CURL
+cmake -B "./$build_dir" -DCMAKE_BUILD_TYPE=Debug -DLLAMA_CUDA=1 -DLLAMA_FATAL_WARNINGS=ON -DLLAMA_CURL=1 || exit 1
 pushd "$build_dir" && make -j || exit 1
 
 # Step 3: Debug the Test
-select_test "$test_suite" "$test_number"
+select_test "$test_suite" "$test_number" "$repo_root"
 
 # Step 4: Return to the directory from which the user ran the command.
-popd || exit 1
-popd || exit 1
-popd || exit 1
+popd > /dev/null || exit 1
+popd > /dev/null || exit 1
diff --git a/scripts/run-single-test.sh b/scripts/run-single-test.sh
new file mode 100755
index 0000000000000..08d20515e9381
--- /dev/null
+++ b/scripts/run-single-test.sh
@@ -0,0 +1,128 @@
+#!/bin/bash
+test_suite=${1:-}
+test_number=${2:-}
+
+PROG=${0##*/}
+build_dir="build-ci-debug"
+
+if [ x"$1" = x"-h" ] || [ x"$1" = x"--help" ]; then
+    echo "Usage: $PROG [OPTION]... <test_regex> (test_number)"
+    echo "Run a specific ctest program."
+    echo
+    echo "Options:"
+    echo "  -h, --help       Display this help and exit"
+    echo
+    echo "Arguments:"
+    echo "  <test_regex>     (Mandatory) Supply one regex to the script to filter tests"
+    echo "  (test_number)    (Optional) Test number to run a specific test"
+    echo
+    echo "Example:"
+    echo "  $PROG test-tokenizer"
+    echo "  $PROG test-tokenizer 3"
+    echo
+    exit 0
+fi
+
+# Function to select and debug a test
+function select_test() {
+    test_suite=${1:-test}
+    test_number=${2:-}
+    repo_root=${3:-}
+
+    # Color
+    red=$(tput setaf 1)
+    green=$(tput setaf 2)
+    yellow=$(tput setaf 3)
+    blue=$(tput setaf 4)
+    normal=$(tput sgr0)
+
+    # Sanity Check If Tests Is Detected
+    printf "\n\nGathering tests that fit REGEX: ${test_suite} ...\n"
+    tests=($(ctest -R ${test_suite} -V -N | grep -E " +Test +#[0-9]+*" | cut -d':' -f2 | awk '{$1=$1};1'))
+    if [ ${#tests[@]} -eq 0 ]
+    then
+        echo "No tests avaliable... check your compliation process..."
+        echo "Exiting."
+        exit 1
+    fi
+
+    if [ -z $test_number ]
+    then
+        # List out avaliable tests
+        printf "Which test would you like to debug?\n"
+        id=0
+        for s in "${tests[@]}"
+        do
+            echo "Test# ${id}"
+            echo "  $s"
+            ((id++))
+        done
+
+        # Prompt user which test they wanted to run
+        printf "\nRun test#? "
+        read test_number
+    else
+        printf "\nUser Already Requested #${test_number}\n"
+    fi
+
+    # Find requested test binary and arguments
+    # Change IFS (Internal Field Separator)
+    sIFS=$IFS
+    IFS=$'\n'
+
+    # Get test args
+    test_args=($(ctest -R ${test_suite} -V -N | grep "Test command" | cut -d':' -f3 | awk '{$1=$1};1' ))
+    IFS=$sIFS
+
+    # Execute Test
+    pushd "$repo_root" || exit 1
+    printf "${blue}Running Test #${test_number}: ${tests[test_number]}${normal}\n"
+    eval "${test_args[test_number]}"
+    exit_code=$?
+    popd
+
+    # Print Result
+    printf "${blue}Ran Test #${test_number}: ${tests[test_number]}${normal}\n"
+    printf "${yellow}Command: ${test_args[test_number]}${normal}\n"
+    if [ $exit_code -eq 0 ]; then
+        printf "${green}TEST PASS${normal}\n"
+    else
+        printf "${red}TEST FAIL${normal}\n"
+    fi
+}
+
+# Step 0: Check the args
+if [ -z "$test_suite" ]
+then
+    echo "Usage: $PROG [OPTION]... <test_regex> (test_number)"
+    echo "Supply one regex to the script to filter tests,"
+    echo "and optionally a test number to run a specific test."
+    echo "Use --help flag for full instructions"
+    exit 1
+fi
+
+# Step 1: Reset and Setup folder context
+## Sanity check that we are actually in a git repo
+repo_root=$(git rev-parse --show-toplevel)
+if [ ! -d "$repo_root" ]; then
+    echo "Error: Not in a Git repository."
+    exit 1
+fi
+
+## Reset folder to root context of git repo
+pushd "$repo_root" || exit 1
+
+## Create and enter build directory
+rm -rf "$build_dir" && mkdir "$build_dir" || exit 1
+
+# Step 2: Setup Build Environment and Compile Test Binaries
+# Note: test-eval-callback requires -DLLAMA_CURL=1
+cmake -B "./$build_dir" -DCMAKE_BUILD_TYPE=Debug -DLLAMA_CUDA=1 -DLLAMA_FATAL_WARNINGS=ON -DLLAMA_CURL=1 || exit 1
+pushd "$build_dir" && make -j || exit 1
+
+# Step 3: Debug the Test
+select_test "$test_suite" "$test_number" "$repo_root"
+
+# Step 4: Return to the directory from which the user ran the command.
+popd > /dev/null || exit 1
+popd > /dev/null || exit 1

From f274de538600451491b337edab5679dd51a9430f Mon Sep 17 00:00:00 2001
From: brian khuu <mofosyne@gmail.com>
Date: Tue, 14 May 2024 23:09:25 +1000
Subject: [PATCH 2/6] debug-test.sh: combined execute and gdb test mode via -g
 flag

---
 docs/debugging-tests.md    |  45 ++++---------
 scripts/debug-test.sh      | 128 ++++++++++++++++++++++++++-----------
 scripts/run-single-test.sh | 128 -------------------------------------
 3 files changed, 101 insertions(+), 200 deletions(-)
 delete mode 100755 scripts/run-single-test.sh

diff --git a/docs/debugging-tests.md b/docs/debugging-tests.md
index 57d0148f1af44..984da80e1b746 100644
--- a/docs/debugging-tests.md
+++ b/docs/debugging-tests.md
@@ -1,55 +1,34 @@
 # Debugging Tests Tips
 
-## How to run & execute a specific test without anything else to keep the feedback loop short?
+## How to run & execute or debug a specific test without anything else to keep the feedback loop short?
 
-There is a script called run-single-test.sh in the scripts folder whose parameter takes a REGEX and an optional test number.
+There is a script called debug-test.sh in the scripts folder whose parameter takes a REGEX and an optional test number.
 
 For example, running the following command will output an interactive list from which you can select a test. It takes this form:
 
-`run-single-test.sh [OPTION]... <test_regex> <test_number>`
+`debug-test.sh [OPTION]... <test_regex> <test_number>`
 
 It will then build & run in the debugger for you.
 
+To just execute a test and get back a PASS or FAIL message run:
+
 ```bash
-./scripts/run-single-test.sh test-tokenizer
+./scripts/debug-test.sh test-tokenizer
 ```
 
-An example of a single test output is shown below. You will get either a green TEST PASS or a red TEST FAIL if a particular test is working or not. This shorter feedback loop will hopefully make it easier for you to figure out the problem you are trying to solve.
+To test in GDB use the `-g` flag to enable gdb test mode.
 
 ```bash
-$ ./scripts/run-single-test.sh test 24
-~/gitextern/llama.cpp ~/gitextern/llama.cpp
-
-... prepping cmake environment ...
-... building test binaries ...
-... running test ...
+./scripts/debug-test.sh -g test-tokenizer
 
-Ran Test #24: test-eval-callback
-Command: /home/mofosyne/gitextern/llama.cpp/build-ci-debug/bin/eval-callback "--hf-repo" "ggml-org/models" "--hf-file" "tinyllamas/stories260K.gguf" "--model" "stories260K.gguf" "--prompt" "hello" "--seed" "42" "-ngl" "0"
-TEST PASS
+# Once in the debugger, i.e. at the chevrons prompt, setting a breakpoint could be as follows:
+>>> b main
 ```
 
-For further reference use `run-single-test.sh -h` to print help.
-
-### How does the script work?
-
-This is similar to `debug-test.sh` so you can follow the similar guide in this page for similar process. Just run the command directly rather than though gdb.
-
-## How to run & debug a specific test without anything else to keep the feedback loop short?
-
-There is a script called debug-test.sh in the scripts folder whose parameter takes a REGEX and an optional test number.
-
-For example, running the following command will output an interactive list from which you can select a test. It takes this form:
-
-`debug-test.sh [OPTION]... <test_regex> <test_number>`
-
-It will then build & run in the debugger for you.
+To speed up the testing loop, if you know your test number you can just run it similar to below:
 
 ```bash
-./scripts/debug-test.sh test-tokenizer
-
-# Once in the debugger, i.e. at the chevrons prompt, setting a breakpoint could be as follows:
->>> b main
+./scripts/debug-test.sh test 23
 ```
 
 For further reference use `debug-test.sh -h` to print help.
diff --git a/scripts/debug-test.sh b/scripts/debug-test.sh
index 806fcb7e2f687..497e44e57812c 100755
--- a/scripts/debug-test.sh
+++ b/scripts/debug-test.sh
@@ -5,24 +5,62 @@ test_number=${2:-}
 PROG=${0##*/}
 build_dir="build-ci-debug"
 
+# Print Color Commands
+red=$(tput setaf 1)
+green=$(tput setaf 2)
+yellow=$(tput setaf 3)
+blue=$(tput setaf 4)
+normal=$(tput sgr0)
+
+print_full_help() {
+  cat << EOF
+Usage: $PROG [OPTION]... <test_regex> (test_number)
+Debug specific ctest program.
+
+Options:
+  -h, --help            display this help and exit
+  -g                    run in gdb mode
+
+Arguments:
+  <test_regex>     (Mandatory) Supply one regex to the script to filter tests
+  (test_number)    (Optional) Test number to run a specific test
+
+Example:
+  $PROG test-tokenizer
+  $PROG test-tokenizer 3
+EOF
+}
+
+abort() {
+  echo "Error: $1" >&2
+  cat << EOF >&2
+Usage: $PROG [OPTION]... <test_regex> (test_number)
+Debug specific ctest program.
+Refer to --help for full instructions.
+EOF
+  exit 1
+}
+
 if [ x"$1" = x"-h" ] || [ x"$1" = x"--help" ]; then
-    echo "Usage: $PROG [OPTION]... <test_regex> (test_number)"
-    echo "Debug specific ctest program."
-    echo
-    echo "Options:"
-    echo "  -h, --help       Display this help and exit"
-    echo
-    echo "Arguments:"
-    echo "  <test_regex>     (Mandatory) Supply one regex to the script to filter tests"
-    echo "  (test_number)    (Optional) Test number to run a specific test"
-    echo
-    echo "Example:"
-    echo "  $PROG test-tokenizer"
-    echo "  $PROG test-tokenizer 3"
-    echo
-    exit 0
+  print_full_help >&2
+  exit 0
 fi
 
+# Parse command-line options
+gdb_mode=false
+while getopts "hg" opt; do
+    case $opt in
+        h)
+            print_full_help >&2
+            exit 0
+            ;;
+        g)
+            gdb_mode=true
+            echo "gdb_mode Mode Enabled"
+            ;;
+    esac
+done
+
 # Function to select and debug a test
 function select_test() {
     test_suite=${1:-test}
@@ -65,31 +103,43 @@ function select_test() {
     IFS=$'\n'
 
     # Get test args
-    gdb_args=($(ctest -R ${test_suite} -V -N | grep "Test command" | cut -d':' -f3 | awk '{$1=$1};1' ))
+    test_args=($(ctest -R ${test_suite} -V -N | grep "Test command" | cut -d':' -f3 | awk '{$1=$1};1' ))
     IFS=$sIFS
-    printf "Debug arguments: ${gdb_args[test_number]}\n\n"
-
-    # Expand paths if needed
-    args=()
-    for x in $(echo ${gdb_args[test_number]} | sed -e 's/"\/\<//' -e 's/\>"//')
-    do
-        args+=($(echo $x | sed -e 's/.*\/..\//..\//'))
-    done
-
-    # Print Command
-    red=$(tput setaf 1)
-    green=$(tput setaf 2)
-    yellow=$(tput setaf 3)
-    blue=$(tput setaf 4)
-    normal=$(tput sgr0)
-    printf "${blue}Ran Test #${test_number}: ${tests[test_number]}${normal}\n"
-    printf "${yellow}GDB args: ${gdb_args[test_number]}${normal}\n"
-    printf "${yellow}GDB args @: ${args[@]}${normal}\n"
-
-    # Execute debugger
-    pushd "$repo_root" || exit 1
-    gdb --args ${args[@]}
-    popd > /dev/null || exit 1
+
+    printf "${blue}Running Test #${test_number}: ${tests[test_number]}${normal}\n"
+    printf "${blue}test_args[test_number]: ${test_args[test_number]}${normal}\n"
+
+    if [ "$gdb_mode" = "true" ]; then
+        # Expand paths if needed
+        gdb_args=()
+        for x in $(echo ${test_args[test_number]} | sed -e 's/"\/\<//' -e 's/\>"//')
+        do
+            gdb_args+=($(echo $x | sed -e 's/.*\/..\//..\//'))
+        done
+
+        printf "${blue}gdb_args @: ${gdb_args[@]}${normal}\n"
+
+        # Execute debugger
+        pushd "$repo_root" || exit 1
+        gdb --args ${gdb_args[@]}
+        popd > /dev/null || exit 1
+    else
+
+        # Execute Test
+        pushd "$repo_root" || exit 1
+        eval "${test_args[test_number]}"
+        exit_code=$?
+        popd > /dev/null || exit 1
+
+        # Print Result
+        printf "${blue}Ran Test #${test_number}: ${tests[test_number]}${normal}\n"
+        printf "${yellow}Command: ${test_args[test_number]}${normal}\n"
+        if [ $exit_code -eq 0 ]; then
+            printf "${green}TEST PASS${normal}\n"
+        else
+            printf "${red}TEST FAIL${normal}\n"
+        fi
+    fi
 }
 
 # Step 0: Check the args
diff --git a/scripts/run-single-test.sh b/scripts/run-single-test.sh
deleted file mode 100755
index 08d20515e9381..0000000000000
--- a/scripts/run-single-test.sh
+++ /dev/null
@@ -1,128 +0,0 @@
-#!/bin/bash
-test_suite=${1:-}
-test_number=${2:-}
-
-PROG=${0##*/}
-build_dir="build-ci-debug"
-
-if [ x"$1" = x"-h" ] || [ x"$1" = x"--help" ]; then
-    echo "Usage: $PROG [OPTION]... <test_regex> (test_number)"
-    echo "Run a specific ctest program."
-    echo
-    echo "Options:"
-    echo "  -h, --help       Display this help and exit"
-    echo
-    echo "Arguments:"
-    echo "  <test_regex>     (Mandatory) Supply one regex to the script to filter tests"
-    echo "  (test_number)    (Optional) Test number to run a specific test"
-    echo
-    echo "Example:"
-    echo "  $PROG test-tokenizer"
-    echo "  $PROG test-tokenizer 3"
-    echo
-    exit 0
-fi
-
-# Function to select and debug a test
-function select_test() {
-    test_suite=${1:-test}
-    test_number=${2:-}
-    repo_root=${3:-}
-
-    # Color
-    red=$(tput setaf 1)
-    green=$(tput setaf 2)
-    yellow=$(tput setaf 3)
-    blue=$(tput setaf 4)
-    normal=$(tput sgr0)
-
-    # Sanity Check If Tests Is Detected
-    printf "\n\nGathering tests that fit REGEX: ${test_suite} ...\n"
-    tests=($(ctest -R ${test_suite} -V -N | grep -E " +Test +#[0-9]+*" | cut -d':' -f2 | awk '{$1=$1};1'))
-    if [ ${#tests[@]} -eq 0 ]
-    then
-        echo "No tests avaliable... check your compliation process..."
-        echo "Exiting."
-        exit 1
-    fi
-
-    if [ -z $test_number ]
-    then
-        # List out avaliable tests
-        printf "Which test would you like to debug?\n"
-        id=0
-        for s in "${tests[@]}"
-        do
-            echo "Test# ${id}"
-            echo "  $s"
-            ((id++))
-        done
-
-        # Prompt user which test they wanted to run
-        printf "\nRun test#? "
-        read test_number
-    else
-        printf "\nUser Already Requested #${test_number}\n"
-    fi
-
-    # Find requested test binary and arguments
-    # Change IFS (Internal Field Separator)
-    sIFS=$IFS
-    IFS=$'\n'
-
-    # Get test args
-    test_args=($(ctest -R ${test_suite} -V -N | grep "Test command" | cut -d':' -f3 | awk '{$1=$1};1' ))
-    IFS=$sIFS
-
-    # Execute Test
-    pushd "$repo_root" || exit 1
-    printf "${blue}Running Test #${test_number}: ${tests[test_number]}${normal}\n"
-    eval "${test_args[test_number]}"
-    exit_code=$?
-    popd
-
-    # Print Result
-    printf "${blue}Ran Test #${test_number}: ${tests[test_number]}${normal}\n"
-    printf "${yellow}Command: ${test_args[test_number]}${normal}\n"
-    if [ $exit_code -eq 0 ]; then
-        printf "${green}TEST PASS${normal}\n"
-    else
-        printf "${red}TEST FAIL${normal}\n"
-    fi
-}
-
-# Step 0: Check the args
-if [ -z "$test_suite" ]
-then
-    echo "Usage: $PROG [OPTION]... <test_regex> (test_number)"
-    echo "Supply one regex to the script to filter tests,"
-    echo "and optionally a test number to run a specific test."
-    echo "Use --help flag for full instructions"
-    exit 1
-fi
-
-# Step 1: Reset and Setup folder context
-## Sanity check that we are actually in a git repo
-repo_root=$(git rev-parse --show-toplevel)
-if [ ! -d "$repo_root" ]; then
-    echo "Error: Not in a Git repository."
-    exit 1
-fi
-
-## Reset folder to root context of git repo
-pushd "$repo_root" || exit 1
-
-## Create and enter build directory
-rm -rf "$build_dir" && mkdir "$build_dir" || exit 1
-
-# Step 2: Setup Build Environment and Compile Test Binaries
-# Note: test-eval-callback requires -DLLAMA_CURL=1
-cmake -B "./$build_dir" -DCMAKE_BUILD_TYPE=Debug -DLLAMA_CUDA=1 -DLLAMA_FATAL_WARNINGS=ON -DLLAMA_CURL=1 || exit 1
-pushd "$build_dir" && make -j || exit 1
-
-# Step 3: Debug the Test
-select_test "$test_suite" "$test_number" "$repo_root"
-
-# Step 4: Return to the directory from which the user ran the command.
-popd > /dev/null || exit 1
-popd > /dev/null || exit 1

From d4239194cb3a89545e4740ae43a309524dda6c88 Mon Sep 17 00:00:00 2001
From: brian khuu <mofosyne@gmail.com>
Date: Tue, 14 May 2024 23:18:57 +1000
Subject: [PATCH 3/6] debug-test.sh: refactor

---
 scripts/debug-test.sh | 33 +++++++++++++++++----------------
 1 file changed, 17 insertions(+), 16 deletions(-)

diff --git a/scripts/debug-test.sh b/scripts/debug-test.sh
index 497e44e57812c..20c2f43a69e58 100755
--- a/scripts/debug-test.sh
+++ b/scripts/debug-test.sh
@@ -1,6 +1,4 @@
 #!/bin/bash
-test_suite=${1:-}
-test_number=${2:-}
 
 PROG=${0##*/}
 build_dir="build-ci-debug"
@@ -48,12 +46,8 @@ fi
 
 # Parse command-line options
 gdb_mode=false
-while getopts "hg" opt; do
+while getopts "g" opt; do
     case $opt in
-        h)
-            print_full_help >&2
-            exit 0
-            ;;
         g)
             gdb_mode=true
             echo "gdb_mode Mode Enabled"
@@ -61,6 +55,22 @@ while getopts "hg" opt; do
     esac
 done
 
+# Shift the option parameters
+shift $((OPTIND - 1))
+
+# Step 0: Check the args
+if [ -z "${1}" ]; then
+    echo "Usage: $PROG [OPTION]... <test_regex> (test_number)"
+    echo "Supply one regex to the script to filter tests,"
+    echo "and optionally a test number to run a specific test."
+    echo "Use --help flag for full instructions"
+    exit 1
+else
+    test_suite=${1:-}
+fi
+
+test_number=${2:-}
+
 # Function to select and debug a test
 function select_test() {
     test_suite=${1:-test}
@@ -142,15 +152,6 @@ function select_test() {
     fi
 }
 
-# Step 0: Check the args
-if [ -z "$test_suite" ]
-then
-    echo "Usage: $PROG [OPTION]... <test_regex> (test_number)"
-    echo "Supply one regex to the script to filter tests,"
-    echo "and optionally a test number to run a specific test."
-    echo "Use --help flag for full instructions"
-    exit 1
-fi
 
 # Step 1: Reset and Setup folder context
 ## Sanity check that we are actually in a git repo

From edec5ba3d7a6fdae73875fe9b2c5aa8175f87e96 Mon Sep 17 00:00:00 2001
From: brian khuu <mofosyne@gmail.com>
Date: Wed, 15 May 2024 02:37:07 +1000
Subject: [PATCH 4/6] debug-test: refactor for clarity

---
 docs/debugging-tests.md |   8 +-
 scripts/debug-test.sh   | 212 +++++++++++++++++++++-------------------
 2 files changed, 118 insertions(+), 102 deletions(-)

diff --git a/docs/debugging-tests.md b/docs/debugging-tests.md
index 984da80e1b746..18407f688f9db 100644
--- a/docs/debugging-tests.md
+++ b/docs/debugging-tests.md
@@ -55,7 +55,7 @@ cmake -DCMAKE_BUILD_TYPE=Debug -DLLAMA_CUDA=1 -DLLAMA_FATAL_WARNINGS=ON ..
 make -j
 ```
 
-#### Step 3.1: Identify Test Command for Debugging
+#### Step 3: Find all tests available that matches REGEX
 
 The output of this command will give you the command & arguments needed to run GDB.
 
@@ -83,11 +83,13 @@ Labels: main
 ...
 ```
 
-So for test #1 we can tell these two pieces of relevant information:
+#### Step 4: Identify Test Command for Debugging
+
+So for test #1 above we can tell these two pieces of relevant information:
 * Test Binary: `~/llama.cpp/build-ci-debug/bin/test-tokenizer-0`
 * Test GGUF Model: `~/llama.cpp/tests/../models/ggml-vocab-llama-spm.gguf`
 
-#### Step 3.2: Run GDB on test command
+#### Step 5: Run GDB on test command
 
 Based on the ctest 'test command' report above we can then run a gdb session via this command below:
 
diff --git a/scripts/debug-test.sh b/scripts/debug-test.sh
index 20c2f43a69e58..8c6fa24d51dd6 100755
--- a/scripts/debug-test.sh
+++ b/scripts/debug-test.sh
@@ -8,6 +8,8 @@ red=$(tput setaf 1)
 green=$(tput setaf 2)
 yellow=$(tput setaf 3)
 blue=$(tput setaf 4)
+magenta=$(tput setaf 5)
+cyan=$(tput setaf 6)
 normal=$(tput sgr0)
 
 print_full_help() {
@@ -39,6 +41,15 @@ EOF
   exit 1
 }
 
+check_dependency() {
+  command -v "$1" >/dev/null 2>&1 || {
+    abort "$1 is required but not found. Please install it and try again."
+  }
+}
+
+check_dependency ctest
+check_dependency cmake
+
 if [ x"$1" = x"-h" ] || [ x"$1" = x"--help" ]; then
   print_full_help >&2
   exit 0
@@ -60,121 +71,124 @@ shift $((OPTIND - 1))
 
 # Step 0: Check the args
 if [ -z "${1}" ]; then
-    echo "Usage: $PROG [OPTION]... <test_regex> (test_number)"
-    echo "Supply one regex to the script to filter tests,"
-    echo "and optionally a test number to run a specific test."
-    echo "Use --help flag for full instructions"
-    exit 1
+    abort "Test regex is required"
 else
     test_suite=${1:-}
 fi
 
 test_number=${2:-}
 
-# Function to select and debug a test
-function select_test() {
-    test_suite=${1:-test}
-    test_number=${2:-}
-    repo_root=${3:-}
-
-    # Sanity Check If Tests Is Detected
-    printf "\n\nGathering tests that fit REGEX: ${test_suite} ...\n"
-    tests=($(ctest -R ${test_suite} -V -N | grep -E " +Test +#[0-9]+*" | cut -d':' -f2 | awk '{$1=$1};1'))
-    if [ ${#tests[@]} -eq 0 ]
-    then
-        echo "No tests avaliable... check your compliation process..."
-        echo "Exiting."
-        exit 1
-    fi
-
-    if [ -z $test_number ]
-    then
-        # List out avaliable tests
-        printf "Which test would you like to debug?\n"
-        id=0
-        for s in "${tests[@]}"
-        do
-            echo "Test# ${id}"
-            echo "  $s"
-            ((id++))
-        done
-
-        # Prompt user which test they wanted to run
-        printf "\nRun test#? "
-        read test_number
-    else
-        printf "\nUser Already Requested #${test_number}\n"
-    fi
-
-    # Start GDB with the requested test binary and arguments
-    printf "Debugging(GDB) test: ${tests[test_number]}\n"
-    # Change IFS (Internal Field Separator)
-    sIFS=$IFS
-    IFS=$'\n'
-
-    # Get test args
-    test_args=($(ctest -R ${test_suite} -V -N | grep "Test command" | cut -d':' -f3 | awk '{$1=$1};1' ))
-    IFS=$sIFS
-
-    printf "${blue}Running Test #${test_number}: ${tests[test_number]}${normal}\n"
-    printf "${blue}test_args[test_number]: ${test_args[test_number]}${normal}\n"
-
-    if [ "$gdb_mode" = "true" ]; then
-        # Expand paths if needed
-        gdb_args=()
-        for x in $(echo ${test_args[test_number]} | sed -e 's/"\/\<//' -e 's/\>"//')
-        do
-            gdb_args+=($(echo $x | sed -e 's/.*\/..\//..\//'))
-        done
-
-        printf "${blue}gdb_args @: ${gdb_args[@]}${normal}\n"
-
-        # Execute debugger
-        pushd "$repo_root" || exit 1
-        gdb --args ${gdb_args[@]}
-        popd > /dev/null || exit 1
-    else
-
-        # Execute Test
-        pushd "$repo_root" || exit 1
-        eval "${test_args[test_number]}"
-        exit_code=$?
-        popd > /dev/null || exit 1
-
-        # Print Result
-        printf "${blue}Ran Test #${test_number}: ${tests[test_number]}${normal}\n"
-        printf "${yellow}Command: ${test_args[test_number]}${normal}\n"
-        if [ $exit_code -eq 0 ]; then
-            printf "${green}TEST PASS${normal}\n"
-        else
-            printf "${red}TEST FAIL${normal}\n"
-        fi
-    fi
-}
-
-
 # Step 1: Reset and Setup folder context
+########################################
+
 ## Sanity check that we are actually in a git repo
 repo_root=$(git rev-parse --show-toplevel)
 if [ ! -d "$repo_root" ]; then
-    echo "Error: Not in a Git repository."
-    exit 1
+    abort "Not in a Git repository."
 fi
 
-## Reset folder to root context of git repo
-pushd "$repo_root" || exit 1
-
-## Create and enter build directory
-rm -rf "$build_dir" && mkdir "$build_dir" || exit 1
+## Reset folder to root context of git repo and Create and enter build directory
+pushd "$repo_root"
+rm -rf "$build_dir" && mkdir "$build_dir" || abort "Failed to make $build_dir"
 
 # Step 2: Setup Build Environment and Compile Test Binaries
+###########################################################
+
 # Note: test-eval-callback requires -DLLAMA_CURL
-cmake -B "./$build_dir" -DCMAKE_BUILD_TYPE=Debug -DLLAMA_CUDA=1 -DLLAMA_FATAL_WARNINGS=ON -DLLAMA_CURL=1 || exit 1
-pushd "$build_dir" && make -j || exit 1
+cmake -B "./$build_dir" -DCMAKE_BUILD_TYPE=Debug -DLLAMA_CUDA=1 -DLLAMA_FATAL_WARNINGS=ON -DLLAMA_CURL=1 || abort "Failed to build enviroment"
+pushd "$build_dir"
+make -j || abort "Failed to compile"
+popd > /dev/null || exit 1
 
-# Step 3: Debug the Test
-select_test "$test_suite" "$test_number" "$repo_root"
+# Step 3: Find all tests available that matches REGEX
+####################################################
+
+# Ctest Gather Tests
+# `-R test-tokenizer` : looks for all the test files named `test-tokenizer*` (R=Regex)
+# `-N` : "show-only" disables test execution & shows test commands that you can feed to GDB.
+# `-V` : Verbose Mode
+printf "\n\nGathering tests that fit REGEX: ${test_suite} ...\n"
+pushd "$build_dir"
+tests=($(ctest -R ${test_suite} -V -N | grep -E " +Test +#[0-9]+*" | cut -d':' -f2 | awk '{$1=$1};1'))
+if [ ${#tests[@]} -eq 0 ]; then
+    abort "No tests avaliable... check your compliation process..."
+fi
+popd > /dev/null || exit 1
+
+# Step 4: Identify Test Command for Debugging
+#############################################
+
+# Select test number
+if [ -z $test_number ]; then
+    # List out avaliable tests
+    printf "Which test would you like to debug?\n"
+    id=0
+    for s in "${tests[@]}"
+    do
+        echo "Test# ${id}"
+        echo "  $s"
+        ((id++))
+    done
+
+    # Prompt user which test they wanted to run
+    printf "\nRun test#? "
+    read test_number
 
-# Step 4: Return to the directory from which the user ran the command.
+else
+    printf "\nUser Already Requested #${test_number}\n"
+
+fi
+
+# Grab all tests commands
+pushd "$build_dir"
+sIFS=$IFS # Save Initial IFS (Internal Field Separator)
+IFS=$'\n' # Change IFS (Internal Field Separator) (So we split ctest output by newline rather than by spaces)
+test_args=($(ctest -R ${test_suite} -V -N | grep "Test command" | cut -d':' -f3 | awk '{$1=$1};1' )) # Get test args
+IFS=$sIFS # Reset IFS (Internal Field Separator)
 popd > /dev/null || exit 1
+
+# Grab specific test command
+single_test_name="${tests[test_number]}"
+single_test_command="${test_args[test_number]}"
+
+# Step 5: Execute or GDB Debug
+##############################
+printf "${magenta}Running Test #${test_number}: ${single_test_name}${normal}\n"
+printf "${cyan}single_test_command: ${single_test_command}${normal}\n"
+
+if [ "$gdb_mode" = "true" ]; then
+    printf "${blue}Replacing single test command string that may have /../ in them with the relative path, i.e. path/../path -> ../path.${normal}\n"
+    gdb_args=()
+    for piece in $(echo ${single_test_command} | sed -e 's/"\/\<//' -e 's/\>"//')
+    do
+        transformed_piece=($(echo $piece | sed -e 's/.*\/..\//..\//'))
+        printf "${blue} - '${piece}' --> '${transformed_piece}' ${normal}\n"
+        gdb_args+="$transformed_piece"
+    done
+
+    printf "${yellow}gdb_args[@]: ${gdb_args[@]}${normal}\n"
+
+    # Execute debugger
+    pushd "$repo_root" || exit 1
+    gdb --args ${gdb_args[@]}
+    popd > /dev/null || exit 1
+
+else
+    # Execute Test
+    pushd "$repo_root" || exit 1
+    eval "${single_test_command}"
+    exit_code=$?
+    popd > /dev/null || exit 1
+
+    # Print Result
+    printf "${blue}Ran Test #${test_number}: ${single_test_name}${normal}\n"
+    printf "${yellow}Command: ${single_test_command}${normal}\n"
+    if [ $exit_code -eq 0 ]; then
+        printf "${green}TEST PASS${normal}\n"
+    else
+        printf "${red}TEST FAIL${normal}\n"
+    fi
+
+fi
+
 popd > /dev/null || exit 1

From 2304113b1c0c4de8a676be998a13018b4b6eb4f9 Mon Sep 17 00:00:00 2001
From: brian khuu <mofosyne@gmail.com>
Date: Wed, 15 May 2024 11:49:28 +1000
Subject: [PATCH 5/6] debug-test.sh: comment style changes

---
 scripts/debug-test.sh | 22 +++++++++++++++++++++-
 1 file changed, 21 insertions(+), 1 deletion(-)

diff --git a/scripts/debug-test.sh b/scripts/debug-test.sh
index 8c6fa24d51dd6..febeb7dba6012 100755
--- a/scripts/debug-test.sh
+++ b/scripts/debug-test.sh
@@ -12,6 +12,10 @@ magenta=$(tput setaf 5)
 cyan=$(tput setaf 6)
 normal=$(tput sgr0)
 
+
+# Print Help Message
+####################
+
 print_full_help() {
   cat << EOF
 Usage: $PROG [OPTION]... <test_regex> (test_number)
@@ -41,6 +45,10 @@ EOF
   exit 1
 }
 
+
+# Dependency Sanity Check
+#########################
+
 check_dependency() {
   command -v "$1" >/dev/null 2>&1 || {
     abort "$1 is required but not found. Please install it and try again."
@@ -50,6 +58,10 @@ check_dependency() {
 check_dependency ctest
 check_dependency cmake
 
+
+# Step 0: Check the args
+########################
+
 if [ x"$1" = x"-h" ] || [ x"$1" = x"--help" ]; then
   print_full_help >&2
   exit 0
@@ -69,15 +81,17 @@ done
 # Shift the option parameters
 shift $((OPTIND - 1))
 
-# Step 0: Check the args
+# Positionial Argument Processing : <test_regex>
 if [ -z "${1}" ]; then
     abort "Test regex is required"
 else
     test_suite=${1:-}
 fi
 
+# Positionial Argument Processing : (test_number)
 test_number=${2:-}
 
+
 # Step 1: Reset and Setup folder context
 ########################################
 
@@ -91,6 +105,7 @@ fi
 pushd "$repo_root"
 rm -rf "$build_dir" && mkdir "$build_dir" || abort "Failed to make $build_dir"
 
+
 # Step 2: Setup Build Environment and Compile Test Binaries
 ###########################################################
 
@@ -100,6 +115,7 @@ pushd "$build_dir"
 make -j || abort "Failed to compile"
 popd > /dev/null || exit 1
 
+
 # Step 3: Find all tests available that matches REGEX
 ####################################################
 
@@ -115,6 +131,7 @@ if [ ${#tests[@]} -eq 0 ]; then
 fi
 popd > /dev/null || exit 1
 
+
 # Step 4: Identify Test Command for Debugging
 #############################################
 
@@ -151,8 +168,10 @@ popd > /dev/null || exit 1
 single_test_name="${tests[test_number]}"
 single_test_command="${test_args[test_number]}"
 
+
 # Step 5: Execute or GDB Debug
 ##############################
+
 printf "${magenta}Running Test #${test_number}: ${single_test_name}${normal}\n"
 printf "${cyan}single_test_command: ${single_test_command}${normal}\n"
 
@@ -191,4 +210,5 @@ else
 
 fi
 
+# Return to the directory from which the user ran the command.
 popd > /dev/null || exit 1

From 2bfab835e6cd3c68eb1e685a203e3358c222f608 Mon Sep 17 00:00:00 2001
From: brian khuu <mofosyne@gmail.com>
Date: Thu, 16 May 2024 09:23:42 +1000
Subject: [PATCH 6/6] debug-test.sh: fix gdb

---
 scripts/debug-test.sh | 15 ++-------------
 1 file changed, 2 insertions(+), 13 deletions(-)

diff --git a/scripts/debug-test.sh b/scripts/debug-test.sh
index febeb7dba6012..7b2b601a96477 100755
--- a/scripts/debug-test.sh
+++ b/scripts/debug-test.sh
@@ -110,7 +110,7 @@ rm -rf "$build_dir" && mkdir "$build_dir" || abort "Failed to make $build_dir"
 ###########################################################
 
 # Note: test-eval-callback requires -DLLAMA_CURL
-cmake -B "./$build_dir" -DCMAKE_BUILD_TYPE=Debug -DLLAMA_CUDA=1 -DLLAMA_FATAL_WARNINGS=ON -DLLAMA_CURL=1 || abort "Failed to build enviroment"
+cmake -B "./$build_dir" -DCMAKE_BUILD_TYPE=Debug -DLLAMA_CUDA=1 -DLLAMA_CURL=1 || abort "Failed to build enviroment"
 pushd "$build_dir"
 make -j || abort "Failed to compile"
 popd > /dev/null || exit 1
@@ -176,20 +176,9 @@ printf "${magenta}Running Test #${test_number}: ${single_test_name}${normal}\n"
 printf "${cyan}single_test_command: ${single_test_command}${normal}\n"
 
 if [ "$gdb_mode" = "true" ]; then
-    printf "${blue}Replacing single test command string that may have /../ in them with the relative path, i.e. path/../path -> ../path.${normal}\n"
-    gdb_args=()
-    for piece in $(echo ${single_test_command} | sed -e 's/"\/\<//' -e 's/\>"//')
-    do
-        transformed_piece=($(echo $piece | sed -e 's/.*\/..\//..\//'))
-        printf "${blue} - '${piece}' --> '${transformed_piece}' ${normal}\n"
-        gdb_args+="$transformed_piece"
-    done
-
-    printf "${yellow}gdb_args[@]: ${gdb_args[@]}${normal}\n"
-
     # Execute debugger
     pushd "$repo_root" || exit 1
-    gdb --args ${gdb_args[@]}
+    eval "gdb --args ${single_test_command}"
     popd > /dev/null || exit 1
 
 else