finetune.cpp command-line arg

add to ggml-opt learning rate (adamw alpha) cmdline arg,
and an optimizer enum defaulting to adamw, including
string->id mapping,
preparatory to work to support SGD

these are in common args a set of optimizer options active
only for the new FINETUNE example (but we drop all the
previous finetune.cpp PERPLEXITY options which we're told
are unused/accidental)

perhaps breaking with precedent, the ggml_opt_optimizer_params
struct is included directly as args - if desired, we can instead
just add learning rate and optimizer type to a struct independent
of ggml-opt.h

as proposed in
#13835

Author: graehl

common/arg.cpp

@@ -1085,48 +1085,47 @@ static void common_params_print_completion(common_params_context & ctx_arg) {
     printf("    esac\n");
     printf("}\n\n");
 
-    std::set<std::string> executables = {
-        "llama-batched",
-        "llama-batched-bench",
-        "llama-bench",
-        "llama-cli",
-        "llama-convert-llama2c-to-ggml",
-        "llama-cvector-generator",
-        "llama-embedding",
-        "llama-eval-callback",
-        "llama-export-lora",
-        "llama-gen-docs",
-        "llama-gguf",
-        "llama-gguf-hash",
-        "llama-gguf-split",
-        "llama-gritlm",
-        "llama-imatrix",
-        "llama-infill",
-        "llama-mtmd-cli",
-        "llama-llava-clip-quantize-cli",
-        "llama-lookahead",
-        "llama-lookup",
-        "llama-lookup-create",
-        "llama-lookup-merge",
-        "llama-lookup-stats",
-        "llama-parallel",
-        "llama-passkey",
-        "llama-perplexity",
-        "llama-q8dot",
-        "llama-quantize",
-        "llama-qwen2vl-cli",
-        "llama-retrieval",
-        "llama-run",
-        "llama-save-load-state",
-        "llama-server",
-        "llama-simple",
-        "llama-simple-chat",
-        "llama-speculative",
-        "llama-speculative-simple",
-        "llama-tokenize",
-        "llama-tts",
-        "llama-vdot"
-    };
+    std::set<std::string> executables = { "llama-batched",
+                                          "llama-batched-bench",
+                                          "llama-bench",
+                                          "llama-cli",
+                                          "llama-convert-llama2c-to-ggml",
+                                          "llama-cvector-generator",
+                                          "llama-embedding",
+                                          "llama-eval-callback",
+                                          "llama-export-lora",
+                                          "llama-finetune",
+                                          "llama-gen-docs",
+                                          "llama-gguf",
+                                          "llama-gguf-hash",
+                                          "llama-gguf-split",
+                                          "llama-gritlm",
+                                          "llama-imatrix",
+                                          "llama-infill",
+                                          "llama-mtmd-cli",
+                                          "llama-llava-clip-quantize-cli",
+                                          "llama-lookahead",
+                                          "llama-lookup",
+                                          "llama-lookup-create",
+                                          "llama-lookup-merge",
+                                          "llama-lookup-stats",
+                                          "llama-parallel",
+                                          "llama-passkey",
+                                          "llama-perplexity",
+                                          "llama-q8dot",
+                                          "llama-quantize",
+                                          "llama-qwen2vl-cli",
+                                          "llama-retrieval",
+                                          "llama-run",
+                                          "llama-save-load-state",
+                                          "llama-server",
+                                          "llama-simple",
+                                          "llama-simple-chat",
+                                          "llama-speculative",
+                                          "llama-speculative-simple",
+                                          "llama-tokenize",
+                                          "llama-tts",
+                                          "llama-vdot" };
 
     for (const auto& exe : executables) {
         printf("complete -F _llama_completions %s\n", exe.c_str());
@@ -1238,6 +1237,8 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
     }
     sampler_type_names.pop_back();
 
+    params.optimize             = ggml_opt_get_default_optimizer_params(NULL);
+    params.optimize.adamw.alpha = 1e-8;  // default 1e-3 is much too high for LLAMA_EXAMPLE_FINETUNE
 
     /**
      * filter options by example
@@ -2181,6 +2182,22 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
             params.ppl_output_type = value;
         }
     ).set_examples({LLAMA_EXAMPLE_PERPLEXITY}));
+    add_opt(common_arg({ "-lr", "--learning-rate" }, "ALPHA",
+                       string_format("adamw optimizer alpha (default: %.1f)", (double) params.optimize.adamw.alpha),
+                       [](common_params & params, const std::string & value) {
+                           params.optimize.adamw.alpha = std::stof(value);
+                       })
+                .set_examples({ LLAMA_EXAMPLE_FINETUNE }));
+    add_opt(common_arg({ "-opt", "--optimizer" }, "sgd|adamw", "adamw or //TODO:sgd",
+                       [](common_params & params, const std::string & name) {
+                           params.optimize.optimizer = named_ggml_opt_optimizer(name.c_str());
+                           if (params.optimize.optimizer == GGML_OPT_OPTIMIZER_COUNT) {
+                               throw std::invalid_argument("invalid --optimizer (try adamw)");
+                           } else if (params.optimize.optimizer == GGML_OPT_OPTIMIZER_SGD) {
+                               throw std::invalid_argument("TODO: implement SGD");
+                           }
+                       })
+                .set_examples({ LLAMA_EXAMPLE_FINETUNE }));
     add_opt(common_arg(
         {"-dt", "--defrag-thold"}, "N",
         string_format("KV cache defragmentation threshold (default: %.1f, < 0 - disabled)", (double)params.defrag_thold),

common/common.h

@@ -2,13 +2,14 @@
 
 #pragma once
 
-#include "llama-cpp.h"
-
 #include <set>
+#include <sstream>
 #include <string>
 #include <string_view>
 #include <vector>
-#include <sstream>
+
+#include "ggml-opt.h"
+#include "llama-cpp.h"
 
 #ifdef _WIN32
 #define DIRECTORY_SEPARATOR '\\'
@@ -80,6 +81,7 @@ enum llama_example {
     LLAMA_EXAMPLE_LOOKUP,
     LLAMA_EXAMPLE_PARALLEL,
     LLAMA_EXAMPLE_TTS,
+    LLAMA_EXAMPLE_FINETUNE,
 
     LLAMA_EXAMPLE_COUNT,
 };
@@ -349,6 +351,8 @@ struct common_params {
     bool no_mmproj = false;         // explicitly disable multimodal model
     std::vector<std::string> image; // path to image file(s)
 
+    // finetune
+    struct ggml_opt_optimizer_params optimize;
     // embedding
     bool embedding         = false; // get only sentence embedding
     int32_t embd_normalize = 2;     // normalisation for embeddings (-1=none, 0=max absolute int16, 1=taxicab, 2=euclidean, >2=p-norm)

examples/training/finetune.cpp

@@ -18,7 +18,7 @@ int main(int argc, char ** argv) {
 
     params.escape = false;
 
-    if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_PERPLEXITY)) {
+    if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_FINETUNE)) {
         return 1;
     }
 
@@ -60,8 +60,8 @@ int main(int argc, char ** argv) {
     std::vector<llama_token> tokens = common_tokenize(ctx.get(), params.prompt, true);
     ggml_opt_dataset_t dataset = common_opt_dataset_init(ctx.get(), tokens, llama_n_ctx(ctx.get())/2);
 
-    struct ggml_opt_optimizer_params optimizer_params = ggml_opt_get_default_optimizer_params(nullptr);
-    optimizer_params.adamw.alpha = 1e-7f; // learning rate
+    struct ggml_opt_optimizer_params & optimizer_params = params.optimize;
+    LOG_INF("-optimizer %d -lr: %.1f", optimizer_params.optimizer, (double) optimizer_params.adamw.alpha);
 
     struct llama_opt_params lopt_params {
         /*n_ctx_train     =*/ 0,

ggml/include/ggml-opt.h

@@ -74,6 +74,17 @@ extern "C" {
         GGML_OPT_BUILD_TYPE_OPT     = 30,
     };
 
+    enum ggml_opt_optimizer {
+        GGML_OPT_OPTIMIZER_ADAMW,
+        GGML_OPT_OPTIMIZER_SGD,
+
+        GGML_OPT_OPTIMIZER_COUNT
+    };
+
+    // "adamw" or "sgd" (case insensitive)
+    GGML_API const char *            ggml_opt_optimizer_name(enum ggml_opt_optimizer);
+    GGML_API enum ggml_opt_optimizer named_ggml_opt_optimizer(const char *);
+
     // parameters that control which optimizer is used and how said optimizer tries to find the minimal loss
     struct ggml_opt_optimizer_params {
         // AdamW optimizer parameters
@@ -84,6 +95,7 @@ extern "C" {
             float eps;   // epsilon for numerical stability
             float wd;    // weight decay for AdamW, use 0.0f to disable
         } adamw;
+        enum ggml_opt_optimizer optimizer;
     };
 
     // callback to calculate optimizer parameters prior to a backward pass

ggml/src/ggml-opt.cpp

@@ -228,10 +228,32 @@ struct ggml_opt_optimizer_params ggml_opt_get_default_optimizer_params(void * us
     result.adamw.beta2 = 0.999f;
     result.adamw.eps   = 1e-8f;
     result.adamw.wd    = 0.0f;
+    result.optimizer   = GGML_OPT_OPTIMIZER_ADAMW;
 
     return result;
 }
 
+GGML_API const char * ggml_opt_optimizer_name(enum ggml_opt_optimizer o) {
+    switch (o) {
+        case GGML_OPT_OPTIMIZER_ADAMW:
+            return "adamw";
+        case GGML_OPT_OPTIMIZER_SGD:
+            return "sgd";
+        default:
+            return "undefined";
+    };
+}
+
+GGML_API enum ggml_opt_optimizer named_ggml_opt_optimizer(const char * n) {
+    if (!strcasecmp("adamw", n)) {
+        return GGML_OPT_OPTIMIZER_ADAMW;
+    } else if (!strcasecmp("sgd", n)) {
+        return GGML_OPT_OPTIMIZER_SGD;
+    } else {
+        return GGML_OPT_OPTIMIZER_COUNT;
+    }
+}
+
 struct ggml_opt_optimizer_params ggml_opt_get_constant_optimizer_params(void * userdata) {
     return *((struct ggml_opt_optimizer_params *) userdata);
 }