[Feat] Multi model support (#931)

* Update Llama class to handle chat_format & caching

* Add settings.py

* Add util.py & update __main__.py

* multimodel

* update settings.py

* cleanup

* delete util.py

* Fix /v1/models endpoint

* MultiLlama now iterable, app check-alive on "/"

* instant model init if file is given

* backward compability

* revert model param mandatory

* fix error

* handle individual model config json

* refactor

* revert chathandler/clip_model changes

* handle chat_handler in MulitLlama()

* split settings into server/llama

* reduce global vars

* Update LlamaProxy to handle config files

* Add free method to LlamaProxy

* update arg parsers & install server alias

* refactor cache settings

* change server executable name

* better var name

* whitespace

* Revert "whitespace"

This reverts commit bc5cf51.

* remove exe_name

* Fix merge bugs

* Fix type annotations

* Fix type annotations

* Fix uvicorn app factory

* Fix settings

* Refactor server

* Remove formatting fix

* Format

* Use default model if not found in model settings

* Fix

* Cleanup

* Fix

* Fix

* Remove unnused CommandLineSettings

* Cleanup

* Support default name for copilot-codex models

---------

Co-authored-by: Andrei Betlen <[email protected]>

Author: D4ve-R

llama_cpp/server/__main__.py

@@ -9,7 +9,7 @@
 
 Then run:
 ```
-uvicorn llama_cpp.server.app:app --reload
+uvicorn llama_cpp.server.app:create_app --reload
 ```
 
 or
@@ -21,81 +21,68 @@
 Then visit http://localhost:8000/docs to see the interactive API docs.
 
 """
+from __future__ import annotations
+
 import os
+import sys
 import argparse
-from typing import List, Literal, Union
 
 import uvicorn
 
-from llama_cpp.server.app import create_app, Settings
-
-def get_base_type(annotation):
-    if getattr(annotation, '__origin__', None) is Literal:
-        return type(annotation.__args__[0])
-    elif getattr(annotation, '__origin__', None) is Union:
-        non_optional_args = [arg for arg in annotation.__args__ if arg is not type(None)]
-        if non_optional_args:
-            return get_base_type(non_optional_args[0])
-    elif getattr(annotation, '__origin__', None) is list or getattr(annotation, '__origin__', None) is List:
-        return get_base_type(annotation.__args__[0])
-    else:
-        return annotation
-
-def contains_list_type(annotation) -> bool:
-    origin = getattr(annotation, '__origin__', None)
-    
-    if origin is list or origin is List:
-        return True
-    elif origin in (Literal, Union):
-        return any(contains_list_type(arg) for arg in annotation.__args__)
-    else:
-        return False
-
-def parse_bool_arg(arg):
-    if isinstance(arg, bytes):
-        arg = arg.decode('utf-8')
-
-    true_values = {'1', 'on', 't', 'true', 'y', 'yes'}
-    false_values = {'0', 'off', 'f', 'false', 'n', 'no'}
-
-    arg_str = str(arg).lower().strip()
-    
-    if arg_str in true_values:
-        return True
-    elif arg_str in false_values:
-        return False
-    else:
-        raise ValueError(f'Invalid boolean argument: {arg}')
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    for name, field in Settings.model_fields.items():
-        description = field.description
-        if field.default is not None and description is not None:
-            description += f" (default: {field.default})"
-        base_type = get_base_type(field.annotation) if field.annotation is not None else str
-        list_type = contains_list_type(field.annotation)
-        if base_type is not bool:
-            parser.add_argument(
-                f"--{name}",
-                dest=name,
-                nargs="*" if list_type else None,
-                type=base_type,
-                help=description,
-            )
-        if base_type is bool:
-            parser.add_argument(
-                f"--{name}",
-                dest=name,
-                type=parse_bool_arg,
-                help=f"{description}",
-            )
-
+from llama_cpp.server.app import create_app
+from llama_cpp.server.settings import (
+    Settings,
+    ServerSettings,
+    ModelSettings,
+    ConfigFileSettings,
+)
+from llama_cpp.server.cli import add_args_from_model, parse_model_from_args
+
+
+def main():
+    description = "🦙 Llama.cpp python server. Host your own LLMs!🚀"
+    parser = argparse.ArgumentParser(description=description)
+
+    add_args_from_model(parser, Settings)
+    parser.add_argument(
+        "--config_file",
+        type=str,
+        help="Path to a config file to load.",
+    )
+    server_settings: ServerSettings | None = None
+    model_settings: list[ModelSettings] = []
     args = parser.parse_args()
-    settings = Settings(**{k: v for k, v in vars(args).items() if v is not None})
-    app = create_app(settings=settings)
-
+    try:
+        # Load server settings from config_file if provided
+        config_file = os.environ.get("CONFIG_FILE", args.config_file)
+        if config_file:
+            if not os.path.exists(config_file):
+                raise ValueError(f"Config file {config_file} not found!")
+            with open(config_file, "rb") as f:
+                config_file_settings = ConfigFileSettings.model_validate_json(f.read())
+                server_settings = ServerSettings.model_validate(config_file_settings)
+                model_settings = config_file_settings.models
+        else:
+            server_settings = parse_model_from_args(ServerSettings, args)
+            model_settings = [parse_model_from_args(ModelSettings, args)]
+    except Exception as e:
+        print(e, file=sys.stderr)
+        parser.print_help()
+        sys.exit(1)
+    assert server_settings is not None
+    assert model_settings is not None
+    app = create_app(
+        server_settings=server_settings,
+        model_settings=model_settings,
+    )
     uvicorn.run(
-        app, host=os.getenv("HOST", settings.host), port=int(os.getenv("PORT", settings.port)),
-        ssl_keyfile=settings.ssl_keyfile, ssl_certfile=settings.ssl_certfile
+        app,
+        host=os.getenv("HOST", server_settings.host),
+        port=int(os.getenv("PORT", server_settings.port)),
+        ssl_keyfile=server_settings.ssl_keyfile,
+        ssl_certfile=server_settings.ssl_certfile,
     )
+
+
+if __name__ == "__main__":
+    main()