refactor

Signed-off-by: Ettore Di Giacinto <[email protected]>

Author: mudler

Makefile

@@ -691,6 +691,13 @@ backend-assets/grpc/llama-cpp-cuda: backend-assets/grpc
 	CMAKE_ARGS="$(CMAKE_ARGS) -DLLAMA_AVX=on -DLLAMA_AVX2=off -DLLAMA_AVX512=off -DLLAMA_FMA=off -DLLAMA_F16C=off -DLLAMA_CUDA=ON" $(MAKE) VARIANT="llama-cuda" build-llama-cpp-grpc-server
 	cp -rfv backend/cpp/llama-cuda/grpc-server backend-assets/grpc/llama-cpp-cuda
 
+backend-assets/grpc/llama-cpp-grpc: backend-assets/grpc
+	cp -rf backend/cpp/llama backend/cpp/llama-grpc
+	$(MAKE) -C backend/cpp/llama-grpc purge
+	$(info ${GREEN}I llama-cpp build info:grpc${RESET})
+	CMAKE_ARGS="$(CMAKE_ARGS) -DLLAMA_RPC=ON -DLLAMA_AVX=off -DLLAMA_AVX2=off -DLLAMA_AVX512=off -DLLAMA_FMA=off -DLLAMA_F16C=off" $(MAKE) VARIANT="llama-grpc" build-llama-cpp-grpc-server
+	cp -rfv backend/cpp/llama-grpc/grpc-server backend-assets/grpc/llama-cpp-grpc
+
 backend-assets/grpc/llama-ggml: sources/go-llama.cpp sources/go-llama.cpp/libbinding.a backend-assets/grpc
 	CGO_LDFLAGS="$(CGO_LDFLAGS)" C_INCLUDE_PATH=$(CURDIR)/sources/go-llama.cpp LIBRARY_PATH=$(CURDIR)/sources/go-llama.cpp \
 	$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/llama-ggml ./backend/go/llm/llama-ggml/
@@ -764,10 +771,3 @@ docker-image-intel-xpu:
 .PHONY: swagger
 swagger:
 	swag init -g core/http/app.go --output swagger
-
-backend-assets/grpc/llama-cpp-grpc: backend-assets/grpc
-	cp -rf backend/cpp/llama backend/cpp/llama-grpc
-	$(MAKE) -C backend/cpp/llama-grpc purge
-	$(info ${GREEN}I llama-cpp build info:grpc${RESET})
-	CMAKE_ARGS="$(CMAKE_ARGS) -DLLAMA_RPC=ON -DLLAMA_AVX=off -DLLAMA_AVX2=off -DLLAMA_AVX512=off -DLLAMA_FMA=off -DLLAMA_F16C=off" $(MAKE) VARIANT="llama-grpc" build-llama-cpp-grpc-server
-	cp -rfv backend/cpp/llama-grpc/grpc-server backend-assets/grpc/llama-cpp-grpc

pkg/model/initializers.go

@@ -29,13 +29,14 @@ var Aliases map[string]string = map[string]string{
 const (
 	LlamaGGML = "llama-ggml"
 
-	LLamaCPP         = "llama-cpp"
+	LLamaCPP = "llama-cpp"
 
 	LLamaCPPCUDA12   = "llama-cpp-cuda12"
 	LLamaCPPAVX2     = "llama-cpp-avx2"
 	LLamaCPPAVX      = "llama-cpp-avx"
 	LLamaCPPFallback = "llama-cpp-fallback"
 	LLamaCPPCUDA     = "llama-cpp-cuda"
+	LLamaCPPGRPC     = "llama-cpp-grpc"
 
 	Gpt4AllLlamaBackend = "gpt4all-llama"
 	Gpt4AllMptBackend   = "gpt4all-mpt"
@@ -81,7 +82,8 @@ ENTRY:
 		}
 	}
 
-	foundLCPPAVX, foundLCPPAVX2, foundLCPPFallback := false, false, false
+	// if we find the llama.cpp variants, show them of as a single backend (llama-cpp)
+	foundLCPPAVX, foundLCPPAVX2, foundLCPPFallback, foundLCPPGRPC := false, false, false, false
 	if _, ok := backends[LLamaCPP]; !ok {
 		for _, e := range entry {
 			if strings.Contains(e.Name(), LLamaCPPAVX2) && !foundLCPPAVX2 {
@@ -96,16 +98,23 @@ ENTRY:
 				backends[LLamaCPP] = append(backends[LLamaCPP], LLamaCPPFallback)
 				foundLCPPFallback = true
 			}
+			if strings.Contains(e.Name(), LLamaCPPGRPC) && !foundLCPPGRPC {
+				backends[LLamaCPP] = append(backends[LLamaCPP], LLamaCPPGRPC)
+				foundLCPPGRPC = true
+			}
 		}
 	}
 
 	// order backends from the asset directory.
 	// as we scan for backends, we want to keep some order which backends are tried of.
 	// for example, llama.cpp should be tried first, and we want to keep the huggingface backend at the last.
-	// sets a priority list
-	// First has more priority
+
+	// sets a priority list - first has more priority
 	priorityList := []string{
-		// First llama.cpp and llama-ggml
+
+		// First llama.cpp(variants) and llama-ggml to follow.
+		// We keep the fallback to prevent that if the llama.cpp variants
+		// that depends on shared libs if breaks have still a safety net.
 		LLamaCPP, LlamaGGML, Gpt4All, LLamaCPPFallback,
 	}
 
@@ -142,6 +151,50 @@ ENTRY:
 	return orderedBackends, nil
 }
 
+// selectGRPCProcess selects the GRPC process to start based on system capabilities
+func selectGRPCProcess(backend, assetDir string) string {
+	foundCUDA := false
+	var grpcProcess string
+
+	// Select backend now just for llama.cpp
+	if backend != LLamaCPP {
+		return ""
+	}
+
+	// Note: This environment variable is read by the LocalAI's llama.cpp grpc-server
+	if os.Getenv("LLAMACPP_GRPC_SERVERS") != "" {
+		return backendPath(assetDir, LLamaCPPGRPC)
+	}
+
+	gpus, err := xsysinfo.GPUs()
+	if err == nil {
+		for _, gpu := range gpus {
+			if strings.Contains(gpu.String(), "nvidia") {
+				log.Info().Msgf("[%s] attempting to load with CUDA variant", backend)
+				grpcProcess = backendPath(assetDir, LLamaCPPCUDA)
+				if _, err := os.Stat(grpcProcess); err == nil {
+					foundCUDA = true
+				}
+			}
+		}
+	}
+
+	if !foundCUDA {
+		if cpu.X86.HasAVX2 {
+			log.Info().Msgf("[%s] attempting to load with AVX2 variant", backend)
+			grpcProcess = backendPath(assetDir, LLamaCPPAVX2)
+		} else if cpu.X86.HasAVX {
+			log.Info().Msgf("[%s] attempting to load with AVX variant", backend)
+			grpcProcess = backendPath(assetDir, LLamaCPPAVX)
+		} else {
+			log.Info().Msgf("[%s] attempting to load with fallback variant", backend)
+			grpcProcess = backendPath(assetDir, LLamaCPPFallback)
+		}
+	}
+
+	return grpcProcess
+}
+
 // starts the grpcModelProcess for the backend, and returns a grpc client
 // It also loads the model
 func (ml *ModelLoader) grpcModel(backend string, o *Options) func(string, string) (ModelAddress, error) {
@@ -192,33 +245,10 @@ func (ml *ModelLoader) grpcModel(backend string, o *Options) func(string, string
 		} else {
 			grpcProcess := backendPath(o.assetDir, backend)
 
-			foundCUDA := false
-			// for llama-cpp, check CPU capabilities and load the appropriate variant
-			if backend == LLamaCPP {
-				gpus, err := xsysinfo.GPUs()
-				if err == nil {
-					for _, gpu := range gpus {
-						if strings.Contains(gpu.String(), "nvidia") {
-							log.Info().Msgf("[%s] attempting to load with CUDA variant", backend)
-							grpcProcess = backendPath(o.assetDir, LLamaCPPCUDA)
-							if _, err := os.Stat(grpcProcess); err == nil {
-								foundCUDA = true
-							}
-						}
-					}
-				}
-
-				if !foundCUDA {
-					if cpu.X86.HasAVX2 {
-						log.Info().Msgf("[%s] attempting to load with AVX2 variant", backend)
-						grpcProcess = backendPath(o.assetDir, LLamaCPPAVX2)
-					} else if cpu.X86.HasAVX {
-						log.Info().Msgf("[%s] attempting to load with AVX variant", backend)
-						grpcProcess = backendPath(o.assetDir, LLamaCPPAVX)
-					} else {
-						log.Info().Msgf("[%s] attempting to load with fallback variant", backend)
-						grpcProcess = backendPath(o.assetDir, LLamaCPPFallback)
-					}
+			if os.Getenv("DISABLE_AUTODETECT") != "true" {
+				// autoDetect GRPC process to start based on system capabilities
+				if selectedProcess := selectGRPCProcess(backend, o.assetDir); selectedProcess != "" {
+					grpcProcess = selectedProcess
 				}
 			}