From 8c8dde93c77f0fad835ae108bbecfedb5c681fd2 Mon Sep 17 00:00:00 2001 From: m0nsky Date: Wed, 19 Jun 2024 10:54:46 +0200 Subject: [PATCH 1/8] Initial Vulkan set up --- LLama/LLamaSharp.Runtime.targets | 8 ++ .../DefaultNativeLibrarySelectingPolicy.cs | 7 +- LLama/Native/Load/NativeLibraryConfig.cs | 34 +++++- LLama/Native/Load/NativeLibraryMetadata.cs | 5 +- LLama/Native/Load/NativeLibraryWithAvx.cs | 2 +- LLama/Native/Load/NativeLibraryWithCuda.cs | 2 +- .../Load/NativeLibraryWithMacOrFallback.cs | 2 +- LLama/Native/Load/NativeLibraryWithVulkan.cs | 65 +++++++++++ LLama/Native/Load/SystemInfo.cs | 107 +++++++++++++++++- .../build/LLamaSharp.Backend.Vulkan.nuspec | 26 +++++ 10 files changed, 249 insertions(+), 9 deletions(-) create mode 100644 LLama/Native/Load/NativeLibraryWithVulkan.cs create mode 100644 LLama/runtimes/build/LLamaSharp.Backend.Vulkan.nuspec diff --git a/LLama/LLamaSharp.Runtime.targets b/LLama/LLamaSharp.Runtime.targets index 3ebf4acf3..2065a9874 100644 --- a/LLama/LLamaSharp.Runtime.targets +++ b/LLama/LLamaSharp.Runtime.targets @@ -28,6 +28,10 @@ PreserveNewest runtimes/win-x64/native/cuda12/llama.dll + + PreserveNewest + runtimes/win-x64/native/vulkan/llama.dll + PreserveNewest @@ -53,6 +57,10 @@ PreserveNewest runtimes/linux-x64/native/cuda12/libllama.so + + PreserveNewest + runtimes/linux-x64/native/vulkan/libllama.so + PreserveNewest diff --git a/LLama/Native/Load/DefaultNativeLibrarySelectingPolicy.cs b/LLama/Native/Load/DefaultNativeLibrarySelectingPolicy.cs index 5cb3b0c5a..690fa706f 100644 --- a/LLama/Native/Load/DefaultNativeLibrarySelectingPolicy.cs +++ b/LLama/Native/Load/DefaultNativeLibrarySelectingPolicy.cs @@ -28,7 +28,12 @@ public IEnumerable Apply(NativeLibraryConfig.Description descrip yield return new NativeLibraryWithCuda(systemInfo.CudaMajorVersion, description.Library, description.SkipCheck); } - if(!description.UseCuda || description.AllowFallback) + if (description.UseVulkan) + { + yield return new NativeLibraryWithVulkan(systemInfo.VulkanVersion, description.Library, description.SkipCheck); + } + + if((!description.UseCuda || !description.UseVulkan) || description.AllowFallback) { if (description.AllowFallback) { diff --git a/LLama/Native/Load/NativeLibraryConfig.cs b/LLama/Native/Load/NativeLibraryConfig.cs index 478102177..02e47b695 100644 --- a/LLama/Native/Load/NativeLibraryConfig.cs +++ b/LLama/Native/Load/NativeLibraryConfig.cs @@ -16,6 +16,7 @@ public sealed partial class NativeLibraryConfig private string? _libraryPath; private bool _useCuda = true; + private bool _useVulkan = true; private AvxLevel _avxLevel; private bool _allowFallback = true; private bool _skipCheck = false; @@ -55,6 +56,20 @@ public NativeLibraryConfig WithCuda(bool enable = true) _useCuda = enable; return this; } + + /// + /// Configure whether to use vulkan backend if possible. Default is true. + /// + /// + /// + /// Thrown if `LibraryHasLoaded` is true. + public NativeLibraryConfig WithVulkan(bool enable = true) + { + ThrowIfLoaded(); + + _useVulkan = enable; + return this; + } /// /// Configure the prefferred avx support level of the backend. @@ -159,6 +174,7 @@ internal Description CheckAndGatherDescription() path, NativeLibraryName, _useCuda, + _useVulkan, _avxLevel, _allowFallback, _skipCheck, @@ -229,7 +245,7 @@ private static bool CheckAVX512() /// /// /// - public record Description(string? Path, NativeLibraryName Library, bool UseCuda, AvxLevel AvxLevel, bool AllowFallback, bool SkipCheck, + public record Description(string? Path, NativeLibraryName Library, bool UseCuda, bool UseVulkan, AvxLevel AvxLevel, bool AllowFallback, bool SkipCheck, string[] SearchDirectories) { /// @@ -250,6 +266,7 @@ public override string ToString() $"- LibraryName: {Library}\n" + $"- Path: '{Path}'\n" + $"- PreferCuda: {UseCuda}\n" + + $"- PreferVulkan: {UseVulkan}\n" + $"- PreferredAvxLevel: {avxLevelString}\n" + $"- AllowFallback: {AllowFallback}\n" + $"- SkipCheck: {SkipCheck}\n" + @@ -425,6 +442,21 @@ public NativeLibraryConfigContainer WithCuda(bool enable = true) } return this; } + + /// + /// Configure whether to use vulkan backend if possible. + /// + /// + /// + /// Thrown if `LibraryHasLoaded` is true. + public NativeLibraryConfigContainer WithVulkan(bool enable = true) + { + foreach(var config in _configs) + { + config.WithVulkan(enable); + } + return this; + } /// /// Configure the prefferred avx support level of the backend. diff --git a/LLama/Native/Load/NativeLibraryMetadata.cs b/LLama/Native/Load/NativeLibraryMetadata.cs index 66c546e6b..d458610b6 100644 --- a/LLama/Native/Load/NativeLibraryMetadata.cs +++ b/LLama/Native/Load/NativeLibraryMetadata.cs @@ -6,12 +6,13 @@ namespace LLama.Native /// /// Which kind of library it is. /// Whether it's compiled with cublas. + /// Whether it's compiled with vulkan. /// Which AvxLevel it's compiled with. - public record class NativeLibraryMetadata(NativeLibraryName NativeLibraryName, bool UseCuda, AvxLevel AvxLevel) + public record class NativeLibraryMetadata(NativeLibraryName NativeLibraryName, bool UseCuda, bool UseVulkan, AvxLevel AvxLevel) { public override string ToString() { - return $"(NativeLibraryName: {NativeLibraryName}, UseCuda: {UseCuda}, AvxLevel: {AvxLevel})"; + return $"(NativeLibraryName: {NativeLibraryName}, UseCuda: {UseCuda}, UseVulkan: {UseVulkan}, AvxLevel: {AvxLevel})"; } } diff --git a/LLama/Native/Load/NativeLibraryWithAvx.cs b/LLama/Native/Load/NativeLibraryWithAvx.cs index 7b5421b4d..52e62845f 100644 --- a/LLama/Native/Load/NativeLibraryWithAvx.cs +++ b/LLama/Native/Load/NativeLibraryWithAvx.cs @@ -19,7 +19,7 @@ public NativeLibraryMetadata? Metadata { get { - return new NativeLibraryMetadata(_libraryName, false, _avxLevel); + return new NativeLibraryMetadata(_libraryName, false, false,_avxLevel); } } diff --git a/LLama/Native/Load/NativeLibraryWithCuda.cs b/LLama/Native/Load/NativeLibraryWithCuda.cs index d3b06b864..982236771 100644 --- a/LLama/Native/Load/NativeLibraryWithCuda.cs +++ b/LLama/Native/Load/NativeLibraryWithCuda.cs @@ -20,7 +20,7 @@ public NativeLibraryMetadata? Metadata { get { - return new NativeLibraryMetadata(_libraryName, true, _avxLevel); + return new NativeLibraryMetadata(_libraryName, true, false, _avxLevel); } } diff --git a/LLama/Native/Load/NativeLibraryWithMacOrFallback.cs b/LLama/Native/Load/NativeLibraryWithMacOrFallback.cs index 4bde4dae1..55734fcc4 100644 --- a/LLama/Native/Load/NativeLibraryWithMacOrFallback.cs +++ b/LLama/Native/Load/NativeLibraryWithMacOrFallback.cs @@ -18,7 +18,7 @@ public NativeLibraryMetadata? Metadata { get { - return new NativeLibraryMetadata(_libraryName, false, AvxLevel.None); + return new NativeLibraryMetadata(_libraryName, false, false, AvxLevel.None); } } diff --git a/LLama/Native/Load/NativeLibraryWithVulkan.cs b/LLama/Native/Load/NativeLibraryWithVulkan.cs new file mode 100644 index 000000000..6227bfe16 --- /dev/null +++ b/LLama/Native/Load/NativeLibraryWithVulkan.cs @@ -0,0 +1,65 @@ +using LLama.Abstractions; +using System.Collections.Generic; +using System.Runtime.InteropServices; + +namespace LLama.Native +{ +#if NET6_0_OR_GREATER + /// + /// A native library compiled with cublas/cuda. + /// + public class NativeLibraryWithVulkan : INativeLibrary + { + private string? _vulkanVersion; + private NativeLibraryName _libraryName; + private AvxLevel _avxLevel; + private bool _skipCheck; + + /// + public NativeLibraryMetadata? Metadata + { + get + { + return new NativeLibraryMetadata(_libraryName, false, true, _avxLevel); + } + } + + /// + /// + /// + /// + /// + /// + public NativeLibraryWithVulkan(string? vulkanVersion, NativeLibraryName libraryName, bool skipCheck) + { + _vulkanVersion = vulkanVersion; + _libraryName = libraryName; + _skipCheck = skipCheck; + } + + /// + public IEnumerable Prepare(SystemInfo systemInfo, NativeLogConfig.LLamaLogCallback? logCallback) + { + // TODO: Avx level is ignored now, needs to be implemented in the future. + if (systemInfo.OSPlatform == OSPlatform.Windows || systemInfo.OSPlatform == OSPlatform.Linux || _skipCheck) + { + if(systemInfo.VulkanVersion != null) + { + var vulkanLibraryPath = GetVulkanPath(systemInfo, logCallback); + if (vulkanLibraryPath is not null) + { + yield return vulkanLibraryPath; + } + } + } + } + + private string? GetVulkanPath(SystemInfo systemInfo, NativeLogConfig.LLamaLogCallback? logCallback) + { + NativeLibraryUtils.GetPlatformPathParts(systemInfo.OSPlatform, out var os, out var fileExtension, out var libPrefix); + var relativePath = $"runtimes/{os}/native/vulkan/{libPrefix}{_libraryName.GetLibraryName()}{fileExtension}"; + return relativePath; + } + } +#endif +} diff --git a/LLama/Native/Load/SystemInfo.cs b/LLama/Native/Load/SystemInfo.cs index 0ffc67e91..a1cff5970 100644 --- a/LLama/Native/Load/SystemInfo.cs +++ b/LLama/Native/Load/SystemInfo.cs @@ -1,7 +1,9 @@ using System; +using System.Diagnostics; using System.IO; using System.Runtime.InteropServices; using System.Text.Json; +using System.Text.RegularExpressions; namespace LLama.Native { @@ -10,7 +12,8 @@ namespace LLama.Native /// /// /// - public record class SystemInfo(OSPlatform OSPlatform, int CudaMajorVersion) + /// + public record class SystemInfo(OSPlatform OSPlatform, int CudaMajorVersion, string? VulkanVersion) { /// /// Get the system information of the current machine. @@ -37,8 +40,108 @@ public static SystemInfo Get() throw new PlatformNotSupportedException(); } - return new SystemInfo(platform, GetCudaMajorVersion()); + return new SystemInfo(platform, GetCudaMajorVersion(), GetVulkanVersion()); } + + #region Vulkan version + private static string? GetVulkanVersion() + { + // Get Vulkan Summary + string? vulkanSummary = GetVulkanSummary(); + + // If we have a Vulkan summary + if (vulkanSummary != null) + { + // Extract Vulkan version from summary + string? vulkanVersion = ExtractVulkanVersionFromSummary(vulkanSummary); + + // If we have a Vulkan version + if (vulkanVersion != null) + { + // Return the Vulkan version + return vulkanVersion; + } + } + + // Return null if we failed to get the Vulkan version + return null; + } + + private static string? GetVulkanSummary() + { + // Note: on Linux, this requires `vulkan-tools` to be installed. (`sudo apt install vulkan-tools`) + try + { + // Set up the process start info + ProcessStartInfo start = new() + { + FileName = "vulkaninfo", + Arguments = "--summary", + RedirectStandardOutput = true, + UseShellExecute = false, + CreateNoWindow = true + }; + + // Start the process + Process process = new() + { + StartInfo = start + }; + process.Start(); + + // Read the output to a string + string output = process.StandardOutput.ReadToEnd(); + + // Wait for the process to exit + process.WaitForExit(); + + // Return the output + return output; + } + catch (Exception e) + { + //Console.WriteLine(e); + + // Return null if we failed to get the Vulkan version + return null; + } + } + + static string? ExtractVulkanVersionFromSummary(string vulkanSummary) + { + // We have three ways of parsing the Vulkan version from the summary (output is a different between Windows and Linux) + // For now, I have decided to go with the full version number, and leave it up to the user to parse it further if needed + // I have left the other patterns in, in case we need them in the future + + // Output on linux : 4206847 (1.3.255) + // Output on windows : 1.3.255 + string pattern = @"apiVersion\s*=\s*([^\r\n]+)"; + + // Output on linux : 4206847 + // Output on windows : 1.3.255 + //string pattern = @"apiVersion\s*=\s*([\d\.]+)"; + + // Output on linux : 1.3.255 + // Output on windows : 1.3.255 + //string pattern = @"apiVersion\s*=\s*(?:\d+\s*)?(?:\(\s*)?([\d]+\.[\d]+\.[\d]+)(?:\s*\))?"; + + // Create a Regex object to match the pattern + Regex regex = new Regex(pattern); + + // Match the pattern in the input string + Match match = regex.Match(vulkanSummary); + + // If a match is found + if (match.Success) + { + // Return the version number + return match.Groups[1].Value; + } + + // Return null if no match is found + return null; + } + #endregion #region CUDA version private static int GetCudaMajorVersion() diff --git a/LLama/runtimes/build/LLamaSharp.Backend.Vulkan.nuspec b/LLama/runtimes/build/LLamaSharp.Backend.Vulkan.nuspec new file mode 100644 index 000000000..95165539f --- /dev/null +++ b/LLama/runtimes/build/LLamaSharp.Backend.Vulkan.nuspec @@ -0,0 +1,26 @@ + + + + LLamaSharp.Backend.Vulkan + $version$ + LLamaSharp.Backend.Vulkan - Vulkan Backend for LLamaSharp + llama.cpp Authors + false + MIT + icon512.png + https://github.com/SciSharp/LLamaSharp + LLamaSharp.Backend.Vulkan is a backend for LLamaSharp to use with Vulkan. + + Copyright 2023 The llama.cpp Authors. All rights reserved. + LLamaSharp LLama LLM GPT AI ChatBot SciSharp + + + + + + + + + + + \ No newline at end of file From 7877f2897648c5014df59f921218ab52fd705dea Mon Sep 17 00:00:00 2001 From: m0nsky Date: Wed, 19 Jun 2024 11:18:57 +0200 Subject: [PATCH 2/8] Replaced old cublas/cuda mention with vulkan --- LLama/Native/Load/NativeLibraryWithVulkan.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LLama/Native/Load/NativeLibraryWithVulkan.cs b/LLama/Native/Load/NativeLibraryWithVulkan.cs index 6227bfe16..867a52f52 100644 --- a/LLama/Native/Load/NativeLibraryWithVulkan.cs +++ b/LLama/Native/Load/NativeLibraryWithVulkan.cs @@ -6,7 +6,7 @@ namespace LLama.Native { #if NET6_0_OR_GREATER /// - /// A native library compiled with cublas/cuda. + /// A native library compiled with vulkan. /// public class NativeLibraryWithVulkan : INativeLibrary { From 1eb8e1c4e1d2b71d9ef8663768b2fbb949b84a9a Mon Sep 17 00:00:00 2001 From: m0nsky Date: Wed, 19 Jun 2024 22:37:24 +0200 Subject: [PATCH 3/8] Make sure we use the correct llama_cpp_commit for Vulkan --- .github/workflows/compile.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/compile.yml b/.github/workflows/compile.yml index e19625f86..43b09f58f 100644 --- a/.github/workflows/compile.yml +++ b/.github/workflows/compile.yml @@ -118,6 +118,8 @@ jobs: uses: actions/checkout@v4 with: repository: ggerganov/llama.cpp + fetch-depth: 0 + ref: '${{ github.event.inputs.llama_cpp_commit }}' - name: Download dependencies - Linux if: ${{ matrix.os == 'ubuntu-22.04' }} From 777df9cfdd01183423f607b28e3bdf6e6a9b1533 Mon Sep 17 00:00:00 2001 From: m0nsky Date: Wed, 19 Jun 2024 22:39:15 +0200 Subject: [PATCH 4/8] Remove duplicate cmake params (already defined in COMMON_DEFINE) --- .github/workflows/compile.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/compile.yml b/.github/workflows/compile.yml index 43b09f58f..c88f98676 100644 --- a/.github/workflows/compile.yml +++ b/.github/workflows/compile.yml @@ -142,7 +142,7 @@ jobs: run: | mkdir build cd build - cmake .. ${{ env.COMMON_DEFINE }} -DLLAMA_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DLLAMA_VULKAN=ON -DBUILD_SHARED_LIBS=ON -DCMAKE_PREFIX_PATH="$env:RUNNER_TEMP/vulkan" + cmake .. ${{ env.COMMON_DEFINE }} -DLLAMA_VULKAN=ON cmake --build . --config Release -j ${env:NUMBER_OF_PROCESSORS} ls -R - name: Build @@ -150,7 +150,7 @@ jobs: run: | mkdir build cd build - cmake .. ${{ env.COMMON_DEFINE }} -DLLAMA_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DLLAMA_VULKAN=ON -DBUILD_SHARED_LIBS=ON + cmake .. ${{ env.COMMON_DEFINE }} -DLLAMA_VULKAN=ON cmake --build . --config Release -j ${env:NUMBER_OF_PROCESSORS} ls -R - name: Upload llama artifacts (Windows) From 9037bea8c2747cf4e01b4d9f95a60071f3eaaafe Mon Sep 17 00:00:00 2001 From: m0nsky Date: Thu, 11 Jul 2024 01:50:54 +0200 Subject: [PATCH 5/8] Add llava & ggml to Vulkan nuspec --- LLama/runtimes/build/LLamaSharp.Backend.Vulkan.nuspec | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/LLama/runtimes/build/LLamaSharp.Backend.Vulkan.nuspec b/LLama/runtimes/build/LLamaSharp.Backend.Vulkan.nuspec index 95165539f..afbecae42 100644 --- a/LLama/runtimes/build/LLamaSharp.Backend.Vulkan.nuspec +++ b/LLama/runtimes/build/LLamaSharp.Backend.Vulkan.nuspec @@ -19,7 +19,12 @@ + + + + + From c954d26a62651b1cb6b4090765e8bf6080383cc6 Mon Sep 17 00:00:00 2001 From: m0nsky Date: Thu, 11 Jul 2024 21:29:30 +0200 Subject: [PATCH 6/8] Updated runtime targets for Vulkan (llava + ggml) --- LLama/LLamaSharp.Runtime.targets | 340 +++++++++++++++++++------------ 1 file changed, 208 insertions(+), 132 deletions(-) diff --git a/LLama/LLamaSharp.Runtime.targets b/LLama/LLamaSharp.Runtime.targets index 2065a9874..9aa2c6e2c 100644 --- a/LLama/LLamaSharp.Runtime.targets +++ b/LLama/LLamaSharp.Runtime.targets @@ -4,143 +4,219 @@ - - PreserveNewest - runtimes/win-x64/native/noavx/llama.dll - - - PreserveNewest - runtimes/win-x64/native/avx/llama.dll - - - PreserveNewest - runtimes/win-x64/native/avx2/llama.dll - - - PreserveNewest - runtimes/win-x64/native/avx512/llama.dll - - - PreserveNewest - runtimes/win-x64/native/cuda11/llama.dll - - - PreserveNewest - runtimes/win-x64/native/cuda12/llama.dll - - - PreserveNewest - runtimes/win-x64/native/vulkan/llama.dll - + + PreserveNewest + runtimes/win-x64/native/noavx/llama.dll + + + PreserveNewest + runtimes/win-x64/native/noavx/ggml.dll + + + PreserveNewest + runtimes/win-x64/native/avx/llama.dll + + + PreserveNewest + runtimes/win-x64/native/avx/ggml.dll + + + PreserveNewest + runtimes/win-x64/native/avx2/llama.dll + + + PreserveNewest + runtimes/win-x64/native/avx2/ggml.dll + + + PreserveNewest + runtimes/win-x64/native/avx512/llama.dll + + + PreserveNewest + runtimes/win-x64/native/avx512/ggml.dll + + + PreserveNewest + runtimes/win-x64/native/cuda11/llama.dll + + + PreserveNewest + runtimes/win-x64/native/cuda11/ggml.dll + + + PreserveNewest + runtimes/win-x64/native/cuda12/llama.dll + + + PreserveNewest + runtimes/win-x64/native/cuda12/ggml.dll + + + PreserveNewest + runtimes/win-x64/native/vulkan/llama.dll + + + PreserveNewest + runtimes/win-x64/native/vulkan/ggml.dll + - - PreserveNewest - runtimes/linux-x64/native/noavx/libllama.so - - - PreserveNewest - runtimes/linux-x64/native/avx/libllama.so - - - PreserveNewest - runtimes/linux-x64/native/avx2/libllama.so - - - PreserveNewest - runtimes/linux-x64/native/avx512/libllama.so - - - PreserveNewest - runtimes/linux-x64/native/cuda11/libllama.so - - - PreserveNewest - runtimes/linux-x64/native/cuda12/libllama.so - - - PreserveNewest - runtimes/linux-x64/native/vulkan/libllama.so - + + PreserveNewest + runtimes/linux-x64/native/noavx/libllama.so + + + PreserveNewest + runtimes/linux-x64/native/noavx/libggml.so + + + PreserveNewest + runtimes/linux-x64/native/avx/libllama.so + + + PreserveNewest + runtimes/linux-x64/native/avx/libggml.so + + + PreserveNewest + runtimes/linux-x64/native/avx2/libllama.so + + + PreserveNewest + runtimes/linux-x64/native/avx2/libggml.so + + + PreserveNewest + runtimes/linux-x64/native/avx512/libllama.so + + + PreserveNewest + runtimes/linux-x64/native/avx512/libggml.so + + + PreserveNewest + runtimes/linux-x64/native/cuda11/libllama.so + + + PreserveNewest + runtimes/linux-x64/native/cuda11/libggml.so + + + PreserveNewest + runtimes/linux-x64/native/cuda12/libllama.so + + + PreserveNewest + runtimes/linux-x64/native/cuda12/libggml.so + + + PreserveNewest + runtimes/linux-x64/native/vulkan/libllama.so + + + PreserveNewest + runtimes/linux-x64/native/vulkan/libggml.so + - - PreserveNewest - runtimes/osx-arm64/native/libllama.dylib - - - PreserveNewest - runtimes/osx-arm64/native/libllava_shared.dylib - - - PreserveNewest - runtimes/osx-arm64/native/ggml-metal.metal - + + PreserveNewest + runtimes/osx-arm64/native/libggml.dylib + + + PreserveNewest + runtimes/osx-arm64/native/libllama.dylib + + + PreserveNewest + runtimes/osx-arm64/native/libllava_shared.dylib + + + PreserveNewest + runtimes/osx-arm64/native/ggml-metal.metal + - - PreserveNewest - runtimes/osx-x64/native/libllama.dylib - - - PreserveNewest - runtimes/osx-x64/native/libllava_shared.dylib - + + PreserveNewest + runtimes/osx-x64/native/libggml.dylib + + + PreserveNewest + runtimes/osx-x64/native/libllama.dylib + + + PreserveNewest + runtimes/osx-x64/native/libllava_shared.dylib + - - PreserveNewest - runtimes/osx-x64/native/rosetta2/libllama.dylib - - - PreserveNewest - runtimes/osx-x64/native/rosetta2/libllava_shared.dylib - + + PreserveNewest + runtimes/osx-x64/native/rosetta2/libggml.dylib + + + PreserveNewest + runtimes/osx-x64/native/rosetta2/libllama.dylib + + + PreserveNewest + runtimes/osx-x64/native/rosetta2/libllava_shared.dylib + - - PreserveNewest - runtimes/win-x64/native/noavx/llava_shared.dll - - - PreserveNewest - runtimes/win-x64/native/avx/llava_shared.dll - - - PreserveNewest - runtimes/win-x64/native/avx2/llava_shared.dll - - - PreserveNewest - runtimes/win-x64/native/avx512/llava_shared.dll - - - PreserveNewest - runtimes/win-x64/native/cuda11/llava_shared.dll - - - PreserveNewest - runtimes/win-x64/native/cuda12/llava_shared.dll - + + PreserveNewest + runtimes/win-x64/native/noavx/llava_shared.dll + + + PreserveNewest + runtimes/win-x64/native/avx/llava_shared.dll + + + PreserveNewest + runtimes/win-x64/native/avx2/llava_shared.dll + + + PreserveNewest + runtimes/win-x64/native/avx512/llava_shared.dll + + + PreserveNewest + runtimes/win-x64/native/cuda11/llava_shared.dll + + + PreserveNewest + runtimes/win-x64/native/cuda12/llava_shared.dll + + + PreserveNewest + runtimes/win-x64/native/vulkan/llava_shared.dll + - - PreserveNewest - runtimes/linux-x64/native/noavx/libllava_shared.so - - - PreserveNewest - runtimes/linux-x64/native/avx/libllava_shared.so - - - PreserveNewest - runtimes/linux-x64/native/avx2/libllava_shared.so - - - PreserveNewest - runtimes/linux-x64/native/avx512/libllava_shared.so - - - PreserveNewest - runtimes/linux-x64/native/cuda11/libllava_shared.so - - - PreserveNewest - runtimes/linux-x64/native/cuda12/libllava_shared.so - + + PreserveNewest + runtimes/linux-x64/native/noavx/libllava_shared.so + + + PreserveNewest + runtimes/linux-x64/native/avx/libllava_shared.so + + + PreserveNewest + runtimes/linux-x64/native/avx2/libllava_shared.so + + + PreserveNewest + runtimes/linux-x64/native/avx512/libllava_shared.so + + + PreserveNewest + runtimes/linux-x64/native/cuda11/libllava_shared.so + + + PreserveNewest + runtimes/linux-x64/native/cuda12/libllava_shared.so + + + PreserveNewest + runtimes/linux-x64/native/vulkan/libllava_shared.so + \ No newline at end of file From 07c4f0eae66327081a07a1225211373cdcf44eed Mon Sep 17 00:00:00 2001 From: m0nsky Date: Thu, 11 Jul 2024 21:47:40 +0200 Subject: [PATCH 7/8] Updated runtime targets for Vulkan (llava + ggml) --- LLama/LLamaSharp.Runtime.targets | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/LLama/LLamaSharp.Runtime.targets b/LLama/LLamaSharp.Runtime.targets index 4f3586667..6466a1204 100644 --- a/LLama/LLamaSharp.Runtime.targets +++ b/LLama/LLamaSharp.Runtime.targets @@ -52,6 +52,14 @@ PreserveNewest runtimes/win-x64/native/cuda12/ggml.dll + + PreserveNewest + runtimes/win-x64/native/vulkan/llama.dll + + + PreserveNewest + runtimes/win-x64/native/vulkan/ggml.dll + PreserveNewest @@ -101,6 +109,14 @@ PreserveNewest runtimes/linux-x64/native/cuda12/libggml.so + + PreserveNewest + runtimes/linux-x64/native/vulkan/libllama.so + + + PreserveNewest + runtimes/linux-x64/native/vulkan/libggml.so + PreserveNewest @@ -169,6 +185,10 @@ PreserveNewest runtimes/win-x64/native/cuda12/llava_shared.dll + + PreserveNewest + runtimes/win-x64/native/vulkan/llava_shared.dll + PreserveNewest @@ -194,5 +214,9 @@ PreserveNewest runtimes/linux-x64/native/cuda12/libllava_shared.so + + PreserveNewest + runtimes/linux-x64/native/vulkan/libllava_shared.so + \ No newline at end of file From 8c3a53bfbdde18c6f725503435dab8e9932a0206 Mon Sep 17 00:00:00 2001 From: m0nsky Date: Thu, 11 Jul 2024 21:58:07 +0200 Subject: [PATCH 8/8] Make Vulkan nuspec match the CUDA nuspec title/order --- .../build/LLamaSharp.Backend.Vulkan.nuspec | 50 +++++++++---------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/LLama/runtimes/build/LLamaSharp.Backend.Vulkan.nuspec b/LLama/runtimes/build/LLamaSharp.Backend.Vulkan.nuspec index afbecae42..33880b037 100644 --- a/LLama/runtimes/build/LLamaSharp.Backend.Vulkan.nuspec +++ b/LLama/runtimes/build/LLamaSharp.Backend.Vulkan.nuspec @@ -1,31 +1,31 @@ - - LLamaSharp.Backend.Vulkan - $version$ - LLamaSharp.Backend.Vulkan - Vulkan Backend for LLamaSharp - llama.cpp Authors - false - MIT - icon512.png - https://github.com/SciSharp/LLamaSharp - LLamaSharp.Backend.Vulkan is a backend for LLamaSharp to use with Vulkan. - - Copyright 2023 The llama.cpp Authors. All rights reserved. - LLamaSharp LLama LLM GPT AI ChatBot SciSharp - - - - + + LLamaSharp.Backend.Vulkan + $version$ + LLamaSharp.Backend.Vulkan, the backend for LLamaSharp + llama.cpp Authors + false + MIT + icon512.png + https://github.com/SciSharp/LLamaSharp + LLamaSharp.Backend.Vulkan is a backend for LLamaSharp to use with Vulkan. + + Copyright 2023 The llama.cpp Authors. All rights reserved. + LLamaSharp LLama LLM GPT AI ChatBot SciSharp + - - - + + - - - + + - - + + + + + + + \ No newline at end of file