Skip to content

Commit eb4ef3e

Browse files
ggerganovhodlen
ggerganov
authored and
hodlen
committed
metal : build metallib + fix embed path (ggml-org#6015)
* metal : build metallib + fix embed path ggml-ci * metal : fix embed build + update library load logic ggml-ci * metal : fix embeded library build ggml-ci * ci : fix iOS builds to use embedded library
1 parent 8718be7 commit eb4ef3e

File tree

6 files changed

+83
-58
lines changed

6 files changed

+83
-58
lines changed

‎.github/workflows/build.yml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -333,6 +333,7 @@ jobs:
333333
mkdir build
334334
cd build
335335
cmake -G Xcode .. \
336+
-DLLAMA_METAL_EMBED_LIBRARY=ON \
336337
-DLLAMA_BUILD_EXAMPLES=OFF \
337338
-DLLAMA_BUILD_TESTS=OFF \
338339
-DLLAMA_BUILD_SERVER=OFF \
@@ -361,6 +362,7 @@ jobs:
361362
mkdir build
362363
cd build
363364
cmake -G Xcode .. \
365+
-DLLAMA_METAL_EMBED_LIBRARY=ON \
364366
-DLLAMA_BUILD_EXAMPLES=OFF \
365367
-DLLAMA_BUILD_TESTS=OFF \
366368
-DLLAMA_BUILD_SERVER=OFF \

‎.gitignore

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,8 @@
2525
.vscode/
2626
.idea/
2727

28+
ggml-metal-embed.metal
29+
2830
lcov-report/
2931
gcovr-report/
3032

‎CMakeLists.txt

Lines changed: 38 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -200,9 +200,6 @@ if (LLAMA_METAL)
200200
add_compile_definitions(GGML_METAL_NDEBUG)
201201
endif()
202202

203-
# get full path to the file
204-
#add_compile_definitions(GGML_METAL_DIR_KERNELS="${CMAKE_CURRENT_SOURCE_DIR}/")
205-
206203
# copy ggml-common.h and ggml-metal.metal to bin directory
207204
configure_file(ggml-common.h ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-common.h COPYONLY)
208205
configure_file(ggml-metal.metal ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal.metal COPYONLY)
@@ -211,53 +208,62 @@ if (LLAMA_METAL)
211208
enable_language(ASM)
212209
add_compile_definitions(GGML_METAL_EMBED_LIBRARY)
213210

211+
set(METALLIB_COMMON "${CMAKE_CURRENT_SOURCE_DIR}/ggml-common.h")
214212
set(METALLIB_SOURCE "${CMAKE_CURRENT_SOURCE_DIR}/ggml-metal.metal")
213+
215214
file(MAKE_DIRECTORY "${CMAKE_BINARY_DIR}/autogenerated")
216-
set(EMBED_METALLIB_ASSEMBLY "${CMAKE_BINARY_DIR}/autogenerated/ggml-embed-metallib.s")
215+
216+
# merge ggml-common.h and ggml-metal.metal into a single file
217+
set(METALLIB_EMBED_ASM "${CMAKE_BINARY_DIR}/autogenerated/ggml-metal-embed.s")
218+
set(METALLIB_SOURCE_EMBED "${CMAKE_BINARY_DIR}/autogenerated/ggml-metal-embed.metal")
217219

218220
add_custom_command(
219-
OUTPUT ${EMBED_METALLIB_ASSEMBLY}
220-
COMMAND echo ".section __DATA,__ggml_metallib" > ${EMBED_METALLIB_ASSEMBLY}
221-
COMMAND echo ".globl _ggml_metallib_start" >> ${EMBED_METALLIB_ASSEMBLY}
222-
COMMAND echo "_ggml_metallib_start:" >> ${EMBED_METALLIB_ASSEMBLY}
223-
COMMAND echo ".incbin \\\"${METALLIB_SOURCE}\\\"" >> ${EMBED_METALLIB_ASSEMBLY}
224-
COMMAND echo ".globl _ggml_metallib_end" >> ${EMBED_METALLIB_ASSEMBLY}
225-
COMMAND echo "_ggml_metallib_end:" >> ${EMBED_METALLIB_ASSEMBLY}
226-
DEPENDS ${METALLIB_SOURCE}
221+
OUTPUT ${METALLIB_EMBED_ASM}
222+
COMMAND echo "Embedding Metal library"
223+
COMMAND sed -e '/\#include \"ggml-common.h\"/r ${METALLIB_COMMON}' -e '/\#include \"ggml-common.h\"/d' < ${METALLIB_SOURCE} > ${METALLIB_SOURCE_EMBED}
224+
COMMAND echo ".section __DATA,__ggml_metallib" > ${METALLIB_EMBED_ASM}
225+
COMMAND echo ".globl _ggml_metallib_start" >> ${METALLIB_EMBED_ASM}
226+
COMMAND echo "_ggml_metallib_start:" >> ${METALLIB_EMBED_ASM}
227+
COMMAND echo ".incbin \\\"${METALLIB_SOURCE_EMBED}\\\"" >> ${METALLIB_EMBED_ASM}
228+
COMMAND echo ".globl _ggml_metallib_end" >> ${METALLIB_EMBED_ASM}
229+
COMMAND echo "_ggml_metallib_end:" >> ${METALLIB_EMBED_ASM}
230+
DEPENDS ggml-metal.metal ggml-common.h
227231
COMMENT "Generate assembly for embedded Metal library"
228232
)
229233

230-
set(GGML_SOURCES_METAL ${GGML_SOURCES_METAL} ${EMBED_METALLIB_ASSEMBLY})
231-
endif()
232-
233-
if (LLAMA_METAL_SHADER_DEBUG)
234-
# custom command to do the following:
235-
# xcrun -sdk macosx metal -fno-fast-math -c ggml-metal.metal -o ggml-metal.air
236-
# xcrun -sdk macosx metallib ggml-metal.air -o default.metallib
237-
#
238-
# note: this is the only way I found to disable fast-math in Metal. it's ugly, but at least it works
239-
# disabling fast math is needed in order to pass tests/test-backend-ops
240-
# note: adding -fno-inline fixes the tests when using MTL_SHADER_VALIDATION=1
241-
# note: unfortunately, we have to call it default.metallib instead of ggml.metallib
242-
# ref: https://github.com/ggerganov/whisper.cpp/issues/1720
243-
set(XC_FLAGS -fno-fast-math -fno-inline -g)
244-
if (LLAMA_QKK_64)
245-
set(XC_FLAGS ${XC_FLAGS} -DQK_K=64)
234+
set(GGML_SOURCES_METAL ${GGML_SOURCES_METAL} ${METALLIB_EMBED_ASM})
235+
else()
236+
if (LLAMA_METAL_SHADER_DEBUG)
237+
# custom command to do the following:
238+
# xcrun -sdk macosx metal -fno-fast-math -c ggml-metal.metal -o ggml-metal.air
239+
# xcrun -sdk macosx metallib ggml-metal.air -o default.metallib
240+
#
241+
# note: this is the only way I found to disable fast-math in Metal. it's ugly, but at least it works
242+
# disabling fast math is needed in order to pass tests/test-backend-ops
243+
# note: adding -fno-inline fixes the tests when using MTL_SHADER_VALIDATION=1
244+
# note: unfortunately, we have to call it default.metallib instead of ggml.metallib
245+
# ref: https://github.com/ggerganov/whisper.cpp/issues/1720
246+
set(XC_FLAGS -fno-fast-math -fno-inline -g)
247+
else()
248+
set(XC_FLAGS -O3)
246249
endif()
247250

248251
add_custom_command(
249252
OUTPUT ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/default.metallib
250253
COMMAND xcrun -sdk macosx metal ${XC_FLAGS} -c ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal.metal -o ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal.air
251254
COMMAND xcrun -sdk macosx metallib ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal.air -o ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/default.metallib
252-
DEPENDS ggml-metal.metal
255+
COMMAND rm -f ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal.air
256+
COMMAND rm -f ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-common.h
257+
COMMAND rm -f ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal.metal
258+
DEPENDS ggml-metal.metal ggml-common.h
253259
COMMENT "Compiling Metal kernels"
254-
)
260+
)
255261

256262
add_custom_target(
257263
ggml-metal ALL
258264
DEPENDS ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/default.metallib
259-
)
260-
endif()
265+
)
266+
endif() # LLAMA_METAL_EMBED_LIBRARY
261267

262268
set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS}
263269
${FOUNDATION_LIBRARY}

‎Makefile

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -557,15 +557,16 @@ ggml-metal.o: ggml-metal.m ggml-metal.h
557557
$(CC) $(CFLAGS) -c $< -o $@
558558

559559
ifdef LLAMA_METAL_EMBED_LIBRARY
560-
ggml-metal-embed.o: ggml-metal.metal
560+
ggml-metal-embed.o: ggml-metal.metal ggml-common.h
561561
@echo "Embedding Metal library"
562+
@sed -e '/#include "ggml-common.h"/r ggml-common.h' -e '/#include "ggml-common.h"/d' < ggml-metal.metal > ggml-metal-embed.metal
562563
$(eval TEMP_ASSEMBLY=$(shell mktemp))
563-
@echo ".section __DATA, __ggml_metallib" > $(TEMP_ASSEMBLY)
564-
@echo ".globl _ggml_metallib_start" >> $(TEMP_ASSEMBLY)
565-
@echo "_ggml_metallib_start:" >> $(TEMP_ASSEMBLY)
566-
@echo ".incbin \"$<\"" >> $(TEMP_ASSEMBLY)
567-
@echo ".globl _ggml_metallib_end" >> $(TEMP_ASSEMBLY)
568-
@echo "_ggml_metallib_end:" >> $(TEMP_ASSEMBLY)
564+
@echo ".section __DATA, __ggml_metallib" > $(TEMP_ASSEMBLY)
565+
@echo ".globl _ggml_metallib_start" >> $(TEMP_ASSEMBLY)
566+
@echo "_ggml_metallib_start:" >> $(TEMP_ASSEMBLY)
567+
@echo ".incbin \"ggml-metal-embed.metal\"" >> $(TEMP_ASSEMBLY)
568+
@echo ".globl _ggml_metallib_end" >> $(TEMP_ASSEMBLY)
569+
@echo "_ggml_metallib_end:" >> $(TEMP_ASSEMBLY)
569570
@$(AS) $(TEMP_ASSEMBLY) -o $@
570571
@rm -f ${TEMP_ASSEMBLY}
571572
endif

‎ggml-metal.m

Lines changed: 33 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -280,19 +280,33 @@ static void ggml_metal_log(enum ggml_log_level level, const char * format, ...){
280280
id<MTLLibrary> metal_library;
281281

282282
// load library
283+
//
284+
// - first check if the library is embedded
285+
// - then check if the library is in the bundle
286+
// - if not found, load the source and compile it
287+
// - if that fails, return NULL
283288
{
284289
NSBundle * bundle = nil;
285290
#ifdef SWIFT_PACKAGE
286291
bundle = SWIFTPM_MODULE_BUNDLE;
287292
#else
288293
bundle = [NSBundle bundleForClass:[GGMLMetalClass class]];
289294
#endif
295+
290296
NSError * error = nil;
291-
NSString * libPath = [bundle pathForResource:@"default" ofType:@"metallib"];
292-
if (libPath != nil) {
297+
298+
#if GGML_METAL_EMBED_LIBRARY
299+
const bool try_metallib = false;
300+
#else
301+
const bool try_metallib = true;
302+
#endif
303+
304+
NSString * path_lib = [bundle pathForResource:@"default" ofType:@"metallib"];
305+
if (try_metallib && path_lib != nil) {
293306
// pre-compiled library found
294-
NSURL * libURL = [NSURL fileURLWithPath:libPath];
295-
GGML_METAL_LOG_INFO("%s: loading '%s'\n", __func__, [libPath UTF8String]);
307+
NSURL * libURL = [NSURL fileURLWithPath:path_lib];
308+
GGML_METAL_LOG_INFO("%s: loading '%s'\n", __func__, [path_lib UTF8String]);
309+
296310
metal_library = [ctx->device newLibraryWithURL:libURL error:&error];
297311
if (error) {
298312
GGML_METAL_LOG_ERROR("%s: error: %s\n", __func__, [[error description] UTF8String]);
@@ -305,31 +319,34 @@ static void ggml_metal_log(enum ggml_log_level level, const char * format, ...){
305319
extern const char ggml_metallib_start[];
306320
extern const char ggml_metallib_end[];
307321

308-
NSString * src = [[NSString alloc] initWithBytes:ggml_metallib_start length:(ggml_metallib_end-ggml_metallib_start) encoding:NSUTF8StringEncoding];
322+
NSString * src = [[NSString alloc] initWithBytes:ggml_metallib_start length:(ggml_metallib_end-ggml_metallib_start) encoding:NSUTF8StringEncoding];
309323
#else
310324
GGML_METAL_LOG_INFO("%s: default.metallib not found, loading from source\n", __func__);
311325

312-
NSString * sourcePath;
313-
NSString * ggmlMetalPathResources = [[NSProcessInfo processInfo].environment objectForKey:@"GGML_METAL_PATH_RESOURCES"];
326+
NSString * path_source;
327+
NSString * path_resource = [[NSProcessInfo processInfo].environment objectForKey:@"GGML_METAL_PATH_RESOURCES"];
314328

315-
GGML_METAL_LOG_INFO("%s: GGML_METAL_PATH_RESOURCES = %s\n", __func__, ggmlMetalPathResources ? [ggmlMetalPathResources UTF8String] : "nil");
329+
GGML_METAL_LOG_INFO("%s: GGML_METAL_PATH_RESOURCES = %s\n", __func__, path_resource ? [path_resource UTF8String] : "nil");
316330

317-
if (ggmlMetalPathResources) {
318-
sourcePath = [ggmlMetalPathResources stringByAppendingPathComponent:@"ggml-metal.metal"];
331+
if (path_resource) {
332+
path_source = [path_resource stringByAppendingPathComponent:@"ggml-metal.metal"];
319333
} else {
320-
sourcePath = [bundle pathForResource:@"ggml-metal" ofType:@"metal"];
334+
path_source = [bundle pathForResource:@"ggml-metal" ofType:@"metal"];
321335
}
322-
if (sourcePath == nil) {
336+
337+
if (path_source == nil) {
323338
GGML_METAL_LOG_WARN("%s: error: could not use bundle path to find ggml-metal.metal, falling back to trying cwd\n", __func__);
324-
sourcePath = @"ggml-metal.metal";
339+
path_source = @"ggml-metal.metal";
325340
}
326-
GGML_METAL_LOG_INFO("%s: loading '%s'\n", __func__, [sourcePath UTF8String]);
327-
NSString * src = [NSString stringWithContentsOfFile:sourcePath encoding:NSUTF8StringEncoding error:&error];
341+
342+
GGML_METAL_LOG_INFO("%s: loading '%s'\n", __func__, [path_source UTF8String]);
343+
344+
NSString * src = [NSString stringWithContentsOfFile:path_source encoding:NSUTF8StringEncoding error:&error];
328345
if (error) {
329346
GGML_METAL_LOG_ERROR("%s: error: %s\n", __func__, [[error description] UTF8String]);
330347
return NULL;
331348
}
332-
#endif
349+
#endif // GGML_METAL_EMBED_LIBRARY
333350

334351
@autoreleasepool {
335352
// dictionary of preprocessor macros

‎ggml-metal.metal

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,6 @@
44

55
#include <metal_stdlib>
66

7-
#define GGML_COMMON_IMPL_METAL
8-
#include "ggml-common.h"
9-
107
using namespace metal;
118

129
#define MAX(x, y) ((x) > (y) ? (x) : (y))

0 commit comments

Comments
 (0)