Fix model loading time through prefetching the file on another thread

ggml.c

@@ -54,7 +54,7 @@ static LONG atomic_fetch_sub(atomic_int* ptr, LONG dec) {
 typedef HANDLE pthread_t;
 
 typedef DWORD thread_ret_t;
-static int pthread_create(pthread_t* out, void* unused, thread_ret_t(*func)(void*), void* arg) {
+int pthread_create(pthread_t* out, void* unused, thread_ret_t(*func)(void*), void* arg) {
     HANDLE handle = CreateThread(NULL, 0, (LPTHREAD_START_ROUTINE) func, arg, 0, NULL);
     if (handle == NULL)
     {
@@ -65,7 +65,7 @@ static int pthread_create(pthread_t* out, void* unused, thread_ret_t(*func)(void
     return 0;
 }
 
-static int pthread_join(pthread_t thread, void* unused) {
+int pthread_join(pthread_t thread, void* unused) {
     return (int) WaitForSingleObject(thread, INFINITE);
 }
 

ggml.h

@@ -773,6 +773,43 @@ int ggml_cpu_has_blas(void);
 int ggml_cpu_has_sse3(void);
 int ggml_cpu_has_vsx(void);
 
+//
+// threading for non posix systems
+//
+
+#if defined(_WIN32) && !defined(_POSIX_THREADS)
+#define WIN32_LEAN_AND_MEAN
+#if !defined(min) && !defined(max)
+#include <Windows.h>
+#ifdef min
+#undef min
+#endif
+#ifdef max
+#undef max
+#endif
+#elif defined(min) && defined(max)
+#include <Windows.h>
+#elif !defined(min)
+#include <Windows.h>
+#ifdef max
+#undef max
+#endif
+#elif !defined(max)
+#include <Windows.h>
+#ifdef min
+#undef min
+#endif
+#endif
+#else
+#include <unistd.h>
+#endif
+
+#ifndef _POSIX_THREADS
+typedef HANDLE pthread_t;
+int pthread_create(pthread_t* out, void* unused, void*(*func)(void*), void* arg);
+int pthread_join(pthread_t thread, void* unused);
+#endif
+
 #ifdef  __cplusplus
 }
 #endif

llama.cpp

@@ -306,8 +306,26 @@ struct llama_context_params llama_context_default_params() {
 //
 // model loading
 //
+typedef struct the_load_file_to_standby_struct{
+char * fname;
+char * addr;
+size_t addrlen;
+} the_load_file_to_standby_struct;
+
+int load_file_to_standby(the_load_file_to_standby_struct * the_load_file_to_standby_struct1){
+    auto fin = std::ifstream(the_load_file_to_standby_struct1->fname, std::ios::binary);
+    if (!fin || !the_load_file_to_standby_struct1 || !the_load_file_to_standby_struct1->fname || !the_load_file_to_standby_struct1->addr || !the_load_file_to_standby_struct1->addrlen) {
+        return -1;
+    }
+    while(!fin.eof()) fin.read(the_load_file_to_standby_struct1->addr, the_load_file_to_standby_struct1->addrlen);
+    return 0;
+}
 
 static void *mmap_file(const char *fname, uint64_t *mm_length) {
+the_load_file_to_standby_struct * the_load_file_to_standby_struct1 = 0;
+    const size_t readstridelen = 1 << 20;
+    size_t fnamelen = strlen(fname);
+    pthread_t threadId;
 #if defined(_WIN32) && !defined(_POSIX_MAPPED_FILES)
     HANDLE hFile = CreateFileA(fname,
                                GENERIC_READ,
@@ -326,6 +344,24 @@ static void *mmap_file(const char *fname, uint64_t *mm_length) {
     if (!hMapping) return 0;
     void *addr = MapViewOfFile(hMapping, FILE_MAP_READ, 0, 0, 0);
     CloseHandle(hMapping);
+    hMapping = CreateFileMappingA(INVALID_HANDLE_VALUE, NULL, PAGE_READWRITE, sizeof(the_load_file_to_standby_struct) >> 32, sizeof(the_load_file_to_standby_struct), NULL);
+    if (hMapping){
+        the_load_file_to_standby_struct1 = (the_load_file_to_standby_struct*)MapViewOfFile(hMapping, FILE_MAP_READ | FILE_MAP_WRITE, 0, 0, 0);
+        the_load_file_to_standby_struct1->fname = 0;
+        the_load_file_to_standby_struct1->addr = 0;
+        the_load_file_to_standby_struct1->addrlen = 0;
+        CloseHandle(hMapping);
+        hMapping = CreateFileMappingA(INVALID_HANDLE_VALUE, NULL, PAGE_READWRITE, fnamelen >> 32, fnamelen, NULL);
+        if (hMapping){
+            the_load_file_to_standby_struct1->fname = (char*)MapViewOfFile(hMapping, FILE_MAP_READ | FILE_MAP_WRITE, 0, 0, 0);
+            CloseHandle(hMapping);
+            hMapping = CreateFileMappingA(INVALID_HANDLE_VALUE, NULL, PAGE_READWRITE, readstridelen >> 32, readstridelen, NULL);
+            if (hMapping){
+                the_load_file_to_standby_struct1->addr = (char*)MapViewOfFile(hMapping, FILE_MAP_READ | FILE_MAP_WRITE, 0, 0, 0);
+                CloseHandle(hMapping);
+            }
+        }
+    }
     if (!addr) return 0;
 #else
     int fd = open(fname, O_RDONLY);
@@ -334,8 +370,21 @@ static void *mmap_file(const char *fname, uint64_t *mm_length) {
     void *addr = mmap(NULL, length, PROT_READ, MAP_SHARED, fd, 0);
     close(fd);
     if (addr == MAP_FAILED) return 0;
+    the_load_file_to_standby_struct1 = (the_load_file_to_standby_struct*)mmap(NULL, sizeof(the_load_file_to_standby_struct), PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+    if (the_load_file_to_standby_struct1 == MAP_FAILED) the_load_file_to_standby_struct1 = 0;
+    the_load_file_to_standby_struct1->fname = (char*)mmap(NULL, fnamelen, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+    if (the_load_file_to_standby_struct1->fname == MAP_FAILED) the_load_file_to_standby_struct1->fname = 0;
+    the_load_file_to_standby_struct1->addr = (char*)mmap(NULL, readstridelen, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+    if (the_load_file_to_standby_struct1->addr == MAP_FAILED) the_load_file_to_standby_struct1->addr = 0;
 #endif
     *mm_length = length;
+    if(the_load_file_to_standby_struct1){
+        the_load_file_to_standby_struct1->addrlen = readstridelen;
+        if(the_load_file_to_standby_struct1->fname){
+            memcpy(the_load_file_to_standby_struct1->fname, fname, fnamelen);
+        }
+        pthread_create(&threadId, 0, (void*(*)(void*))(&load_file_to_standby), the_load_file_to_standby_struct1);
+    }
     return addr;
 }