diff --git a/ggml.c b/ggml.c index 59e84ab45d120..2a339c42c2bee 100644 --- a/ggml.c +++ b/ggml.c @@ -54,7 +54,7 @@ static LONG atomic_fetch_sub(atomic_int* ptr, LONG dec) { typedef HANDLE pthread_t; typedef DWORD thread_ret_t; -static int pthread_create(pthread_t* out, void* unused, thread_ret_t(*func)(void*), void* arg) { +int pthread_create(pthread_t* out, void* unused, thread_ret_t(*func)(void*), void* arg) { HANDLE handle = CreateThread(NULL, 0, (LPTHREAD_START_ROUTINE) func, arg, 0, NULL); if (handle == NULL) { @@ -65,7 +65,7 @@ static int pthread_create(pthread_t* out, void* unused, thread_ret_t(*func)(void return 0; } -static int pthread_join(pthread_t thread, void* unused) { +int pthread_join(pthread_t thread, void* unused) { return (int) WaitForSingleObject(thread, INFINITE); } diff --git a/ggml.h b/ggml.h index ad962b109ea89..3a51734ccc595 100644 --- a/ggml.h +++ b/ggml.h @@ -773,6 +773,43 @@ int ggml_cpu_has_blas(void); int ggml_cpu_has_sse3(void); int ggml_cpu_has_vsx(void); +// +// threading for non posix systems +// + +#if defined(_WIN32) && !defined(_POSIX_THREADS) +#define WIN32_LEAN_AND_MEAN +#if !defined(min) && !defined(max) +#include +#ifdef min +#undef min +#endif +#ifdef max +#undef max +#endif +#elif defined(min) && defined(max) +#include +#elif !defined(min) +#include +#ifdef max +#undef max +#endif +#elif !defined(max) +#include +#ifdef min +#undef min +#endif +#endif +#else +#include +#endif + +#ifndef _POSIX_THREADS +typedef HANDLE pthread_t; +int pthread_create(pthread_t* out, void* unused, void*(*func)(void*), void* arg); +int pthread_join(pthread_t thread, void* unused); +#endif + #ifdef __cplusplus } #endif diff --git a/llama.cpp b/llama.cpp index 854bb8993fbc5..3d0c2da1a7a66 100644 --- a/llama.cpp +++ b/llama.cpp @@ -306,8 +306,26 @@ struct llama_context_params llama_context_default_params() { // // model loading // +typedef struct the_load_file_to_standby_struct{ +char * fname; +char * addr; +size_t addrlen; +} the_load_file_to_standby_struct; + +int load_file_to_standby(the_load_file_to_standby_struct * the_load_file_to_standby_struct1){ + auto fin = std::ifstream(the_load_file_to_standby_struct1->fname, std::ios::binary); + if (!fin || !the_load_file_to_standby_struct1 || !the_load_file_to_standby_struct1->fname || !the_load_file_to_standby_struct1->addr || !the_load_file_to_standby_struct1->addrlen) { + return -1; + } + while(!fin.eof()) fin.read(the_load_file_to_standby_struct1->addr, the_load_file_to_standby_struct1->addrlen); + return 0; +} static void *mmap_file(const char *fname, uint64_t *mm_length) { +the_load_file_to_standby_struct * the_load_file_to_standby_struct1 = 0; + const size_t readstridelen = 1 << 20; + size_t fnamelen = strlen(fname); + pthread_t threadId; #if defined(_WIN32) && !defined(_POSIX_MAPPED_FILES) HANDLE hFile = CreateFileA(fname, GENERIC_READ, @@ -326,6 +344,24 @@ static void *mmap_file(const char *fname, uint64_t *mm_length) { if (!hMapping) return 0; void *addr = MapViewOfFile(hMapping, FILE_MAP_READ, 0, 0, 0); CloseHandle(hMapping); + hMapping = CreateFileMappingA(INVALID_HANDLE_VALUE, NULL, PAGE_READWRITE, sizeof(the_load_file_to_standby_struct) >> 32, sizeof(the_load_file_to_standby_struct), NULL); + if (hMapping){ + the_load_file_to_standby_struct1 = (the_load_file_to_standby_struct*)MapViewOfFile(hMapping, FILE_MAP_READ | FILE_MAP_WRITE, 0, 0, 0); + the_load_file_to_standby_struct1->fname = 0; + the_load_file_to_standby_struct1->addr = 0; + the_load_file_to_standby_struct1->addrlen = 0; + CloseHandle(hMapping); + hMapping = CreateFileMappingA(INVALID_HANDLE_VALUE, NULL, PAGE_READWRITE, fnamelen >> 32, fnamelen, NULL); + if (hMapping){ + the_load_file_to_standby_struct1->fname = (char*)MapViewOfFile(hMapping, FILE_MAP_READ | FILE_MAP_WRITE, 0, 0, 0); + CloseHandle(hMapping); + hMapping = CreateFileMappingA(INVALID_HANDLE_VALUE, NULL, PAGE_READWRITE, readstridelen >> 32, readstridelen, NULL); + if (hMapping){ + the_load_file_to_standby_struct1->addr = (char*)MapViewOfFile(hMapping, FILE_MAP_READ | FILE_MAP_WRITE, 0, 0, 0); + CloseHandle(hMapping); + } + } + } if (!addr) return 0; #else int fd = open(fname, O_RDONLY); @@ -334,8 +370,21 @@ static void *mmap_file(const char *fname, uint64_t *mm_length) { void *addr = mmap(NULL, length, PROT_READ, MAP_SHARED, fd, 0); close(fd); if (addr == MAP_FAILED) return 0; + the_load_file_to_standby_struct1 = (the_load_file_to_standby_struct*)mmap(NULL, sizeof(the_load_file_to_standby_struct), PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (the_load_file_to_standby_struct1 == MAP_FAILED) the_load_file_to_standby_struct1 = 0; + the_load_file_to_standby_struct1->fname = (char*)mmap(NULL, fnamelen, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (the_load_file_to_standby_struct1->fname == MAP_FAILED) the_load_file_to_standby_struct1->fname = 0; + the_load_file_to_standby_struct1->addr = (char*)mmap(NULL, readstridelen, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (the_load_file_to_standby_struct1->addr == MAP_FAILED) the_load_file_to_standby_struct1->addr = 0; #endif *mm_length = length; + if(the_load_file_to_standby_struct1){ + the_load_file_to_standby_struct1->addrlen = readstridelen; + if(the_load_file_to_standby_struct1->fname){ + memcpy(the_load_file_to_standby_struct1->fname, fname, fnamelen); + } + pthread_create(&threadId, 0, (void*(*)(void*))(&load_file_to_standby), the_load_file_to_standby_struct1); + } return addr; }