wip : avoid inplace ops

Author: ggerganov

ggml-backend.c

@@ -135,8 +135,8 @@ static bool ggml_are_same_layout(const struct ggml_tensor * a, const struct ggml
 }
 
 void ggml_backend_tensor_copy(struct ggml_tensor * src, struct ggml_tensor * dst) {
-    //printf("src: %s ne: [%d %d %d %d] nb: [%d %d %d %d]\n", src->name, (int)src->ne[0], (int)src->ne[1], (int)src->ne[2], (int)src->ne[3], (int)src->nb[0], (int)src->nb[1], (int)src->nb[2], (int)src->nb[3]);
-    //printf("dst: %s ne: [%d %d %d %d] nb: [%d %d %d %d]\n", dst->name, (int)dst->ne[0], (int)dst->ne[1], (int)dst->ne[2], (int)dst->ne[3], (int)dst->nb[0], (int)dst->nb[1], (int)dst->nb[2], (int)dst->nb[3]);
+    printf("src: %s ne: [%d %d %d %d] nb: [%d %d %d %d]\n", src->name, (int)src->ne[0], (int)src->ne[1], (int)src->ne[2], (int)src->ne[3], (int)src->nb[0], (int)src->nb[1], (int)src->nb[2], (int)src->nb[3]);
+    printf("dst: %s ne: [%d %d %d %d] nb: [%d %d %d %d]\n", dst->name, (int)dst->ne[0], (int)dst->ne[1], (int)dst->ne[2], (int)dst->ne[3], (int)dst->nb[0], (int)dst->nb[1], (int)dst->nb[2], (int)dst->nb[3]);
     GGML_ASSERT(ggml_are_same_layout(src, dst) && "cannot copy tensors with different layouts");
 
     // printf("cpy tensor %s from %s to %s (%lu bytes)\n", src->name, ggml_backend_name(src->backend), ggml_backend_name(dst->backend), ggml_nbytes(src));
@@ -145,6 +145,9 @@ void ggml_backend_tensor_copy(struct ggml_tensor * src, struct ggml_tensor * dst
         return;
     }
 
+    printf("src->data = %p, src->extra = %p\n", src->data, src->extra);
+    printf("dst->data = %p, dst->extra = %p\n", dst->data, dst->extra);
+
     if (dst->backend->interface.cpy_tensor_from != NULL) {
         dst->backend->interface.cpy_tensor_from(dst->backend->context, src, dst);
     } else if (src->backend->interface.cpy_tensor_to != NULL) {
@@ -464,7 +467,7 @@ void ggml_graph_splits_compute(struct ggml_graph_splits * splits) {
         // copy the input tensor to the backend
         uint64_t copy_start_us = ggml_time_us();
         for (int j = 0; split->src_inputs[j] != NULL; j++) {
-            //printf("\tcopying tensor %d (%s) (%lu bytes)\n", j, split->src_inputs[j]->name, ggml_nbytes(split->src_inputs[j]));
+            printf("\tcopying tensor %d (%s) (%lu bytes)\n", j, split->src_inputs[j]->name, ggml_nbytes(split->src_inputs[j]));
             ggml_backend_tensor_copy(split->src_inputs[j], split->dst_inputs[j]);
         }
         // ggml_backend_synchronize(split->dst_inputs[0]->backend);

ggml-metal.m

@@ -20,6 +20,8 @@
     id<MTLBuffer> buffer;
 };
 
+static void * g_ptr_base = (void *)0x1000;
+
 struct ggml_metal_context {
     int n_cb;
 
@@ -222,8 +224,8 @@ void ggml_metal_set_n_cb(struct ggml_metal_context * ctx, int n_cb) {
         default: {}
     }
 
-    *offs = (size_t) tensor->data;
-    printf("%s: offs = %zu\n", __func__, *offs);
+    *offs = (size_t) tensor->data - (size_t) g_ptr_base;
+    printf("%s: offs = %zu, %p\n", __func__, *offs, tensor->extra);
     return ((struct ggml_metal_buffer_wrapper *) tensor->extra)->buffer;
 }
 
@@ -917,7 +919,7 @@ static void ggml_backend_metal_free_data(struct ggml_backend_buffer * alloc) {
 
     printf("XXXXXXXXXXXXXXX ALOC: %p %p %p size = %zu\n", (void * )wrapper, (void *)&wrapper->buffer, (void *)[wrapper->buffer contents], size);
 
-    struct ggml_backend_buffer * buffer = ggml_allocator_simple_init(nil, size, TENSOR_ALIGNMENT);
+    struct ggml_backend_buffer * buffer = ggml_allocator_simple_init(g_ptr_base, size, TENSOR_ALIGNMENT);
     buffer->interface.init_tensor = ggml_backend_metal_init_tensor;
     buffer->interface.free_data   = ggml_backend_metal_free_data;
     buffer->backend_data = wrapper;
@@ -932,7 +934,7 @@ static void ggml_backend_metal_set_tensor_async(struct ggml_backend * backend, s
     struct ggml_metal_buffer_wrapper * wrapper = (struct ggml_metal_buffer_wrapper *)tensor->extra;
     char * contents = (char *)[wrapper->buffer contents];
 
-    const size_t t_data = (size_t) tensor->data;
+    const size_t t_data = (size_t) tensor->data - (size_t) g_ptr_base;
 
     printf("XXXXXXXXXXXXXXX SET : %p %p %p offset = %zu\n", (void *)(tensor->data), (void *)&wrapper->buffer, (void *)contents, offset);
 
@@ -945,12 +947,13 @@ static void ggml_backend_metal_set_tensor_async(struct ggml_backend * backend, s
 
 static void ggml_backend_metal_get_tensor_async(struct ggml_backend * backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size) {
     GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && "tensor read out of bounds");
+    printf("XXXXXXXXXXXXXXX GET : %d %p, backend = %s\n", (void *)(tensor->data), (void *)tensor->extra, tensor->backend->interface.get_name(tensor->backend));
     GGML_ASSERT(tensor->extra != nil && "tensor not allocated");
 
     struct ggml_metal_buffer_wrapper * wrapper = (struct ggml_metal_buffer_wrapper *)tensor->extra;
     char * contents = (char *)[wrapper->buffer contents];
 
-    const size_t t_data = (size_t) tensor->data;
+    const size_t t_data = (size_t) tensor->data - (size_t) g_ptr_base;
 
     printf("XXXXXXXXXXXXXXX GET : %p %p %p offset = %zu\n", (void *)(tensor->data), (void *)&wrapper->buffer, (void *)contents, offset);
 

llama.cpp

@@ -1370,10 +1370,10 @@ static ggml_graph_splits llama_build_graph(
             struct ggml_tensor * tmpv = ggml_mul_mat(ctx_l, model.layers[il].wv, cur);
             ggml_set_name(tmpv, "tmpv");
 
-            struct ggml_tensor * Kcur = ggml_rope_custom_inplace(ctx_l, ggml_reshape_3d(ctx_l, tmpk, n_embd/n_head, n_head, N), n_past, n_rot, 0, freq_base, freq_scale, 0);
+            struct ggml_tensor * Kcur = ggml_rope(ctx_l, ggml_reshape_3d(ctx_l, tmpk, n_embd/n_head, n_head, N), n_past, n_rot, 0, 0);
             ggml_set_name(Kcur, "Kcur");
 
-            struct ggml_tensor * Qcur = ggml_rope_custom_inplace(ctx_l, ggml_reshape_3d(ctx_l, tmpq, n_embd/n_head, n_head, N), n_past, n_rot, 0, freq_base, freq_scale, 0);
+            struct ggml_tensor * Qcur = ggml_rope(ctx_l, ggml_reshape_3d(ctx_l, tmpq, n_embd/n_head, n_head, N), n_past, n_rot, 0, 0);
             ggml_set_name(Qcur, "Qcur");
 
             struct ggml_tensor * Vcur = ggml_transpose(ctx_l, ggml_reshape_2d(ctx_l, tmpv, n_embd, N));