Skip to content

Commit 2d099e5

Browse files
slarenslaren
authored
ggml: add names to tensors (#1268)
* ggml: add names to tensors * minor improvements to dot file formatting
1 parent f4cef87 commit 2d099e5

File tree

3 files changed

+67
-19
lines changed

3 files changed

+67
-19
lines changed

‎ggml.c

Lines changed: 40 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -4541,6 +4541,7 @@ struct ggml_tensor * ggml_new_tensor_impl(
45414541
/*.perf_cycles =*/ 0,
45424542
/*.perf_time_us =*/ 0,
45434543
/*.data =*/ (data == NULL && !ctx->no_alloc) ? (void *)(result + 1) : data,
4544+
/*.name =*/ { 0 },
45444545
/*.pad =*/ { 0 },
45454546
};
45464547

@@ -4895,6 +4896,15 @@ float * ggml_get_data_f32(const struct ggml_tensor * tensor) {
48954896
return (float *)(tensor->data);
48964897
}
48974898

4899+
const char * ggml_get_name(const struct ggml_tensor * tensor) {
4900+
return tensor->name;
4901+
}
4902+
4903+
void ggml_set_name(struct ggml_tensor * tensor, const char * name) {
4904+
strncpy(tensor->name, name, sizeof(tensor->name));
4905+
tensor->name[sizeof(tensor->name) - 1] = '\0';
4906+
}
4907+
48984908
struct ggml_tensor * ggml_view_tensor(
48994909
struct ggml_context * ctx,
49004910
const struct ggml_tensor * src) {
@@ -5994,6 +6004,7 @@ struct ggml_tensor * ggml_diag_mask_inf(
59946004
//struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
59956005
struct ggml_tensor * result = ggml_view_tensor(ctx, a);
59966006
struct ggml_tensor * b = ggml_new_i32(ctx, n_past);
6007+
ggml_set_name(b, "n_past");
59976008

59986009
result->op = GGML_OP_DIAG_MASK_INF;
59996010
result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
@@ -6051,6 +6062,7 @@ struct ggml_tensor * ggml_rope(
60516062
((int32_t *) b->data)[0] = n_past;
60526063
((int32_t *) b->data)[1] = n_dims;
60536064
((int32_t *) b->data)[2] = mode;
6065+
ggml_set_name(b, "n_past, n_dims, mode");
60546066

60556067
result->op = GGML_OP_ROPE;
60566068
result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
@@ -12118,10 +12130,16 @@ void ggml_graph_dump_dot(const struct ggml_cgraph * gb, const struct ggml_cgraph
1211812130
snprintf(color, sizeof(color), "white");
1211912131
}
1212012132

12121-
fprintf(fp, " \"%p\" [ \
12122-
style = filled; fillcolor = %s; shape = record; \
12123-
label=\"%d [%" PRId64 ", %" PRId64 "] | <x>%s",
12124-
(void *) node, color,
12133+
fprintf(fp, " \"%p\" [ "
12134+
"style = filled; fillcolor = %s; shape = record; "
12135+
"label=\"",
12136+
(void *) node, color);
12137+
12138+
if (strlen(node->name) > 0) {
12139+
fprintf(fp, "%s |", node->name);
12140+
}
12141+
12142+
fprintf(fp, "%d [%" PRId64 ", %" PRId64 "] | <x>%s",
1212512143
i, node->ne[0], node->ne[1],
1212612144
GGML_OP_SYMBOL[node->op]);
1212712145

@@ -12137,18 +12155,26 @@ label=\"%d [%" PRId64 ", %" PRId64 "] | <x>%s",
1213712155

1213812156
snprintf(color, sizeof(color), "pink");
1213912157

12158+
fprintf(fp, " \"%p\" [ "
12159+
"style = filled; fillcolor = %s; shape = record; "
12160+
"label=\"<x>",
12161+
(void *) node, color);
12162+
12163+
if (strlen(node->name) > 0) {
12164+
fprintf(fp, "%s | ", node->name);
12165+
}
1214012166
if (ggml_nelements(node) == 1) {
12141-
fprintf(fp, " \"%p\" [ \
12142-
style = filled; fillcolor = %s; shape = record; \
12143-
label=\"<x>%.1e\"; ]\n",
12144-
(void *) node, color, (double)ggml_get_f32_1d(node, 0));
12145-
} else {
12146-
fprintf(fp, " \"%p\" [ \
12147-
style = filled; fillcolor = %s; shape = record; \
12148-
label=\"<x>CONST %d [%" PRId64 ", %" PRId64 "]\"; ]\n",
12149-
(void *) node, color,
12150-
i, node->ne[0], node->ne[1]);
12167+
if (node->type == GGML_TYPE_I8 || node->type == GGML_TYPE_I16 || node->type == GGML_TYPE_I32) {
12168+
fprintf(fp, "%d", ggml_get_i32_1d(node, 0));
12169+
}
12170+
else {
12171+
fprintf(fp, "%.1e", (double)ggml_get_f32_1d(node, 0));
12172+
}
12173+
}
12174+
else {
12175+
fprintf(fp, "CONST %d [%" PRId64 ", %" PRId64 "]", i, node->ne[0], node->ne[1]);
1215112176
}
12177+
fprintf(fp, "\"; ]\n");
1215212178
}
1215312179

1215412180
for (int i = 0; i < gb->n_nodes; i++) {

‎ggml.h

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -350,7 +350,10 @@ extern "C" {
350350
int64_t perf_time_us;
351351

352352
void * data;
353-
char padding[8];
353+
354+
char name[32];
355+
356+
char padding[8]; // TODO: remove and add padding to name?
354357
};
355358

356359
// computation graph
@@ -473,6 +476,9 @@ extern "C" {
473476
GGML_API void * ggml_get_data (const struct ggml_tensor * tensor);
474477
GGML_API float * ggml_get_data_f32(const struct ggml_tensor * tensor);
475478

479+
GGML_API const char * ggml_get_name(const struct ggml_tensor * tensor);
480+
GGML_API void ggml_set_name(struct ggml_tensor * tensor, const char * name);
481+
476482
//
477483
// operations on tensors with backpropagation
478484
//

‎llama.cpp

Lines changed: 20 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -659,6 +659,7 @@ struct llama_model_loader {
659659
LLAMA_ASSERT(lt.ne.size() == 1);
660660
tensor = ggml_new_tensor_1d(ggml_ctx, lt.type, lt.ne.at(0));
661661
}
662+
ggml_set_name(tensor, lt.name.c_str());
662663
LLAMA_ASSERT(lt.ggml_tensor == NULL); // if this fails, we called get_tensor twice on the same tensor
663664
lt.ggml_tensor = tensor;
664665
num_ggml_tensors_created++;
@@ -798,6 +799,8 @@ static bool kv_cache_init(
798799

799800
cache.k = ggml_new_tensor_1d(cache.ctx, wtype, n_elements);
800801
cache.v = ggml_new_tensor_1d(cache.ctx, wtype, n_elements);
802+
ggml_set_name(cache.k, "cache_k");
803+
ggml_set_name(cache.v, "cache_v");
801804

802805
return true;
803806
}
@@ -1084,6 +1087,7 @@ static bool llama_eval_internal(
10841087
gf.n_threads = N >= 32 && ggml_cpu_has_blas() && !ggml_cpu_has_gpublas() ? 1 : n_threads;
10851088

10861089
struct ggml_tensor * embd = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
1090+
ggml_set_name(embd, "embd");
10871091
memcpy(embd->data, tokens, N*ggml_element_size(embd));
10881092

10891093
struct ggml_tensor * inpL = ggml_get_rows(ctx0, model.tok_embeddings, embd);
@@ -1110,6 +1114,8 @@ static bool llama_eval_internal(
11101114
// compute Q and K and RoPE them
11111115
struct ggml_tensor * Qcur = ggml_rope(ctx0, ggml_reshape_3d(ctx0, ggml_mul_mat(ctx0, model.layers[il].wq, cur), n_embd/n_head, n_head, N), n_past, n_rot, 0);
11121116
struct ggml_tensor * Kcur = ggml_rope(ctx0, ggml_reshape_3d(ctx0, ggml_mul_mat(ctx0, model.layers[il].wk, cur), n_embd/n_head, n_head, N), n_past, n_rot, 0);
1117+
ggml_set_name(Qcur, "Qcur");
1118+
ggml_set_name(Kcur, "Kcur");
11131119

11141120
// store key and value to memory
11151121
{
@@ -1130,28 +1136,34 @@ static bool llama_eval_internal(
11301136
ggml_permute(ctx0,
11311137
Qcur,
11321138
0, 2, 1, 3);
1139+
ggml_set_name(Q, "Q");
11331140

11341141
struct ggml_tensor * K =
11351142
ggml_permute(ctx0,
11361143
ggml_reshape_3d(ctx0,
11371144
ggml_view_1d(ctx0, kv_self.k, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(kv_self.k)*n_embd),
11381145
n_embd/n_head, n_head, n_past + N),
11391146
0, 2, 1, 3);
1147+
ggml_set_name(K, "K");
11401148

11411149
// K * Q
11421150
struct ggml_tensor * KQ = ggml_mul_mat(ctx0, K, Q);
1151+
ggml_set_name(KQ, "KQ");
11431152

11441153
// KQ_scaled = KQ / sqrt(n_embd/n_head)
1145-
struct ggml_tensor * KQ_scaled =
1146-
ggml_scale(ctx0,
1147-
KQ,
1148-
ggml_new_f32(ctx0, 1.0f/sqrtf(float(n_embd)/n_head)));
1154+
struct ggml_tensor * KQ_scale = ggml_new_f32(ctx0, 1.0f/sqrtf(float(n_embd)/n_head));
1155+
ggml_set_name(KQ_scale, "1/sqrt(n_embd/n_head)");
1156+
1157+
struct ggml_tensor * KQ_scaled = ggml_scale(ctx0, KQ, KQ_scale);
1158+
ggml_set_name(KQ_scaled, "KQ_scaled");
11491159

11501160
// KQ_masked = mask_past(KQ_scaled)
11511161
struct ggml_tensor * KQ_masked = ggml_diag_mask_inf(ctx0, KQ_scaled, n_past);
1162+
ggml_set_name(KQ_masked, "KQ_masked");
11521163

11531164
// KQ = soft_max(KQ_masked)
11541165
struct ggml_tensor * KQ_soft_max = ggml_soft_max(ctx0, KQ_masked);
1166+
ggml_set_name(KQ_soft_max, "KQ_soft_max");
11551167

11561168
// split cached V into n_head heads
11571169
struct ggml_tensor * V =
@@ -1160,9 +1172,11 @@ static bool llama_eval_internal(
11601172
n_ctx*ggml_element_size(kv_self.v),
11611173
n_ctx*ggml_element_size(kv_self.v)*n_embd/n_head,
11621174
il*n_ctx*ggml_element_size(kv_self.v)*n_embd);
1175+
ggml_set_name(V, "V");
11631176

11641177
#if 1
11651178
struct ggml_tensor * KQV = ggml_mul_mat(ctx0, V, KQ_soft_max);
1179+
ggml_set_name(KQV, "KQV");
11661180
#else
11671181
// make V contiguous in memory to speed up the matmul, however we waste time on the copy
11681182
// on M1 this is faster for the perplexity computation, but ~5% slower for the single-token generation
@@ -1173,11 +1187,13 @@ static bool llama_eval_internal(
11731187

11741188
// KQV_merged = KQV.permute(0, 2, 1, 3)
11751189
struct ggml_tensor * KQV_merged = ggml_permute(ctx0, KQV, 0, 2, 1, 3);
1190+
ggml_set_name(KQV_merged, "KQV_merged");
11761191

11771192
// cur = KQV_merged.contiguous().view(n_embd, N)
11781193
cur = ggml_cpy(ctx0,
11791194
KQV_merged,
11801195
ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, N));
1196+
ggml_set_name(cur, "KQV_merged_contiguous");
11811197

11821198
// projection (no bias)
11831199
cur = ggml_mul_mat(ctx0,

0 commit comments

Comments
 (0)