change batch.logits to batch.output

Septa2112 · Septa2112 · commit cbb5dd7b12fa · 2024-08-16T17:40:51.000+08:00
diff --git a/common/common.cpp b/common/common.cpp
@@ -2666,7 +2666,7 @@ void llama_batch_add(
     for (size_t i = 0; i < seq_ids.size(); ++i) {
         batch.seq_id[batch.n_tokens][i] = seq_ids[i];
     }
-    batch.logits  [batch.n_tokens] = logits;
+    batch.output  [batch.n_tokens] = logits;
 
     batch.n_tokens++;
 }
diff --git a/common/log.h b/common/log.h
@@ -686,7 +686,7 @@ inline std::string LOG_BATCH_TOSTR_PRETTY(const C & ctx, const B & batch)
             << ":pos " << std::to_string(batch.pos[i])
             << ":n_seq_id  " << std::to_string(batch.n_seq_id[i])
             << ":seq_id " << std::to_string(batch.seq_id[i][0])
-            << ":logits " << std::to_string(batch.logits[i]);
+            << ":logits " << std::to_string(batch.output[i]);
     }
     buf << " ]";
 
diff --git a/examples/batched-bench/batched-bench.cpp b/examples/batched-bench/batched-bench.cpp
@@ -94,7 +94,7 @@ int main(int argc, char ** argv) {
                 batch.pos      + i,
                 batch.n_seq_id + i,
                 batch.seq_id   + i,
-                batch.logits   + i,
+                batch.output   + i,
                 0, 0, 0, // unused
             };
 
@@ -149,7 +149,7 @@ int main(int argc, char ** argv) {
                         llama_batch_add(batch, 0, i, { j }, false);
                     }
                 }
-                batch.logits[batch.n_tokens - 1] = true;
+                batch.output[batch.n_tokens - 1] = true;
 
                 const auto t_pp_start = ggml_time_us();
 
diff --git a/examples/batched.swift/Sources/main.swift b/examples/batched.swift/Sources/main.swift
@@ -86,11 +86,11 @@ for (i, token) in tokens.enumerated() {
     if let seq_id = batch.seq_id[i] {
         seq_id[0] = 0
     }
-    batch.logits[i] = 0
+    batch.output[i] = 0
 }
 
 // llama_decode will output logits only for the last token of the prompt
-batch.logits[Int(batch.n_tokens) - 1] = 1
+batch.output[Int(batch.n_tokens) - 1] = 1
 
 if llama_decode(context, batch) != 0 {
     print("llama_decode() failed")
@@ -178,7 +178,7 @@ while n_cur <= n_len {
         if let seq_id = batch.seq_id[Int(batch.n_tokens)] {
             seq_id[0] = Int32(i)
         }
-        batch.logits[Int(batch.n_tokens)] = 1
+        batch.output[Int(batch.n_tokens)] = 1
 
         i_batch[i] = batch.n_tokens
 
diff --git a/examples/batched/batched.cpp b/examples/batched/batched.cpp
@@ -122,7 +122,7 @@ int main(int argc, char ** argv) {
     }
 
     // llama_decode will output logits only for the last token of the prompt
-    batch.logits[batch.n_tokens - 1] = true;
+    batch.output[batch.n_tokens - 1] = true;
 
     if (llama_decode(ctx, batch) != 0) {
         LOG_TEE("%s: llama_decode() failed\n", __func__);
diff --git a/examples/embedding/embedding.cpp b/examples/embedding/embedding.cpp
@@ -52,7 +52,7 @@ static void batch_decode(llama_context * ctx, llama_batch & batch, float * outpu
     }
 
     for (int i = 0; i < batch.n_tokens; i++) {
-        if (!batch.logits[i]) {
+        if (!batch.output[i]) {
             continue;
         }
 
diff --git a/examples/gritlm/gritlm.cpp b/examples/gritlm/gritlm.cpp
@@ -102,21 +102,21 @@ static std::string generate(llama_context * ctx, const std::string & prompt, boo
     llama_set_embeddings(ctx, false);
     llama_set_causal_attn(ctx, true);
 
-    llama_batch bat = llama_batch_init(llama_n_batch(ctx), 0, 1);
+    llama_batch batch = llama_batch_init(llama_n_batch(ctx), 0, 1);
 
     std::vector<llama_token> inputs = llama_tokenize(mdl, prompt, false, true);
     int32_t i_current_token = 0;
 
     while (true) {
-        llama_batch_clear(bat);
+        llama_batch_clear(batch);
         auto n_inputs = (int32_t)inputs.size();
         for (int32_t i = 0; i < n_inputs; i++) {
-            llama_batch_add(bat, inputs[i], i_current_token++, { 0 }, i == n_inputs - 1);
+            llama_batch_add(batch, inputs[i], i_current_token++, { 0 }, i == n_inputs - 1);
         }
         inputs.clear();
 
-        llama_decode(ctx, bat);
-        auto logits = llama_get_logits_ith(ctx, bat.n_tokens - 1);
+        llama_decode(ctx, batch);
+        auto logits = llama_get_logits_ith(ctx, batch.n_tokens - 1);
 
         auto candidates = std::vector<llama_token_data>(llama_n_vocab(mdl));
         auto n_candidates = (int32_t)candidates.size();
@@ -145,7 +145,7 @@ static std::string generate(llama_context * ctx, const std::string & prompt, boo
         std::printf("\n");
     }
 
-    llama_batch_free(bat);
+    llama_batch_free(batch);
 
     return result;
 }
diff --git a/examples/imatrix/imatrix.cpp b/examples/imatrix/imatrix.cpp
@@ -513,7 +513,7 @@ static bool compute_imatrix(llama_context * ctx, const gpt_params & params) {
                 tokens[batch_start] = llama_token_bos(llama_get_model(ctx));
             }
 
-            // TODO: use batch.logits to save computations instead of relying on logits_all == true
+            // TODO: use batch.output to save computations instead of relying on logits_all == true
             if (llama_decode(ctx, llama_batch_get_one(tokens.data() + batch_start, batch_size, j * n_batch, 0))) {
                 fprintf(stderr, "%s : failed to eval\n", __func__);
                 return false;
diff --git a/examples/llama.android/llama/src/main/cpp/llama-android.cpp b/examples/llama.android/llama/src/main/cpp/llama-android.cpp
@@ -193,7 +193,7 @@ Java_android_llama_cpp_LLamaAndroid_bench_1model(
             llama_batch_add(*batch, 0, i, { 0 }, false);
         }
 
-        batch->logits[batch->n_tokens - 1] = true;
+        batch->output[batch->n_tokens - 1] = true;
         llama_kv_cache_clear(context);
 
         const auto t_pp_start = ggml_time_us();
@@ -306,7 +306,7 @@ Java_android_llama_cpp_LLamaAndroid_new_1batch(JNIEnv *, jobject, jint n_tokens,
     for (int i = 0; i < n_tokens; ++i) {
         batch->seq_id[i] = (llama_seq_id *) malloc(sizeof(llama_seq_id) * n_seq_max);
     }
-    batch->logits   = (int8_t *)        malloc(sizeof(int8_t)         * n_tokens);
+    batch->output   = (int8_t *)        malloc(sizeof(int8_t)         * n_tokens);
 
     return reinterpret_cast<jlong>(batch);
 }
@@ -363,7 +363,7 @@ Java_android_llama_cpp_LLamaAndroid_completion_1init(
     }
 
     // llama_decode will output logits only for the last token of the prompt
-    batch->logits[batch->n_tokens - 1] = true;
+    batch->output[batch->n_tokens - 1] = true;
 
     if (llama_decode(context, *batch) != 0) {
         LOGe("llama_decode() failed");
diff --git a/examples/llama.swiftui/llama.cpp.swift/LibLlama.swift b/examples/llama.swiftui/llama.cpp.swift/LibLlama.swift
@@ -16,7 +16,7 @@ func llama_batch_add(_ batch: inout llama_batch, _ id: llama_token, _ pos: llama
     for i in 0..<seq_ids.count {
         batch.seq_id[Int(batch.n_tokens)]![Int(i)] = seq_ids[i]
     }
-    batch.logits  [Int(batch.n_tokens)] = logits ? 1 : 0
+    batch.output  [Int(batch.n_tokens)] = logits ? 1 : 0
 
     batch.n_tokens += 1
 }
@@ -132,7 +132,7 @@ actor LlamaContext {
             let i = Int(i1)
             llama_batch_add(&batch, tokens_list[i], Int32(i), [0], false)
         }
-        batch.logits[Int(batch.n_tokens) - 1] = 1 // true
+        batch.output[Int(batch.n_tokens) - 1] = 1 // true
 
         if llama_decode(context, batch) != 0 {
             print("llama_decode() failed")
@@ -214,7 +214,7 @@ actor LlamaContext {
             for i in 0..<n_tokens {
                 llama_batch_add(&batch, 0, Int32(i), [0], false)
             }
-            batch.logits[Int(batch.n_tokens) - 1] = 1 // true
+            batch.output[Int(batch.n_tokens) - 1] = 1 // true
 
             llama_kv_cache_clear(context)
 
diff --git a/examples/parallel/parallel.cpp b/examples/parallel/parallel.cpp
@@ -265,7 +265,7 @@ int main(int argc, char ** argv) {
 
                     // extract the logits only for the last token
                     if (batch.n_tokens > 0) {
-                        batch.logits[batch.n_tokens - 1] = true;
+                        batch.output[batch.n_tokens - 1] = true;
                     }
 
                     client.n_prompt  = tokens_prompt.size();
@@ -308,7 +308,7 @@ int main(int argc, char ** argv) {
                 batch.pos      + i,
                 batch.n_seq_id + i,
                 batch.seq_id   + i,
-                batch.logits   + i,
+                batch.output   + i,
                 0, 0, 0, // unused
             };
 
diff --git a/examples/passkey/passkey.cpp b/examples/passkey/passkey.cpp
@@ -140,7 +140,7 @@ int main(int argc, char ** argv) {
         }
 
         if (i + n_batch >= n_tokens_all) {
-            batch.logits[batch.n_tokens - 1] = true;
+            batch.output[batch.n_tokens - 1] = true;
         }
 
         if (llama_decode(ctx, batch) != 0) {
@@ -174,7 +174,7 @@ int main(int argc, char ** argv) {
         }
 
         if (i + n_batch >= n_tokens_all) {
-            batch.logits[batch.n_tokens - 1] = true;
+            batch.output[batch.n_tokens - 1] = true;
         }
 
         if (llama_decode(ctx, batch) != 0) {
diff --git a/examples/perplexity/perplexity.cpp b/examples/perplexity/perplexity.cpp
@@ -407,7 +407,7 @@ static results_perplexity perplexity_v2(llama_context * ctx, const gpt_params &
             const int batch_size  = std::min(end - batch_start, n_batch);
 
             //fprintf(stderr, "    Batch %d: starts at %d, size is %d, n_past is %d\n",j,batch_start,batch_size,j * n_batch);
-            // TODO: use llama_batch.logits instead of relying on logits_all == true
+            // TODO: use llama_batch.output instead of relying on logits_all == true
             if (llama_decode(ctx, llama_batch_get_one(tokens.data() + batch_start, batch_size, j * n_batch, 0))) {
                 //fprintf(stderr, "%s : failed to eval\n", __func__);
                 return {tokens, -1, logit_history, prob_history};
@@ -601,9 +601,9 @@ static results_perplexity perplexity(llama_context * ctx, const gpt_params & par
                     batch.pos     [idx]    = j*n_batch + k;
                     batch.n_seq_id[idx]    = 1;
                     batch.seq_id  [idx][0] = seq;
-                    batch.logits  [idx]    = batch.pos[idx] >= first ? 1 : 0;
+                    batch.output  [idx]    = batch.pos[idx] >= first ? 1 : 0;
 
-                    n_outputs += batch.logits[idx] != 0;
+                    n_outputs += batch.output[idx] != 0;
                 }
                 batch.n_tokens += batch_size;
 
@@ -697,7 +697,7 @@ static bool decode_helper(llama_context * ctx, llama_batch & batch, std::vector<
             batch.pos      + i,
             batch.n_seq_id + i,
             batch.seq_id   + i,
-            batch.logits   + i,
+            batch.output   + i,
             0, 0, 0, // unused
         };
 
@@ -709,7 +709,7 @@ static bool decode_helper(llama_context * ctx, llama_batch & batch, std::vector<
 
         int n_outputs = 0;
         for (int i = 0; i < n_tokens; ++i) {
-            n_outputs += batch_view.logits[i] != 0;
+            n_outputs += batch_view.output[i] != 0;
         }
 
         memcpy(batch_logits.data() + prev_outputs*n_vocab, llama_get_logits(ctx), n_outputs*n_vocab*sizeof(float));
@@ -917,7 +917,7 @@ static void hellaswag_score(llama_context * ctx, const gpt_params & params) {
             for (size_t i = 0; i < hs_cur.common_prefix; ++i) {
                 llama_batch_add(batch, hs_cur.seq_tokens[0][i], i, { s0 + 0, s0 + 1, s0 + 2, s0 + 3 }, false);
             }
-            batch.logits[batch.n_tokens - 1] = true; // we need logits for the last token of the common prefix
+            batch.output[batch.n_tokens - 1] = true; // we need logits for the last token of the common prefix
             n_logits += 1;
 
             for (int s = 0; s < 4; ++s) {
@@ -1196,7 +1196,7 @@ static void winogrande_score(llama_context * ctx, const gpt_params & params) {
             for (size_t i = 0; i < data[i1].common_prefix; ++i) {
                 llama_batch_add(batch, data[i1].seq_tokens[0][i], i, { s0 + 0, s0 + 1 }, false);
             }
-            batch.logits[batch.n_tokens - 1] = true;
+            batch.output[batch.n_tokens - 1] = true;
             n_logits += 1;
 
             for (int s = 0; s < 2; ++s) {
@@ -1565,7 +1565,7 @@ static void multiple_choice_score(llama_context * ctx, const gpt_params & params
                 //llama_batch_add(batch, cur_task.seq_tokens[0][i], i, { s0 + 0, s0 + 1, s0 + 2, s0 + 3}, false);
                 llama_batch_add(batch, cur_task.seq_tokens[0][i], i, batch_indeces, false);
             }
-            batch.logits[batch.n_tokens - 1] = true; // we need logits for the last token of the common prefix
+            batch.output[batch.n_tokens - 1] = true; // we need logits for the last token of the common prefix
             n_logits += 1;
 
             for (int s = 0; s < int(cur_task.seq_tokens.size()); ++s) {
@@ -1794,7 +1794,7 @@ static void kl_divergence(llama_context * ctx, const gpt_params & params) {
                 tokens[batch_start] = llama_token_bos(llama_get_model(ctx));
             }
 
-            // TODO: use llama_batch.logits instead of relying on logits_all == true
+            // TODO: use llama_batch.output instead of relying on logits_all == true
             if (llama_decode(ctx, llama_batch_get_one(tokens.data() + batch_start, batch_size, j * n_batch, 0))) {
                 fprintf(stderr, "%s : failed to eval\n", __func__);
                 return;
diff --git a/examples/retrieval/retrieval.cpp b/examples/retrieval/retrieval.cpp
@@ -91,7 +91,7 @@ static void batch_decode(llama_context * ctx, llama_batch & batch, float * outpu
     }
 
     for (int i = 0; i < batch.n_tokens; i++) {
-        if (!batch.logits[i]) {
+        if (!batch.output[i]) {
             continue;
         }
 
diff --git a/examples/server/server.cpp b/examples/server/server.cpp
@@ -1480,7 +1480,7 @@ struct server_context {
         std::vector<float> embd_res(n_embd, 0.0f);
 
         for (int i = 0; i < batch.n_tokens; ++i) {
-            if (!batch.logits[i] || batch.seq_id[i][0] != slot.id + 1) {
+            if (!batch.output[i] || batch.seq_id[i][0] != slot.id + 1) {
                 continue;
             }
 
@@ -2269,7 +2269,7 @@ struct server_context {
                         GGML_ASSERT(batch.n_tokens > 0);
 
                         // extract the logits only for the last token
-                        batch.logits[batch.n_tokens - 1] = true;
+                        batch.output[batch.n_tokens - 1] = true;
 
                         slot.n_decoded = 0;
                         slot.i_batch   = batch.n_tokens - 1;
@@ -2341,7 +2341,7 @@ struct server_context {
                 batch.pos      + i,
                 batch.n_seq_id + i,
                 batch.seq_id   + i,
-                batch.logits   + i,
+                batch.output   + i,
                 0, 0, 0, // unused
             };
 
diff --git a/examples/simple/simple.cpp b/examples/simple/simple.cpp
@@ -93,7 +93,7 @@ int main(int argc, char ** argv) {
     }
 
     // llama_decode will output logits only for the last token of the prompt
-    batch.logits[batch.n_tokens - 1] = true;
+    batch.output[batch.n_tokens - 1] = true;
 
     if (llama_decode(ctx, batch) != 0) {
         LOG_TEE("%s: llama_decode() failed\n", __func__);
diff --git a/include/llama.h b/include/llama.h
@@ -220,7 +220,7 @@ extern "C" {
     // - embd   : token embeddings (i.e. float vector of size n_embd) (used when token is NULL)
     // - pos    : the positions of the respective token in the sequence
     // - seq_id : the sequence to which the respective token belongs
-    // - logits : if zero, the logits (and/or the embeddings) for the respective token will not be output
+    // - output : if zero, the logits (and/or the embeddings) for the respective token will not be output
     //
     typedef struct llama_batch {
         int32_t n_tokens;
@@ -230,7 +230,7 @@ extern "C" {
         llama_pos    *  pos;
         int32_t      *  n_seq_id;
         llama_seq_id ** seq_id;
-        int8_t       *  logits; // TODO: rename this to "output"
+        int8_t       *  output; // Previously named 'logits', renamed to 'output' now.
 
         // NOTE: helpers for smooth API transition - can be deprecated in the future
         //       for future-proof code, use the above fields instead and ignore everything below
@@ -328,7 +328,7 @@ extern "C" {
         enum ggml_type type_v; // data type for V cache [EXPERIMENTAL]
 
         // Keep the booleans together to avoid misalignment during copy-by-value.
-        bool logits_all;  // the llama_decode() call computes all logits, not just the last one (DEPRECATED - set llama_batch.logits instead)
+        bool logits_all;  // the llama_decode() call computes all logits, not just the last one (DEPRECATED - set llama_batch.output instead)
         bool embeddings;  // if true, extract embeddings (together with logits)
         bool offload_kqv; // whether to offload the KQV ops (including the KV cache) to GPU
         bool flash_attn;  // whether to use flash attention [EXPERIMENTAL]
@@ -859,9 +859,9 @@ extern "C" {
     LLAMA_API void llama_synchronize(struct llama_context * ctx);
 
     // Token logits obtained from the last call to llama_decode()
-    // The logits for which llama_batch.logits[i] != 0 are stored contiguously
+    // The logits for which llama_batch.output[i] != 0 are stored contiguously
     // in the order they have appeared in the batch.
-    // Rows: number of tokens for which llama_batch.logits[i] != 0
+    // Rows: number of tokens for which llama_batch.output[i] != 0
     // Cols: n_vocab
     LLAMA_API float * llama_get_logits(struct llama_context * ctx);
 
@@ -873,7 +873,7 @@ extern "C" {
 
     // Get all output token embeddings.
     // when pooling_type == LLAMA_POOLING_TYPE_NONE or when using a generative model,
-    // the embeddings for which llama_batch.logits[i] != 0 are stored contiguously
+    // the embeddings for which llama_batch.output[i] != 0 are stored contiguously
     // in the order they have appeared in the batch.
     // shape: [n_outputs*n_embd]
     // Otherwise, returns NULL.
diff --git a/src/llama.cpp b/src/llama.cpp

Original file line number	Diff line number	Diff line change
`@@ -2666,7 +2666,7 @@ void llama_batch_add(`
`2666`	`2666`	`for (size_t i = 0; i < seq_ids.size(); ++i) {`
`2667`	`2667`	`batch.seq_id[batch.n_tokens][i] = seq_ids[i];`
`2668`	`2668`	`}`
`2669`		`- batch.logits [batch.n_tokens] = logits;`
	`2669`	`+ batch.output [batch.n_tokens] = logits;`
`2670`	`2670`
`2671`	`2671`	`batch.n_tokens++;`
`2672`	`2672`	`}`
Original file line number	Diff line number	Diff line change
`@@ -686,7 +686,7 @@ inline std::string LOG_BATCH_TOSTR_PRETTY(const C & ctx, const B & batch)`
`686`	`686`	`<< ":pos " << std::to_string(batch.pos[i])`
`687`	`687`	`<< ":n_seq_id " << std::to_string(batch.n_seq_id[i])`
`688`	`688`	`<< ":seq_id " << std::to_string(batch.seq_id[i][0])`
`689`		`- << ":logits " << std::to_string(batch.logits[i]);`
	`689`	`+ << ":logits " << std::to_string(batch.output[i]);`
`690`	`690`	`}`
`691`	`691`	`buf << " ]";`
`692`	`692`
Original file line number	Diff line number	Diff line change
`@@ -94,7 +94,7 @@ int main(int argc, char ** argv) {`
`94`	`94`	`batch.pos + i,`
`95`	`95`	`batch.n_seq_id + i,`
`96`	`96`	`batch.seq_id + i,`
`97`		`- batch.logits + i,`
	`97`	`+ batch.output + i,`
`98`	`98`	`0, 0, 0, // unused`
`99`	`99`	`};`
`100`	`100`
`@@ -149,7 +149,7 @@ int main(int argc, char ** argv) {`
`149`	`149`	`llama_batch_add(batch, 0, i, { j }, false);`
`150`	`150`	`}`
`151`	`151`	`}`
`152`		`- batch.logits[batch.n_tokens - 1] = true;`
	`152`	`+ batch.output[batch.n_tokens - 1] = true;`
`153`	`153`
`154`	`154`	`const auto t_pp_start = ggml_time_us();`
`155`	`155`
Original file line number	Diff line number	Diff line change
`@@ -86,11 +86,11 @@ for (i, token) in tokens.enumerated() {`
`86`	`86`	`if let seq_id = batch.seq_id[i] {`
`87`	`87`	`seq_id[0] = 0`
`88`	`88`	`}`
`89`		`- batch.logits[i] = 0`
	`89`	`+ batch.output[i] = 0`
`90`	`90`	`}`
`91`	`91`
`92`	`92`	`// llama_decode will output logits only for the last token of the prompt`
`93`		`-batch.logits[Int(batch.n_tokens) - 1] = 1`
	`93`	`+batch.output[Int(batch.n_tokens) - 1] = 1`
`94`	`94`
`95`	`95`	`if llama_decode(context, batch) != 0 {`
`96`	`96`	`print("llama_decode() failed")`
`@@ -178,7 +178,7 @@ while n_cur <= n_len {`
`178`	`178`	`if let seq_id = batch.seq_id[Int(batch.n_tokens)] {`
`179`	`179`	`seq_id[0] = Int32(i)`
`180`	`180`	`}`
`181`		`- batch.logits[Int(batch.n_tokens)] = 1`
	`181`	`+ batch.output[Int(batch.n_tokens)] = 1`
`182`	`182`
`183`	`183`	`i_batch[i] = batch.n_tokens`
`184`	`184`
Original file line number	Diff line number	Diff line change
`@@ -122,7 +122,7 @@ int main(int argc, char ** argv) {`
`122`	`122`	`}`
`123`	`123`
`124`	`124`	`// llama_decode will output logits only for the last token of the prompt`
`125`		`- batch.logits[batch.n_tokens - 1] = true;`
	`125`	`+ batch.output[batch.n_tokens - 1] = true;`
`126`	`126`
`127`	`127`	`if (llama_decode(ctx, batch) != 0) {`
`128`	`128`	`LOG_TEE("%s: llama_decode() failed\n", __func__);`
Original file line number	Diff line number	Diff line change
`@@ -52,7 +52,7 @@ static void batch_decode(llama_context * ctx, llama_batch & batch, float * outpu`
`52`	`52`	`}`
`53`	`53`
`54`	`54`	`for (int i = 0; i < batch.n_tokens; i++) {`
`55`		`- if (!batch.logits[i]) {`
	`55`	`+ if (!batch.output[i]) {`
`56`	`56`	`continue;`
`57`	`57`	`}`
`58`	`58`
Original file line number	Diff line number	Diff line change
`@@ -513,7 +513,7 @@ static bool compute_imatrix(llama_context * ctx, const gpt_params & params) {`
`513`	`513`	`tokens[batch_start] = llama_token_bos(llama_get_model(ctx));`
`514`	`514`	`}`
`515`	`515`
`516`		`- // TODO: use batch.logits to save computations instead of relying on logits_all == true`
	`516`	`+ // TODO: use batch.output to save computations instead of relying on logits_all == true`
`517`	`517`	`if (llama_decode(ctx, llama_batch_get_one(tokens.data() + batch_start, batch_size, j * n_batch, 0))) {`
`518`	`518`	`fprintf(stderr, "%s : failed to eval\n", __func__);`
`519`	`519`	`return false;`
Original file line number	Diff line number	Diff line change
`@@ -193,7 +193,7 @@ Java_android_llama_cpp_LLamaAndroid_bench_1model(`
`193`	`193`	`llama_batch_add(*batch, 0, i, { 0 }, false);`
`194`	`194`	`}`
`195`	`195`
`196`		`- batch->logits[batch->n_tokens - 1] = true;`
	`196`	`+ batch->output[batch->n_tokens - 1] = true;`
`197`	`197`	`llama_kv_cache_clear(context);`
`198`	`198`
`199`	`199`	`const auto t_pp_start = ggml_time_us();`
`@@ -306,7 +306,7 @@ Java_android_llama_cpp_LLamaAndroid_new_1batch(JNIEnv *, jobject, jint n_tokens,`
`306`	`306`	`for (int i = 0; i < n_tokens; ++i) {`
`307`	`307`	`batch->seq_id[i] = (llama_seq_id ) malloc(sizeof(llama_seq_id) n_seq_max);`
`308`	`308`	`}`
`309`		`- batch->logits = (int8_t ) malloc(sizeof(int8_t) n_tokens);`
	`309`	`+ batch->output = (int8_t ) malloc(sizeof(int8_t) n_tokens);`
`310`	`310`
`311`	`311`	`return reinterpret_cast<jlong>(batch);`
`312`	`312`	`}`
`@@ -363,7 +363,7 @@ Java_android_llama_cpp_LLamaAndroid_completion_1init(`
`363`	`363`	`}`
`364`	`364`
`365`	`365`	`// llama_decode will output logits only for the last token of the prompt`
`366`		`- batch->logits[batch->n_tokens - 1] = true;`
	`366`	`+ batch->output[batch->n_tokens - 1] = true;`
`367`	`367`
`368`	`368`	`if (llama_decode(context, *batch) != 0) {`
`369`	`369`	`LOGe("llama_decode() failed");`
Original file line number	Diff line number	Diff line change
`@@ -16,7 +16,7 @@ func llama_batch_add(_ batch: inout llama_batch, _ id: llama_token, _ pos: llama`
`16`	`16`	`for i in 0..<seq_ids.count {`
`17`	`17`	`batch.seq_id[Int(batch.n_tokens)]![Int(i)] = seq_ids[i]`
`18`	`18`	`}`
`19`		`- batch.logits [Int(batch.n_tokens)] = logits ? 1 : 0`
	`19`	`+ batch.output [Int(batch.n_tokens)] = logits ? 1 : 0`
`20`	`20`
`21`	`21`	`batch.n_tokens += 1`
`22`	`22`	`}`
`@@ -132,7 +132,7 @@ actor LlamaContext {`
`132`	`132`	`let i = Int(i1)`
`133`	`133`	`llama_batch_add(&batch, tokens_list[i], Int32(i), [0], false)`
`134`	`134`	`}`
`135`		`- batch.logits[Int(batch.n_tokens) - 1] = 1 // true`
	`135`	`+ batch.output[Int(batch.n_tokens) - 1] = 1 // true`
`136`	`136`
`137`	`137`	`if llama_decode(context, batch) != 0 {`
`138`	`138`	`print("llama_decode() failed")`
`@@ -214,7 +214,7 @@ actor LlamaContext {`
`214`	`214`	`for i in 0..<n_tokens {`
`215`	`215`	`llama_batch_add(&batch, 0, Int32(i), [0], false)`
`216`	`216`	`}`
`217`		`- batch.logits[Int(batch.n_tokens) - 1] = 1 // true`
	`217`	`+ batch.output[Int(batch.n_tokens) - 1] = 1 // true`
`218`	`218`
`219`	`219`	`llama_kv_cache_clear(context)`
`220`	`220`