Merge pull request #3 from saddam213/feature/gguf

Fix Tokenize of new line, Remove space inserts

Author: martindevans

LLama/LLamaEmbedder.cs

@@ -65,10 +65,6 @@ public float[] GetEmbeddings(string text)
         /// <exception cref="RuntimeError"></exception>
         public float[] GetEmbeddings(string text, bool addBos)
         {
-            if (addBos)
-            {
-                text = text.Insert(0, " ");
-            }
 
             var embed_inp_array = _ctx.Tokenize(text, addBos);
 

LLama/LLamaInstructExecutor.cs

@@ -114,7 +114,6 @@ protected override void PreprocessInputs(string text, InferStateArgs args)
             if (_is_prompt_run)
             {
                 // When running the first input (prompt) in inteactive mode, we should specially process it.
-                text = " " + text;
                 _embed_inps = Context.Tokenize(text, true).ToList();
             }
             else

LLama/LLamaInteractExecutor.cs

@@ -26,7 +26,7 @@ public class InteractiveExecutor : StatefulExecutorBase
         /// <param name="context"></param>
         public InteractiveExecutor(LLamaContext context) : base(context)
         {
-            _llama_token_newline = Context.NativeHandle.Tokenize("\n", false, Context.Encoding);
+            _llama_token_newline = new [] { NativeApi.llama_token_nl(Context.NativeHandle) };
         }
 
         /// <inheritdoc />
@@ -104,7 +104,6 @@ protected override void PreprocessInputs(string text, InferStateArgs args)
             if (_is_prompt_run)
             {
                 // When running the first input (prompt) in inteactive mode, we should specially process it.
-                text = " " + text;
                 _embed_inps = Context.Tokenize(text, true).ToList();
             }
             else