From aa39c0cdcc8c446bdee90241ad7085f305f0c1fd Mon Sep 17 00:00:00 2001
From: potassiummmm <zhou.hansong@outlook.com>
Date: Wed, 18 Dec 2024 17:54:23 +0800
Subject: [PATCH 1/2] fix version requirement of transformers pypi package and
 model list for codegen

---
 3rdparty/llama.cpp | 2 +-
 setup_env.py       | 7 +++++--
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/3rdparty/llama.cpp b/3rdparty/llama.cpp
index 2ce8604..957b59d 160000
--- a/3rdparty/llama.cpp
+++ b/3rdparty/llama.cpp
@@ -1 +1 @@
-Subproject commit 2ce86040364799c44a48bb5a8407351812045dc6
+Subproject commit 957b59d2207370cd5061dd1bb12d079aa267fbab
diff --git a/setup_env.py b/setup_env.py
index 8440929..fa92b50 100644
--- a/setup_env.py
+++ b/setup_env.py
@@ -140,6 +140,9 @@ def setup_gguf():
 
 def gen_code():
     _, arch = system_info()
+    
+    llama3_f3_models = ["Llama3-8B-1.58-100B-tokens", "Falcon3-7B-1.58bit", "Falcon3-10B-1.58bit", "Falcon3-3B-1.58bit", "Falcon3-1B-1.58bit", "Falcon3-1B-Instruct-1.58bit", "Falcon3-3B-Instruct-1.58bit", "Falcon3-7B-Instruct-1.58bit", "Falcon3-10B-Instruct-1.58bit"]
+
     if arch == "arm64":
         if args.use_pretuned:
             pretuned_kernels = os.path.join("preset_kernels", get_model_name())
@@ -154,7 +157,7 @@ def gen_code():
                 shutil.copyfile(os.path.join(pretuned_kernels, "kernel_config_tl2.ini"), "include/kernel_config.ini")
         if get_model_name() == "bitnet_b1_58-large":
             run_command([sys.executable, "utils/codegen_tl1.py", "--model", "bitnet_b1_58-large", "--BM", "256,128,256", "--BK", "128,64,128", "--bm", "32,64,32"], log_step="codegen")
-        elif get_model_name() in ["Llama3-8B-1.58-100B-tokens", "Falcon3-7B-1.58bit", "Falcon3-10B-1.58bit", "Falcon3-3B-1.58bit", "Falcon3-1B-1.58bit"]:
+        elif get_model_name() in llama3_f3_models:
             run_command([sys.executable, "utils/codegen_tl1.py", "--model", "Llama3-8B-1.58-100B-tokens", "--BM", "256,128,256,128", "--BK", "128,64,128,64", "--bm", "32,64,32,64"], log_step="codegen")
         elif get_model_name() == "bitnet_b1_58-3B":
             run_command([sys.executable, "utils/codegen_tl1.py", "--model", "bitnet_b1_58-3B", "--BM", "160,320,320", "--BK", "64,128,64", "--bm", "32,64,32"], log_step="codegen")
@@ -170,7 +173,7 @@ def gen_code():
             shutil.copyfile(os.path.join(pretuned_kernels, "bitnet-lut-kernels-tl2.h"), "include/bitnet-lut-kernels.h")
         if get_model_name() == "bitnet_b1_58-large":
             run_command([sys.executable, "utils/codegen_tl2.py", "--model", "bitnet_b1_58-large", "--BM", "256,128,256", "--BK", "96,192,96", "--bm", "32,32,32"], log_step="codegen")
-        elif get_model_name() in ["Llama3-8B-1.58-100B-tokens", "Falcon3-7B-1.58bit", "Falcon3-10B-1.58bit", "Falcon3-3B-1.58bit", "Falcon3-1B-1.58bit"]:
+        elif get_model_name() in llama3_f3_models:
             run_command([sys.executable, "utils/codegen_tl2.py", "--model", "Llama3-8B-1.58-100B-tokens", "--BM", "256,128,256,128", "--BK", "96,96,96,96", "--bm", "32,32,32,32"], log_step="codegen")
         elif get_model_name() == "bitnet_b1_58-3B":
             run_command([sys.executable, "utils/codegen_tl2.py", "--model", "bitnet_b1_58-3B", "--BM", "160,320,320", "--BK", "96,96,96", "--bm", "32,32,32"], log_step="codegen")

From 0a446952e18bf6ac9d2de624af7a229dcab931c8 Mon Sep 17 00:00:00 2001
From: potassiummmm <zhou.hansong@outlook.com>
Date: Wed, 18 Dec 2024 21:20:26 +0800
Subject: [PATCH 2/2] fix readme issue and -cnv option issue

---
 README.md        | 5 +++--
 run_inference.py | 3 ++-
 setup_env.py     | 8 ++++----
 3 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/README.md b/README.md
index 4739214..8b6fce8 100644
--- a/README.md
+++ b/README.md
@@ -159,7 +159,7 @@ optional arguments:
 ### Basic usage
 ```bash
 # Run inference with the quantized model
-python run_inference.py -m models/Falcon3-7B-Instruct-1.58bit/ggml-model-i2_s.gguf -cnv "You are a helpful assistant"
+python run_inference.py -m models/Falcon3-7B-Instruct-1.58bit/ggml-model-i2_s.gguf -p "You are a helpful assistant" -cnv
 
 # Output:
 # Daniel went back to the the the garden. Mary travelled to the kitchen. Sandra journeyed to the kitchen. Sandra went to the hallway. John went to the bedroom. Mary went back to the garden. Where is Mary?
@@ -167,7 +167,7 @@ python run_inference.py -m models/Falcon3-7B-Instruct-1.58bit/ggml-model-i2_s.gg
 
 ```
 <pre>
-usage: run_inference.py [-h] [-m MODEL] [-n N_PREDICT] -p PROMPT [-t THREADS] [-c CTX_SIZE] [-temp TEMPERATURE]
+usage: run_inference.py [-h] [-m MODEL] [-n N_PREDICT] -p PROMPT [-t THREADS] [-c CTX_SIZE] [-temp TEMPERATURE] [-cnv]
 
 Run inference
 
@@ -185,6 +185,7 @@ optional arguments:
                         Size of the prompt context
   -temp TEMPERATURE, --temperature TEMPERATURE
                         Temperature, a hyperparameter that controls the randomness of the generated text
+  -cnv, --conversation  Whether to enable chat mode or not (for instruct models.)
 </pre>
 
 ### Benchmark
diff --git a/run_inference.py b/run_inference.py
index 75a7246..f3ab727 100644
--- a/run_inference.py
+++ b/run_inference.py
@@ -31,8 +31,9 @@ def run_inference():
         '-c', str(args.ctx_size),
         '--temp', str(args.temperature),
         "-b", "1",
-        "-cnv" if args.conversation else ""
     ]
+    if args.conversation:
+        command.append("-cnv")
     run_command(command)
 
 def signal_handler(sig, frame):
diff --git a/setup_env.py b/setup_env.py
index fa92b50..4b38013 100644
--- a/setup_env.py
+++ b/setup_env.py
@@ -27,19 +27,19 @@ SUPPORTED_HF_MODELS = {
         "model_name": "Falcon3-7B-1.58bit",
     },
     "tiiuae/Falcon3-10B-Instruct-1.58bit": {
-        "model_name": "Falcon3-10B-1.58bit",
+        "model_name": "Falcon3-10B-Instruct-1.58bit",
     },
     "tiiuae/Falcon3-10B-1.58bit": {
         "model_name": "Falcon3-10B-1.58bit",
     },
     "tiiuae/Falcon3-3B-Instruct-1.58bit": {
-        "model_name": "Falcon3-3B-1.58bit",
+        "model_name": "Falcon3-3B-Instruct-1.58bit",
     },
     "tiiuae/Falcon3-3B-1.58bit": {
         "model_name": "Falcon3-3B-1.58bit",
     },
     "tiiuae/Falcon3-1B-Instruct-1.58bit": {
-        "model_name": "Falcon3-1B-1.58bit",
+        "model_name": "Falcon3-1B-Instruct-1.58bit",
     },
 }
 
@@ -141,7 +141,7 @@ def setup_gguf():
 def gen_code():
     _, arch = system_info()
     
-    llama3_f3_models = ["Llama3-8B-1.58-100B-tokens", "Falcon3-7B-1.58bit", "Falcon3-10B-1.58bit", "Falcon3-3B-1.58bit", "Falcon3-1B-1.58bit", "Falcon3-1B-Instruct-1.58bit", "Falcon3-3B-Instruct-1.58bit", "Falcon3-7B-Instruct-1.58bit", "Falcon3-10B-Instruct-1.58bit"]
+    llama3_f3_models = set([model['model_name'] for model in SUPPORTED_HF_MODELS.values() if model['model_name'].startswith("Falcon3") or model['model_name'].startswith("Llama")])
 
     if arch == "arm64":
         if args.use_pretuned: