From 43da5e5f760887d5b061c95605cd89a7e63db76b Mon Sep 17 00:00:00 2001
From: deva100
 <azureuser@a100dev2.vmlraidoxzcexnpbwxrwjxjste.px.internal.cloudapp.net>
Date: Tue, 23 Dec 2025 07:23:14 +0000
Subject: [PATCH] [fix] make demo_benchmark.sh more fast

---
 demo_benchmark.sh | 77 ++++++++++++++++++++++++++---------------------
 1 file changed, 43 insertions(+), 34 deletions(-)

diff --git a/demo_benchmark.sh b/demo_benchmark.sh
index 8845a3f..dad999b 100755
--- a/demo_benchmark.sh
+++ b/demo_benchmark.sh
@@ -15,7 +15,7 @@ STATS_DIR="stats/demo_$(date +%Y%m%d_%H%M%S)"
 mkdir -p "${STATS_DIR}"
 
 echo -e "${BLUE}========================================${NC}"
-echo -e "${BLUE}Quick Benchmark Demo${NC}"
+echo -e "${BLUE}Quick Benchmark Demo (< 2 mins)${NC}"
 echo -e "${BLUE}========================================${NC}"
 echo ""
 echo "Output directory: ${STATS_DIR}"
@@ -34,11 +34,11 @@ echo -e "${GREEN}[1/3] Collecting machine info...${NC}"
 echo ""
 
 # Test 2: Quick benchmark test
-echo -e "${GREEN}[2/3] Running quick benchmark (2 threads only)...${NC}"
+echo -e "${GREEN}[2/2] Running quick benchmark (single thread, minimal tokens)...${NC}"
 if [[ -f "build/bin/llama-bench" ]] && [[ -f "models/BitNet-b1.58-2B-4T/ggml-model-i2_s_embed_q6_k.gguf" ]]; then
     ./build/bin/llama-bench \
         -m models/BitNet-b1.58-2B-4T/ggml-model-i2_s_embed_q6_k.gguf \
-        -p 128 -n 128 -t 1,2,4 -ngl 0 \
+        -p 32 -n 32 -t 1 -ngl 0 \
         2>&1 | tee "${STATS_DIR}/bench_quick.txt"
     
     # Parse results
@@ -67,51 +67,60 @@ else
 fi
 echo ""
 
-# Test 3: Quick PPL test (one dataset only)
-echo -e "${GREEN}[3/3] Running quick PPL test (wikitext-2 only, 2 embed types)...${NC}"
-if [[ -f "build/bin/llama-perplexity" ]] && [[ -f "data/wikitext-2-raw/wiki.test.raw" ]]; then
+# Test 3: Quick PPL test (using simplified dataset)
+echo -e "${GREEN}[3/3] Running quick PPL test (wiki.simple, 1 embed type)...${NC}"
+
+# Create simplified dataset if needed (first 100 lines for quick demo)
+if [[ -f "data/wikitext-2-raw/wiki.test.raw" ]]; then
+    echo "Creating simplified dataset (100 lines)..."
+    head -100 data/wikitext-2-raw/wiki.test.raw > data/wikitext-2-raw/wiki.simple.raw
+fi
+
+if [[ -f "build/bin/llama-perplexity" ]] && [[ -f "data/wikitext-2-raw/wiki.simple.raw" ]]; then
     {
-        echo "# Quick PPL Test"
+        echo "# Quick PPL Test (Simplified Dataset)"
         echo ""
-        echo "| Embed Type | PPL |"
-        echo "|------------|-----|"
+        echo "| Embed Type | Dataset | PPL |"
+        echo "|------------|---------|-----|"
         
-        for embed in i2_s q6_k; do
-            model="models/BitNet-b1.58-2B-4T/ggml-model-i2_s_embed_${embed}.gguf"
-            if [[ -f "$model" ]]; then
-                echo "Testing: $embed..."
-                output=$(./build/bin/llama-perplexity \
-                    -m "$model" \
-                    -f data/wikitext-2-raw/wiki.test.raw \
-                    -t 4 -ngl 0 2>&1 || true)
-                
-                ppl=$(echo "$output" | awk '
-                    /Final estimate/ && /PPL/ {
-                        if (match($0, /PPL[[:space:]]*=[[:space:]]*([0-9]+(\.[0-9]+)?)/, m)) {
-                            print m[1];
-                            exit;
-                        }
+        # Test only one embed type with simplified dataset for speed
+        embed="q6_k"
+        model="models/BitNet-b1.58-2B-4T/ggml-model-i2_s_embed_${embed}.gguf"
+        if [[ -f "$model" ]]; then
+            echo "Testing: $embed on wiki.simple..."
+            output=$(./build/bin/llama-perplexity \
+                -m "$model" \
+                -f data/wikitext-2-raw/wiki.simple.raw \
+                -t 4 -ngl 0 2>&1 || true)
+            
+            ppl=$(echo "$output" | awk '
+                /Final estimate/ && /PPL/ {
+                    if (match($0, /PPL[[:space:]]*=[[:space:]]*([0-9]+(\.[0-9]+)?)/, m)) {
+                        print m[1];
+                        exit;
                     }
-                ')
-                
-                if [[ -n "$ppl" ]]; then
-                    echo "| $embed | $ppl |"
-                else
-                    echo "| $embed | N/A |"
-                fi
+                }
+            ')
+            
+            if [[ -n "$ppl" ]]; then
+                echo "| $embed | wiki.simple | $ppl |"
+            else
+                echo "| $embed | wiki.simple | N/A |"
             fi
-        done
+        fi
     } | tee "${STATS_DIR}/ppl_quick.md"
     
     echo ""
     echo -e "${GREEN}Results saved to: ${STATS_DIR}/ppl_quick.md${NC}"
+    cat "${STATS_DIR}/ppl_quick.md"
 else
-    echo "Skipping PPL test (binary or dataset not found)"
+    echo "Skipping PPL test (binary or simplified dataset not found)"
+    echo "Note: Full PPL test available in: ./run_paper_benchmarks.sh"
 fi
 echo ""
 
 echo -e "${BLUE}========================================${NC}"
-echo -e "${GREEN}Demo completed!${NC}"
+echo -e "${GREEN}Demo completed! (Fast mode - PPL skipped)${NC}"
 echo -e "${BLUE}========================================${NC}"
 echo ""
 echo "All results in: ${STATS_DIR}/"