lee/BitNet
1
0
Fork 0
mirror of https://github.com/microsoft/BitNet.git synced 2026-05-03 19:30:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

[fix] make demo_benchmark.sh more fast

Browse Source
This commit is contained in:
deva100
2025-12-23 07:23:14 +00:00
parent 41cc304868
commit 43da5e5f76
1 changed files with 43 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
demo_benchmark.sh
+43 -34
View File
@@ -15,7 +15,7 @@ STATS_DIR="stats/demo_$(date +%Y%m%d_%H%M%S)"
mkdir -p "${STATS_DIR}"
echo -e "${BLUE}========================================${NC}"
echo -e "${BLUE}Quick Benchmark Demo${NC}"
echo -e "${BLUE}Quick Benchmark Demo (< 2 mins)${NC}"
echo -e "${BLUE}========================================${NC}"
echo ""
echo "Output directory: ${STATS_DIR}"
@@ -34,11 +34,11 @@ echo -e "${GREEN}[1/3] Collecting machine info...${NC}"
echo ""
# Test 2: Quick benchmark test
echo -e "${GREEN}[2/3] Running quick benchmark (2 threads only)...${NC}"
echo -e "${GREEN}[2/2] Running quick benchmark (single thread, minimal tokens)...${NC}"
if [[ -f "build/bin/llama-bench" ]] && [[ -f "models/BitNet-b1.58-2B-4T/ggml-model-i2_s_embed_q6_k.gguf" ]]; then
./build/bin/llama-bench \
-m models/BitNet-b1.58-2B-4T/ggml-model-i2_s_embed_q6_k.gguf \
-p 128 -n 128 -t 1,2,4 -ngl 0 \
-p 32 -n 32 -t 1 -ngl 0 \
2>&1 | tee "${STATS_DIR}/bench_quick.txt"
# Parse results
@@ -67,51 +67,60 @@ else
fi
echo ""
# Test 3: Quick PPL test (one dataset only)
echo -e "${GREEN}[3/3] Running quick PPL test (wikitext-2 only, 2 embed types)...${NC}"
if [[ -f "build/bin/llama-perplexity" ]] && [[ -f "data/wikitext-2-raw/wiki.test.raw" ]]; then
# Test 3: Quick PPL test (using simplified dataset)
echo -e "${GREEN}[3/3] Running quick PPL test (wiki.simple, 1 embed type)...${NC}"
# Create simplified dataset if needed (first 100 lines for quick demo)
if [[ -f "data/wikitext-2-raw/wiki.test.raw" ]]; then
echo "Creating simplified dataset (100 lines)..."
head -100 data/wikitext-2-raw/wiki.test.raw > data/wikitext-2-raw/wiki.simple.raw
fi
if [[ -f "build/bin/llama-perplexity" ]] && [[ -f "data/wikitext-2-raw/wiki.simple.raw" ]]; then
{
echo "# Quick PPL Test"
echo "# Quick PPL Test (Simplified Dataset)"
echo ""
echo "| Embed Type | PPL |"
echo "|------------|-----|"
echo "| Embed Type | Dataset | PPL |"
echo "|------------|---------|-----|"
for embed in i2_s q6_k; do
model="models/BitNet-b1.58-2B-4T/ggml-model-i2_s_embed_${embed}.gguf"
if [[ -f "$model" ]]; then
echo "Testing: $embed..."
output=$(./build/bin/llama-perplexity \
-m "$model" \
-f data/wikitext-2-raw/wiki.test.raw \
-t 4 -ngl 0 2>&1 || true)
ppl=$(echo "$output" | awk '
/Final estimate/ && /PPL/ {
if (match($0, /PPL[[:space:]]*=[[:space:]]*([0-9]+(\.[0-9]+)?)/, m)) {
print m[1];
exit;
}
# Test only one embed type with simplified dataset for speed
embed="q6_k"
model="models/BitNet-b1.58-2B-4T/ggml-model-i2_s_embed_${embed}.gguf"
if [[ -f "$model" ]]; then
echo "Testing: $embed on wiki.simple..."
output=$(./build/bin/llama-perplexity \
-m "$model" \
-f data/wikitext-2-raw/wiki.simple.raw \
-t 4 -ngl 0 2>&1 || true)
ppl=$(echo "$output" | awk '
/Final estimate/ && /PPL/ {
if (match($0, /PPL[[:space:]]*=[[:space:]]*([0-9]+(\.[0-9]+)?)/, m)) {
print m[1];
exit;
}
')
if [[ -n "$ppl" ]]; then
echo "| $embed | $ppl |"
else
echo "| $embed | N/A |"
fi
}
')
if [[ -n "$ppl" ]]; then
echo "| $embed | wiki.simple | $ppl |"
else
echo "| $embed | wiki.simple | N/A |"
fi
done
fi
} | tee "${STATS_DIR}/ppl_quick.md"
echo ""
echo -e "${GREEN}Results saved to: ${STATS_DIR}/ppl_quick.md${NC}"
cat "${STATS_DIR}/ppl_quick.md"
else
echo "Skipping PPL test (binary or dataset not found)"
echo "Skipping PPL test (binary or simplified dataset not found)"
echo "Note: Full PPL test available in: ./run_paper_benchmarks.sh"
fi
echo ""
echo -e "${BLUE}========================================${NC}"
echo -e "${GREEN}Demo completed!${NC}"
echo -e "${GREEN}Demo completed! (Fast mode - PPL skipped)${NC}"
echo -e "${BLUE}========================================${NC}"
echo ""
echo "All results in: ${STATS_DIR}/"