From 2da929d38d279efee37ebd8bed3782e1031f854f Mon Sep 17 00:00:00 2001 From: Dmitry Ng <19asdek91@gmail.com> Date: Tue, 4 Aug 2026 00:42:25 +0300 Subject: [PATCH] feat(providers): add xAI provider support and configuration details - Updated the README to include xAI as a supported provider, along with pre-configured provider files for testing. - Added a new configuration file for xAI with detailed model settings and pricing. - Created a testing report for xAI, summarizing the performance and success rates of various agents. - Enhanced the VSCode launch configuration to include xAI provider options for testing. --- .vscode/launch.json | 2 + README.md | 12 +- examples/configs/xai.provider.yml | 143 ++++++++ examples/tests/xai-report.md | 556 ++++++++++++++++++++++++++++++ 4 files changed, 709 insertions(+), 4 deletions(-) create mode 100644 examples/configs/xai.provider.yml create mode 100644 examples/tests/xai-report.md diff --git a/.vscode/launch.json b/.vscode/launch.json index 10253c9d..dfa9ebed 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -82,6 +82,7 @@ // "-config", "${workspaceFolder}/examples/configs/openrouter.provider.yml", // "-config", "${workspaceFolder}/examples/configs/deepinfra.provider.yml", // "-config", "${workspaceFolder}/examples/configs/novita.provider.yml", + // "-config", "${workspaceFolder}/examples/configs/xai.provider.yml", // "-report", "${workspaceFolder}/examples/tests/openai-report.md", // "-report", "${workspaceFolder}/examples/tests/anthropic-report.md", // "-report", "${workspaceFolder}/examples/tests/gemini-report.md", @@ -106,6 +107,7 @@ // "-report", "${workspaceFolder}/examples/tests/openrouter-report.md", // "-report", "${workspaceFolder}/examples/tests/deepinfra-report.md", // "-report", "${workspaceFolder}/examples/tests/novita-report.md", + // "-report", "${workspaceFolder}/examples/tests/xai-report.md", "-agents", "all", "-groups", "all", "-workers", "8", diff --git a/README.md b/README.md index 32bbf3b8..83775430 100644 --- a/README.md +++ b/README.md @@ -3236,7 +3236,7 @@ docker run --rm \ #### Using Pre-configured Providers -The Docker image comes with built-in support for major providers (OpenAI, Anthropic, Gemini, Ollama) and pre-configured provider files for additional services (OpenRouter, OrcaRouter, DeepInfra, DeepSeek, Moonshot, Novita): +The Docker image comes with built-in support for major providers (OpenAI, Anthropic, Gemini, Ollama) and pre-configured provider files for additional services (OpenRouter, OrcaRouter, DeepInfra, DeepSeek, Moonshot, Novita, xAI): ```bash # Test with OpenRouter configuration @@ -3269,6 +3269,9 @@ docker exec -it pentagi /opt/pentagi/bin/ctester -config /opt/pentagi/conf/moons # Test with Novita configuration docker exec -it pentagi /opt/pentagi/bin/ctester -config /opt/pentagi/conf/novita.provider.yml +# Test with xAI configuration +docker exec -it pentagi /opt/pentagi/bin/ctester -config /opt/pentagi/conf/xai.provider.yml + # Test with OpenAI configuration docker exec -it pentagi /opt/pentagi/bin/ctester -type openai @@ -3297,11 +3300,11 @@ docker exec -it pentagi /opt/pentagi/bin/ctester -config /opt/pentagi/conf/ollam To use these configurations, your `.env` file only needs to contain: ``` -LLM_SERVER_URL=https://openrouter.ai/api/v1 # or https://api.orcarouter.ai/v1 or https://api.deepinfra.com/v1/openai or https://api.openai.com/v1 or https://api.novita.ai/openai or https://api.atlascloud.ai/v1 +LLM_SERVER_URL=https://openrouter.ai/api/v1 # or https://api.deepinfra.com/v1/openai or https://api.openai.com/v1 or https://api.novita.ai/openai or https://api.atlascloud.ai/v1 or https://api.orcarouter.ai/v1 or https://api.x.ai/v1 LLM_SERVER_KEY=your_api_key LLM_SERVER_MODEL= # Leave empty, as models are specified in the config -LLM_SERVER_CONFIG_PATH=/opt/pentagi/conf/openrouter.provider.yml # or orcarouter.provider.yml or deepinfra.provider.ymll or custom-openai.provider.yml or novita.provider.yml or atlas.provider.yml -LLM_SERVER_PROVIDER= # Provider name for LiteLLM proxy (e.g., openrouter, orcarouter, deepseek, moonshot, novita) +LLM_SERVER_CONFIG_PATH=/opt/pentagi/conf/openrouter.provider.yml # or deepinfra.provider.ymll or custom-openai.provider.yml or novita.provider.yml or atlas.provider.yml or orcarouter.provider.yml or xai.provider.yml +LLM_SERVER_PROVIDER= # Provider name for LiteLLM proxy (e.g., openrouter, deepseek, moonshot, novita, orcarouter, xai) LLM_SERVER_LEGACY_REASONING=false # Controls reasoning format, for OpenAI must be true (default: false) LLM_SERVER_PRESERVE_REASONING=false # Preserve reasoning content in multi-turn conversations (required by Moonshot, default: false) @@ -3424,6 +3427,7 @@ When using LiteLLM proxy, set the corresponding `*_PROVIDER` variable to enable - `orcarouter` - for OrcaRouter aggregator - `deepinfra` - for DeepInfra hosting - `novita` - for Novita AI +- `xai` - for xAI (`grok-*` models) - Any other provider name configured in your LiteLLM instance **Example with LiteLLM:** diff --git a/examples/configs/xai.provider.yml b/examples/configs/xai.provider.yml new file mode 100644 index 00000000..06e0ec33 --- /dev/null +++ b/examples/configs/xai.provider.yml @@ -0,0 +1,143 @@ +simple: + model: grok-4.20-0309-non-reasoning + temperature: 0.3 + top_p: 0.9 + n: 1 + max_tokens: 8192 + price: + input: 1.25 + output: 2.50 + cache_read: 0.20 + +simple_json: + model: grok-4.20-0309-non-reasoning + temperature: 0.3 + top_p: 0.9 + n: 1 + max_tokens: 4096 + json: true + price: + input: 1.25 + output: 2.50 + cache_read: 0.20 + +primary_agent: + model: grok-4.5 + n: 1 + max_tokens: 16384 + reasoning: + effort: high + price: + input: 2.00 + output: 6.00 + cache_read: 0.30 + +assistant: + model: grok-4.5 + n: 1 + max_tokens: 16384 + reasoning: + effort: high + price: + input: 2.00 + output: 6.00 + cache_read: 0.30 + +generator: + model: grok-4.5 + n: 1 + max_tokens: 32768 + reasoning: + effort: high + price: + input: 2.00 + output: 6.00 + cache_read: 0.30 + +refiner: + model: grok-4.5 + n: 1 + max_tokens: 32768 + reasoning: + effort: high + price: + input: 2.00 + output: 6.00 + cache_read: 0.30 + +adviser: + model: grok-4.5 + n: 1 + max_tokens: 16384 + reasoning: + effort: high + price: + input: 2.00 + output: 6.00 + cache_read: 0.30 + +reflector: + model: grok-4.20-0309-non-reasoning + temperature: 0.5 + top_p: 0.9 + n: 1 + max_tokens: 8192 + price: + input: 1.25 + output: 2.50 + cache_read: 0.20 + +searcher: + model: grok-4.20-0309-non-reasoning + temperature: 0.5 + top_p: 0.9 + n: 1 + max_tokens: 4096 + price: + input: 1.25 + output: 2.50 + cache_read: 0.20 + +enricher: + model: grok-4.20-0309-non-reasoning + temperature: 0.5 + top_p: 0.9 + n: 1 + max_tokens: 4096 + price: + input: 1.25 + output: 2.50 + cache_read: 0.20 + +coder: + model: grok-build-0.1 + n: 1 + max_tokens: 20480 + reasoning: + effort: high + price: + input: 1.00 + output: 2.00 + cache_read: 0.20 + +installer: + model: grok-4.20-0309-reasoning + n: 1 + max_tokens: 12288 + reasoning: + effort: high + price: + input: 1.25 + output: 2.50 + cache_read: 0.20 + +pentester: + model: grok-4.5 + n: 1 + max_tokens: 16384 + reasoning: + effort: high + price: + input: 2.00 + output: 6.00 + cache_read: 0.30 diff --git a/examples/tests/xai-report.md b/examples/tests/xai-report.md new file mode 100644 index 00000000..b6a122bb --- /dev/null +++ b/examples/tests/xai-report.md @@ -0,0 +1,556 @@ +# LLM Agent Testing Report + +Generated: Mon, 03 Aug 2026 21:25:49 UTC + +## Overall Results + +| Agent | Model | Reasoning | Success Rate | Average Latency | +|-------|-------|-----------|--------------|-----------------| +| simple | grok-4.20-0309-non-reasoning | false | 24/24 (100.00%) | 0.807s | +| simple_json | grok-4.20-0309-non-reasoning | false | 7/7 (100.00%) | 0.637s | +| primary_agent | grok-4.5 | true | 24/24 (100.00%) | 1.876s | +| assistant | grok-4.5 | true | 24/24 (100.00%) | 1.934s | +| generator | grok-4.5 | true | 24/24 (100.00%) | 2.239s | +| refiner | grok-4.5 | true | 24/24 (100.00%) | 2.133s | +| adviser | grok-4.5 | true | 24/24 (100.00%) | 1.788s | +| reflector | grok-4.20-0309-non-reasoning | true | 24/24 (100.00%) | 0.726s | +| searcher | grok-4.20-0309-non-reasoning | true | 24/24 (100.00%) | 0.771s | +| enricher | grok-4.20-0309-non-reasoning | true | 24/24 (100.00%) | 0.721s | +| coder | grok-build-0.1 | true | 24/24 (100.00%) | 2.738s | +| installer | grok-4.20-0309-reasoning | true | 23/24 (95.83%) | 2.326s | +| pentester | grok-4.5 | true | 24/24 (100.00%) | 1.867s | + +**Total**: 294/295 (99.66%) successful tests +**Overall average latency**: 1.636s + +## Detailed Results + +### simple (grok-4.20-0309-non-reasoning) + +#### Basic Tests + +| Test | Result | Latency | Error | +|------|--------|---------|-------| +| Simple Math | ✅ Pass | 1.221s | | +| Text Transform Uppercase | ✅ Pass | 0.484s | | +| Count from 1 to 5 | ✅ Pass | 0.525s | | +| Math Calculation | ✅ Pass | 0.514s | | +| Basic Echo Function | ✅ Pass | 0.723s | | +| Streaming Simple Math Streaming | ✅ Pass | 0.506s | | +| Streaming Count from 1 to 3 Streaming | ✅ Pass | 0.607s | | +| Streaming Basic Echo Function Streaming | ✅ Pass | 0.767s | | + +#### Advanced Tests + +| Test | Result | Latency | Error | +|------|--------|---------|-------| +| JSON Response Function | ✅ Pass | 1.004s | | +| Search Query Function | ✅ Pass | 0.670s | | +| Ask Advice Function | ✅ Pass | 0.688s | | +| Streaming Search Query Function Streaming | ✅ Pass | 0.679s | | +| Basic Context Memory Test | ✅ Pass | 0.639s | | +| Function Argument Memory Test | ✅ Pass | 0.521s | | +| Function Response Memory Test | ✅ Pass | 0.490s | | +| Penetration Testing Memory with Tool Call | ✅ Pass | 0.802s | | +| Cybersecurity Workflow Memory Test | ✅ Pass | 0.497s | | +| Read a file, then edit it via unified diff | ✅ Pass | 1.495s | | +| Penetration Testing Methodology | ✅ Pass | 1.504s | | +| Vulnerability Assessment Tools | ✅ Pass | 1.706s | | +| SQL Injection Attack Type | ✅ Pass | 0.646s | | +| Penetration Testing Framework | ✅ Pass | 1.137s | | +| Web Application Security Scanner | ✅ Pass | 0.809s | | +| Penetration Testing Tool Selection | ✅ Pass | 0.732s | | + +**Summary**: 24/24 (100.00%) successful tests + +**Average latency**: 0.807s + +--- + +### simple_json (grok-4.20-0309-non-reasoning) + +#### Advanced Tests + +| Test | Result | Latency | Error | +|------|--------|---------|-------| +| Vulnerability Report Memory Test | ✅ Pass | 0.633s | | +| Person Information JSON | ✅ Pass | 0.624s | | +| Project Information JSON | ✅ Pass | 0.538s | | +| User Profile JSON | ✅ Pass | 0.609s | | +| JSON Array Response Without Schema | ✅ Pass | 0.622s | | +| Streaming Person Information JSON Streaming | ✅ Pass | 0.700s | | + +#### Capability Tests + +| Test | Capability | Result | Latency | Note | +|------|------------|--------|---------|------| +| Structured Output With JSON Schema | structured_output | ✅ Pass | 0.732s | | + +**Summary**: 7/7 (100.00%) successful tests + +**Average latency**: 0.637s + +--- + +### primary_agent (grok-4.5) + +#### Basic Tests + +| Test | Result | Latency | Error | +|------|--------|---------|-------| +| Simple Math | ✅ Pass | 1.845s | | +| Text Transform Uppercase | ✅ Pass | 1.289s | | +| Count from 1 to 5 | ✅ Pass | 1.296s | | +| Math Calculation | ✅ Pass | 1.127s | | +| Basic Echo Function | ✅ Pass | 0.930s | | +| Streaming Simple Math Streaming | ✅ Pass | 1.185s | | +| Streaming Count from 1 to 3 Streaming | ✅ Pass | 1.273s | | +| Streaming Basic Echo Function Streaming | ✅ Pass | 1.144s | | + +#### Advanced Tests + +| Test | Result | Latency | Error | +|------|--------|---------|-------| +| JSON Response Function | ✅ Pass | 1.484s | | +| Search Query Function | ✅ Pass | 1.134s | | +| Ask Advice Function | ✅ Pass | 1.496s | | +| Streaming Search Query Function Streaming | ✅ Pass | 1.137s | | +| Basic Context Memory Test | ✅ Pass | 2.021s | | +| Function Argument Memory Test | ✅ Pass | 1.550s | | +| Function Response Memory Test | ✅ Pass | 1.563s | | +| Penetration Testing Memory with Tool Call | ✅ Pass | 2.265s | | +| Cybersecurity Workflow Memory Test | ✅ Pass | 1.709s | | +| Read a file, then edit it via unified diff | ✅ Pass | 3.104s | | +| Penetration Testing Methodology | ✅ Pass | 3.804s | | +| Vulnerability Assessment Tools | ✅ Pass | 4.579s | | +| SQL Injection Attack Type | ✅ Pass | 1.892s | | +| Penetration Testing Framework | ✅ Pass | 3.664s | | +| Web Application Security Scanner | ✅ Pass | 2.335s | | +| Penetration Testing Tool Selection | ✅ Pass | 1.181s | | + +**Summary**: 24/24 (100.00%) successful tests + +**Average latency**: 1.876s + +--- + +### assistant (grok-4.5) + +#### Basic Tests + +| Test | Result | Latency | Error | +|------|--------|---------|-------| +| Simple Math | ✅ Pass | 1.168s | | +| Text Transform Uppercase | ✅ Pass | 1.116s | | +| Count from 1 to 5 | ✅ Pass | 1.350s | | +| Math Calculation | ✅ Pass | 1.217s | | +| Basic Echo Function | ✅ Pass | 1.782s | | +| Streaming Simple Math Streaming | ✅ Pass | 1.366s | | +| Streaming Count from 1 to 3 Streaming | ✅ Pass | 1.477s | | +| Streaming Basic Echo Function Streaming | ✅ Pass | 1.087s | | + +#### Advanced Tests + +| Test | Result | Latency | Error | +|------|--------|---------|-------| +| JSON Response Function | ✅ Pass | 1.519s | | +| Search Query Function | ✅ Pass | 0.971s | | +| Ask Advice Function | ✅ Pass | 1.364s | | +| Streaming Search Query Function Streaming | ✅ Pass | 1.039s | | +| Basic Context Memory Test | ✅ Pass | 1.753s | | +| Function Argument Memory Test | ✅ Pass | 1.434s | | +| Function Response Memory Test | ✅ Pass | 0.848s | | +| Penetration Testing Memory with Tool Call | ✅ Pass | 2.493s | | +| Cybersecurity Workflow Memory Test | ✅ Pass | 1.561s | | +| Read a file, then edit it via unified diff | ✅ Pass | 2.970s | | +| Penetration Testing Methodology | ✅ Pass | 3.787s | | +| Vulnerability Assessment Tools | ✅ Pass | 6.609s | | +| SQL Injection Attack Type | ✅ Pass | 1.340s | | +| Penetration Testing Framework | ✅ Pass | 3.854s | | +| Web Application Security Scanner | ✅ Pass | 3.004s | | +| Penetration Testing Tool Selection | ✅ Pass | 1.286s | | + +**Summary**: 24/24 (100.00%) successful tests + +**Average latency**: 1.934s + +--- + +### generator (grok-4.5) + +#### Basic Tests + +| Test | Result | Latency | Error | +|------|--------|---------|-------| +| Simple Math | ✅ Pass | 1.356s | | +| Text Transform Uppercase | ✅ Pass | 1.209s | | +| Count from 1 to 5 | ✅ Pass | 1.193s | | +| Math Calculation | ✅ Pass | 1.054s | | +| Basic Echo Function | ✅ Pass | 1.548s | | +| Streaming Simple Math Streaming | ✅ Pass | 1.583s | | +| Streaming Count from 1 to 3 Streaming | ✅ Pass | 1.422s | | +| Streaming Basic Echo Function Streaming | ✅ Pass | 1.043s | | + +#### Advanced Tests + +| Test | Result | Latency | Error | +|------|--------|---------|-------| +| JSON Response Function | ✅ Pass | 2.088s | | +| Search Query Function | ✅ Pass | 0.986s | | +| Ask Advice Function | ✅ Pass | 1.701s | | +| Streaming Search Query Function Streaming | ✅ Pass | 1.023s | | +| Basic Context Memory Test | ✅ Pass | 5.516s | | +| Function Argument Memory Test | ✅ Pass | 1.519s | | +| Function Response Memory Test | ✅ Pass | 1.451s | | +| Penetration Testing Memory with Tool Call | ✅ Pass | 2.068s | | +| Cybersecurity Workflow Memory Test | ✅ Pass | 1.650s | | +| Read a file, then edit it via unified diff | ✅ Pass | 3.279s | | +| Penetration Testing Methodology | ✅ Pass | 7.178s | | +| Vulnerability Assessment Tools | ✅ Pass | 5.469s | | +| SQL Injection Attack Type | ✅ Pass | 1.346s | | +| Penetration Testing Framework | ✅ Pass | 3.533s | | +| Web Application Security Scanner | ✅ Pass | 3.128s | | +| Penetration Testing Tool Selection | ✅ Pass | 1.376s | | + +**Summary**: 24/24 (100.00%) successful tests + +**Average latency**: 2.239s + +--- + +### refiner (grok-4.5) + +#### Basic Tests + +| Test | Result | Latency | Error | +|------|--------|---------|-------| +| Simple Math | ✅ Pass | 1.262s | | +| Text Transform Uppercase | ✅ Pass | 1.209s | | +| Count from 1 to 5 | ✅ Pass | 1.154s | | +| Math Calculation | ✅ Pass | 1.090s | | +| Basic Echo Function | ✅ Pass | 1.655s | | +| Streaming Simple Math Streaming | ✅ Pass | 1.109s | | +| Streaming Count from 1 to 3 Streaming | ✅ Pass | 1.352s | | +| Streaming Basic Echo Function Streaming | ✅ Pass | 1.094s | | + +#### Advanced Tests + +| Test | Result | Latency | Error | +|------|--------|---------|-------| +| JSON Response Function | ✅ Pass | 1.427s | | +| Search Query Function | ✅ Pass | 1.081s | | +| Ask Advice Function | ✅ Pass | 1.280s | | +| Streaming Search Query Function Streaming | ✅ Pass | 1.093s | | +| Basic Context Memory Test | ✅ Pass | 7.274s | | +| Function Argument Memory Test | ✅ Pass | 1.311s | | +| Function Response Memory Test | ✅ Pass | 0.883s | | +| Penetration Testing Memory with Tool Call | ✅ Pass | 2.285s | | +| Cybersecurity Workflow Memory Test | ✅ Pass | 1.773s | | +| Read a file, then edit it via unified diff | ✅ Pass | 3.301s | | +| Penetration Testing Methodology | ✅ Pass | 4.107s | | +| Vulnerability Assessment Tools | ✅ Pass | 4.685s | | +| SQL Injection Attack Type | ✅ Pass | 1.599s | | +| Penetration Testing Framework | ✅ Pass | 5.428s | | +| Web Application Security Scanner | ✅ Pass | 2.458s | | +| Penetration Testing Tool Selection | ✅ Pass | 1.257s | | + +**Summary**: 24/24 (100.00%) successful tests + +**Average latency**: 2.133s + +--- + +### adviser (grok-4.5) + +#### Basic Tests + +| Test | Result | Latency | Error | +|------|--------|---------|-------| +| Simple Math | ✅ Pass | 1.145s | | +| Text Transform Uppercase | ✅ Pass | 1.152s | | +| Count from 1 to 5 | ✅ Pass | 1.330s | | +| Math Calculation | ✅ Pass | 1.096s | | +| Basic Echo Function | ✅ Pass | 1.467s | | +| Streaming Simple Math Streaming | ✅ Pass | 1.185s | | +| Streaming Count from 1 to 3 Streaming | ✅ Pass | 1.319s | | +| Streaming Basic Echo Function Streaming | ✅ Pass | 1.115s | | + +#### Advanced Tests + +| Test | Result | Latency | Error | +|------|--------|---------|-------| +| JSON Response Function | ✅ Pass | 1.273s | | +| Search Query Function | ✅ Pass | 0.986s | | +| Ask Advice Function | ✅ Pass | 1.185s | | +| Streaming Search Query Function Streaming | ✅ Pass | 1.192s | | +| Basic Context Memory Test | ✅ Pass | 1.718s | | +| Function Argument Memory Test | ✅ Pass | 1.361s | | +| Function Response Memory Test | ✅ Pass | 0.872s | | +| Penetration Testing Memory with Tool Call | ✅ Pass | 2.029s | | +| Cybersecurity Workflow Memory Test | ✅ Pass | 1.603s | | +| Read a file, then edit it via unified diff | ✅ Pass | 3.113s | | +| Penetration Testing Methodology | ✅ Pass | 4.391s | | +| Vulnerability Assessment Tools | ✅ Pass | 3.740s | | +| SQL Injection Attack Type | ✅ Pass | 1.516s | | +| Penetration Testing Framework | ✅ Pass | 4.216s | | +| Web Application Security Scanner | ✅ Pass | 2.738s | | +| Penetration Testing Tool Selection | ✅ Pass | 1.166s | | + +**Summary**: 24/24 (100.00%) successful tests + +**Average latency**: 1.788s + +--- + +### reflector (grok-4.20-0309-non-reasoning) + +#### Basic Tests + +| Test | Result | Latency | Error | +|------|--------|---------|-------| +| Simple Math | ✅ Pass | 0.496s | | +| Text Transform Uppercase | ✅ Pass | 0.556s | | +| Count from 1 to 5 | ✅ Pass | 0.711s | | +| Math Calculation | ✅ Pass | 0.540s | | +| Basic Echo Function | ✅ Pass | 0.501s | | +| Streaming Simple Math Streaming | ✅ Pass | 0.578s | | +| Streaming Count from 1 to 3 Streaming | ✅ Pass | 0.602s | | +| Streaming Basic Echo Function Streaming | ✅ Pass | 0.580s | | + +#### Advanced Tests + +| Test | Result | Latency | Error | +|------|--------|---------|-------| +| JSON Response Function | ✅ Pass | 0.859s | | +| Search Query Function | ✅ Pass | 0.447s | | +| Ask Advice Function | ✅ Pass | 0.544s | | +| Streaming Search Query Function Streaming | ✅ Pass | 0.693s | | +| Basic Context Memory Test | ✅ Pass | 0.527s | | +| Function Argument Memory Test | ✅ Pass | 0.497s | | +| Function Response Memory Test | ✅ Pass | 0.586s | | +| Penetration Testing Memory with Tool Call | ✅ Pass | 0.628s | | +| Cybersecurity Workflow Memory Test | ✅ Pass | 0.470s | | +| Read a file, then edit it via unified diff | ✅ Pass | 1.326s | | +| Penetration Testing Methodology | ✅ Pass | 1.583s | | +| Vulnerability Assessment Tools | ✅ Pass | 1.584s | | +| SQL Injection Attack Type | ✅ Pass | 0.635s | | +| Penetration Testing Framework | ✅ Pass | 1.050s | | +| Web Application Security Scanner | ✅ Pass | 0.822s | | +| Penetration Testing Tool Selection | ✅ Pass | 0.600s | | + +**Summary**: 24/24 (100.00%) successful tests + +**Average latency**: 0.726s + +--- + +### searcher (grok-4.20-0309-non-reasoning) + +#### Basic Tests + +| Test | Result | Latency | Error | +|------|--------|---------|-------| +| Simple Math | ✅ Pass | 0.494s | | +| Text Transform Uppercase | ✅ Pass | 0.539s | | +| Count from 1 to 5 | ✅ Pass | 0.475s | | +| Math Calculation | ✅ Pass | 0.482s | | +| Basic Echo Function | ✅ Pass | 0.788s | | +| Streaming Simple Math Streaming | ✅ Pass | 0.539s | | +| Streaming Count from 1 to 3 Streaming | ✅ Pass | 0.595s | | +| Streaming Basic Echo Function Streaming | ✅ Pass | 0.619s | | + +#### Advanced Tests + +| Test | Result | Latency | Error | +|------|--------|---------|-------| +| JSON Response Function | ✅ Pass | 1.240s | | +| Search Query Function | ✅ Pass | 0.518s | | +| Ask Advice Function | ✅ Pass | 0.631s | | +| Streaming Search Query Function Streaming | ✅ Pass | 0.648s | | +| Basic Context Memory Test | ✅ Pass | 0.542s | | +| Function Argument Memory Test | ✅ Pass | 0.501s | | +| Function Response Memory Test | ✅ Pass | 0.574s | | +| Penetration Testing Memory with Tool Call | ✅ Pass | 0.638s | | +| Cybersecurity Workflow Memory Test | ✅ Pass | 0.532s | | +| Read a file, then edit it via unified diff | ✅ Pass | 1.399s | | +| Penetration Testing Methodology | ✅ Pass | 1.575s | | +| Vulnerability Assessment Tools | ✅ Pass | 1.853s | | +| SQL Injection Attack Type | ✅ Pass | 0.649s | | +| Penetration Testing Framework | ✅ Pass | 1.129s | | +| Web Application Security Scanner | ✅ Pass | 0.908s | | +| Penetration Testing Tool Selection | ✅ Pass | 0.624s | | + +**Summary**: 24/24 (100.00%) successful tests + +**Average latency**: 0.771s + +--- + +### enricher (grok-4.20-0309-non-reasoning) + +#### Basic Tests + +| Test | Result | Latency | Error | +|------|--------|---------|-------| +| Simple Math | ✅ Pass | 0.483s | | +| Text Transform Uppercase | ✅ Pass | 0.452s | | +| Count from 1 to 5 | ✅ Pass | 0.519s | | +| Math Calculation | ✅ Pass | 0.496s | | +| Basic Echo Function | ✅ Pass | 0.795s | | +| Streaming Simple Math Streaming | ✅ Pass | 0.617s | | +| Streaming Count from 1 to 3 Streaming | ✅ Pass | 0.580s | | +| Streaming Basic Echo Function Streaming | ✅ Pass | 0.689s | | + +#### Advanced Tests + +| Test | Result | Latency | Error | +|------|--------|---------|-------| +| JSON Response Function | ✅ Pass | 0.549s | | +| Search Query Function | ✅ Pass | 0.527s | | +| Ask Advice Function | ✅ Pass | 0.573s | | +| Streaming Search Query Function Streaming | ✅ Pass | 0.610s | | +| Basic Context Memory Test | ✅ Pass | 0.566s | | +| Function Argument Memory Test | ✅ Pass | 0.478s | | +| Function Response Memory Test | ✅ Pass | 0.515s | | +| Penetration Testing Memory with Tool Call | ✅ Pass | 0.593s | | +| Cybersecurity Workflow Memory Test | ✅ Pass | 0.458s | | +| Read a file, then edit it via unified diff | ✅ Pass | 1.230s | | +| Penetration Testing Methodology | ✅ Pass | 1.456s | | +| Vulnerability Assessment Tools | ✅ Pass | 1.870s | | +| SQL Injection Attack Type | ✅ Pass | 0.638s | | +| Penetration Testing Framework | ✅ Pass | 1.157s | | +| Web Application Security Scanner | ✅ Pass | 0.872s | | +| Penetration Testing Tool Selection | ✅ Pass | 0.573s | | + +**Summary**: 24/24 (100.00%) successful tests + +**Average latency**: 0.721s + +--- + +### coder (grok-build-0.1) + +#### Basic Tests + +| Test | Result | Latency | Error | +|------|--------|---------|-------| +| Simple Math | ✅ Pass | 1.649s | | +| Text Transform Uppercase | ✅ Pass | 1.563s | | +| Count from 1 to 5 | ✅ Pass | 2.030s | | +| Math Calculation | ✅ Pass | 1.293s | | +| Basic Echo Function | ✅ Pass | 2.304s | | +| Streaming Simple Math Streaming | ✅ Pass | 1.025s | | +| Streaming Count from 1 to 3 Streaming | ✅ Pass | 1.856s | | +| Streaming Basic Echo Function Streaming | ✅ Pass | 1.875s | | + +#### Advanced Tests + +| Test | Result | Latency | Error | +|------|--------|---------|-------| +| JSON Response Function | ✅ Pass | 1.841s | | +| Search Query Function | ✅ Pass | 1.673s | | +| Ask Advice Function | ✅ Pass | 1.683s | | +| Streaming Search Query Function Streaming | ✅ Pass | 2.231s | | +| Basic Context Memory Test | ✅ Pass | 1.904s | | +| Function Argument Memory Test | ✅ Pass | 2.705s | | +| Function Response Memory Test | ✅ Pass | 2.171s | | +| Penetration Testing Memory with Tool Call | ✅ Pass | 3.996s | | +| Cybersecurity Workflow Memory Test | ✅ Pass | 6.073s | | +| Read a file, then edit it via unified diff | ✅ Pass | 3.945s | | +| Penetration Testing Methodology | ✅ Pass | 3.424s | | +| Vulnerability Assessment Tools | ✅ Pass | 7.914s | | +| SQL Injection Attack Type | ✅ Pass | 2.726s | | +| Penetration Testing Framework | ✅ Pass | 3.132s | | +| Web Application Security Scanner | ✅ Pass | 4.601s | | +| Penetration Testing Tool Selection | ✅ Pass | 2.079s | | + +**Summary**: 24/24 (100.00%) successful tests + +**Average latency**: 2.738s + +--- + +### installer (grok-4.20-0309-reasoning) + +#### Basic Tests + +| Test | Result | Latency | Error | +|------|--------|---------|-------| +| Simple Math | ✅ Pass | 1.064s | | +| Text Transform Uppercase | ✅ Pass | 1.107s | | +| Count from 1 to 5 | ✅ Pass | 1.737s | | +| Math Calculation | ✅ Pass | 1.186s | | +| Basic Echo Function | ✅ Pass | 2.403s | | +| Streaming Simple Math Streaming | ✅ Pass | 3.255s | | +| Streaming Count from 1 to 3 Streaming | ✅ Pass | 1.509s | | +| Streaming Basic Echo Function Streaming | ✅ Pass | 1.742s | | + +#### Advanced Tests + +| Test | Result | Latency | Error | +|------|--------|---------|-------| +| JSON Response Function | ✅ Pass | 1.758s | | +| Search Query Function | ✅ Pass | 1.338s | | +| Ask Advice Function | ✅ Pass | 1.663s | | +| Streaming Search Query Function Streaming | ✅ Pass | 1.077s | | +| Basic Context Memory Test | ✅ Pass | 1.257s | | +| Function Argument Memory Test | ✅ Pass | 1.405s | | +| Function Response Memory Test | ✅ Pass | 1.645s | | +| Penetration Testing Memory with Tool Call | ✅ Pass | 1.746s | | +| Cybersecurity Workflow Memory Test | ✅ Pass | 2.975s | | +| Read a file, then edit it via unified diff | ❌ Fail | 3.645s | edit\_file's diff did not apply: invalid diff line \(must start with ' ', '\-', or '+'\): "Status: draft" | +| Penetration Testing Methodology | ✅ Pass | 2.628s | | +| Vulnerability Assessment Tools | ✅ Pass | 9.348s | | +| SQL Injection Attack Type | ✅ Pass | 2.879s | | +| Penetration Testing Framework | ✅ Pass | 3.056s | | +| Web Application Security Scanner | ✅ Pass | 3.682s | | +| Penetration Testing Tool Selection | ✅ Pass | 1.696s | | + +**Summary**: 23/24 (95.83%) successful tests + +**Average latency**: 2.326s + +--- + +### pentester (grok-4.5) + +#### Basic Tests + +| Test | Result | Latency | Error | +|------|--------|---------|-------| +| Simple Math | ✅ Pass | 1.260s | | +| Text Transform Uppercase | ✅ Pass | 1.169s | | +| Count from 1 to 5 | ✅ Pass | 1.384s | | +| Math Calculation | ✅ Pass | 0.933s | | +| Basic Echo Function | ✅ Pass | 1.732s | | +| Streaming Simple Math Streaming | ✅ Pass | 1.352s | | +| Streaming Count from 1 to 3 Streaming | ✅ Pass | 1.393s | | +| Streaming Basic Echo Function Streaming | ✅ Pass | 1.114s | | + +#### Advanced Tests + +| Test | Result | Latency | Error | +|------|--------|---------|-------| +| JSON Response Function | ✅ Pass | 1.519s | | +| Search Query Function | ✅ Pass | 1.005s | | +| Ask Advice Function | ✅ Pass | 1.883s | | +| Streaming Search Query Function Streaming | ✅ Pass | 1.194s | | +| Basic Context Memory Test | ✅ Pass | 1.896s | | +| Function Argument Memory Test | ✅ Pass | 1.463s | | +| Function Response Memory Test | ✅ Pass | 0.841s | | +| Penetration Testing Memory with Tool Call | ✅ Pass | 2.286s | | +| Cybersecurity Workflow Memory Test | ✅ Pass | 1.429s | | +| Read a file, then edit it via unified diff | ✅ Pass | 3.019s | | +| Penetration Testing Methodology | ✅ Pass | 3.691s | | +| Vulnerability Assessment Tools | ✅ Pass | 4.333s | | +| SQL Injection Attack Type | ✅ Pass | 1.306s | | +| Penetration Testing Framework | ✅ Pass | 5.069s | | +| Web Application Security Scanner | ✅ Pass | 2.253s | | +| Penetration Testing Tool Selection | ✅ Pass | 1.271s | | + +**Summary**: 24/24 (100.00%) successful tests + +**Average latency**: 1.867s + +--- +