diff --git a/.env.example b/.env.example index 5e5e1137..e30f7e34 100644 --- a/.env.example +++ b/.env.example @@ -36,7 +36,7 @@ GRAPHITI_IMAGE=vxcontrol/graphiti:latest LANGFUSE_WORKER_IMAGE=langfuse/langfuse-worker:3 LANGFUSE_WEB_IMAGE=langfuse/langfuse:3 CLICKHOUSE_IMAGE=clickhouse/clickhouse-server:24 -MINIO_IMAGE=minio/minio:RELEASE.2025-07-23T15-54-02Z +MINIO_IMAGE=cgr.dev/chainguard/minio:latest REDIS_IMAGE=redis:7 POSTGRES_IMAGE=postgres:16 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cda7d063..9f565497 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -152,7 +152,8 @@ jobs: TAG_COMMIT=$(git rev-list -n 1 "$LATEST_TAG" 2>/dev/null || echo "") if [ "$CURRENT_COMMIT" != "$TAG_COMMIT" ]; then - PACKAGE_REV=$(git rev-parse --short HEAD) + # Fixed width, as in scripts/version.sh: --short widens with the size of the clone. + PACKAGE_REV=$(printf '%s' "$CURRENT_COMMIT" | cut -c1-7) else PACKAGE_REV="" fi @@ -204,7 +205,8 @@ jobs: # Set revision only if current commit differs from tag commit if [ "$CURRENT_COMMIT" != "$TAG_COMMIT" ]; then - PACKAGE_REV=$(git rev-parse --short HEAD) + # Fixed width, as in scripts/version.sh: --short widens with the size of the clone. + PACKAGE_REV=$(printf '%s' "$CURRENT_COMMIT" | cut -c1-7) echo "revision=${PACKAGE_REV}" >> $GITHUB_OUTPUT echo "is_release=false" >> $GITHUB_OUTPUT echo "Building development version: ${VERSION}-${PACKAGE_REV}" diff --git a/README.md b/README.md index d1494bcd..cc33ab6e 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,7 @@ - [How to Use PentAGI After Login](#how-to-use-pentagi-after-login) - [API Access](#api-access) - [LLM Provider Configuration](#custom-llm-provider-configuration) + - [Azure OpenAI](#using-azure-openai) - [Ollama](#ollama-provider-configuration) - [OpenAI](#openai-provider-configuration) - [Anthropic](#anthropic-provider-configuration) @@ -77,7 +78,7 @@ You can watch the video **PentAGI overview**: - Persistent Storage. All commands and outputs are stored in PostgreSQL with [pgvector](https://hub.docker.com/r/vxcontrol/pgvector) extension. - Scalable Architecture. Microservices-based design supporting horizontal scaling. - Self-Hosted Solution. Complete control over your deployment and data. -- Flexible Authentication. Support for 10+ LLM providers ([OpenAI](https://platform.openai.com/), [Anthropic](https://www.anthropic.com/), [Google AI/Gemini](https://ai.google.dev/), [AWS Bedrock](https://aws.amazon.com/bedrock/), [Ollama](https://ollama.com/), [DeepSeek](https://www.deepseek.com/en/), [GLM](https://z.ai/), [Kimi](https://platform.moonshot.ai/), [Qwen](https://www.alibabacloud.com/en/), [MiniMax](https://www.minimax.io/), [Mistral](https://mistral.ai/), [xAI](https://x.ai/), Custom) plus aggregators ([OpenRouter](https://openrouter.ai/), [DeepInfra](https://deepinfra.com/), [Atlas Cloud](https://www.atlascloud.ai/), [OpenCode Go plan](https://opencode.ai/en/go)). For production local deployments, see our [vLLM + Qwen3.5-27B-FP8 guide](examples/guides/vllm-qwen35-27b-fp8.md). +- Flexible Authentication. Support for 10+ LLM providers ([OpenAI](https://platform.openai.com/), [Anthropic](https://www.anthropic.com/), [Google AI/Gemini](https://ai.google.dev/), [AWS Bedrock](https://aws.amazon.com/bedrock/), [Ollama](https://ollama.com/), [DeepSeek](https://www.deepseek.com/en/), [GLM](https://z.ai/), [Kimi](https://platform.moonshot.ai/), [Qwen](https://www.alibabacloud.com/en/), [MiniMax](https://www.minimax.io/), [Mistral](https://mistral.ai/), [xAI](https://x.ai/), Custom for any OpenAI-compatible endpoint including [Azure OpenAI](#using-azure-openai)) plus aggregators ([OpenRouter](https://openrouter.ai/), [DeepInfra](https://deepinfra.com/), [Atlas Cloud](https://www.atlascloud.ai/), [OpenCode Go plan](https://opencode.ai/en/go)). For production local deployments, see our [vLLM + Qwen3.5-27B-FP8 guide](examples/guides/vllm-qwen35-27b-fp8.md). - API Token Authentication. Secure Bearer token system for programmatic access to REST and GraphQL APIs. - Quick Deployment. Easy setup through [Docker Compose](https://docs.docker.com/compose/) with comprehensive environment configuration. @@ -1438,14 +1439,16 @@ When using custom LLM providers with the `LLM_SERVER_*` variables, you can fine- > [!TIP] > For production-grade local deployments, consider using **vLLM** with **Qwen3.5-27B-FP8** for optimal performance. See our [comprehensive deployment guide](examples/guides/vllm-qwen35-27b-fp8.md) which includes hardware requirements, configuration templates ([thinking mode](examples/configs/vllm-qwen3.5-27b-fp8.provider.yml) and [non-thinking mode](examples/configs/vllm-qwen3.5-27b-fp8-no-think.provider.yml)), and performance benchmarks showing 13K TPS prompt processing on 4× RTX 5090 GPUs. -| Variable | Default | Description | -| ------------------------------- | ------- | --------------------------------------------------------------------------------------- | -| `LLM_SERVER_URL` | | Base URL for the custom LLM API endpoint | -| `LLM_SERVER_KEY` | | API key for the custom LLM provider | -| `LLM_SERVER_MODEL` | | Default model to use (can be overridden in provider config) | -| `LLM_SERVER_CONFIG_PATH` | | Path to the YAML configuration file for agent-specific models | -| `LLM_SERVER_PROVIDER` | | Provider name prefix for model names (e.g., `openrouter`, `deepseek` for LiteLLM proxy) | -| `LLM_SERVER_PRESERVE_REASONING` | `false` | Preserve reasoning content in multi-turn conversations (required by some providers) | +| Variable | Default | Description | +| ------------------------------- | ------------ | ------------------------------------------------------------------------------------------------------------------ | +| `LLM_SERVER_URL` | | Base URL for the custom LLM API endpoint | +| `LLM_SERVER_KEY` | | API key for the custom LLM provider | +| `LLM_SERVER_MODEL` | | Default model to use (can be overridden in provider config) | +| `LLM_SERVER_CONFIG_PATH` | | Path to the YAML configuration file for agent-specific models | +| `LLM_SERVER_PROVIDER` | | Provider name prefix for model names (e.g., `openrouter`, `deepseek` for LiteLLM proxy) | +| `LLM_SERVER_PRESERVE_REASONING` | `false` | Preserve reasoning content in multi-turn conversations (required by some providers) | +| `LLM_SERVER_API_TYPE` | | `azure` or `azure_ad` for an Azure OpenAI deployment, empty for a plain endpoint; see [Using Azure OpenAI](#using-azure-openai) | +| `LLM_SERVER_API_VERSION` | `2024-10-21` | The `api-version` Azure requires; ignored by a plain endpoint | The `LLM_SERVER_PROVIDER` setting is particularly useful when using **LiteLLM proxy**, which adds a provider prefix to model names. For example, when connecting to Moonshot API through LiteLLM, models like `kimi-2.5` become `moonshot/kimi-2.5`. By setting `LLM_SERVER_PROVIDER=moonshot`, you can use the same provider configuration file for both direct API access and LiteLLM proxy access without modifications. @@ -1455,6 +1458,38 @@ The `LLM_SERVER_PRESERVE_REASONING` setting controls whether reasoning content i This setting is required by some LLM providers (e.g., Moonshot) that return errors like "thinking is enabled but reasoning_content is missing in assistant tool call message" when reasoning content is not included in multi-turn conversations. Enable this setting if your provider requires reasoning content to be preserved. +#### Using Azure OpenAI + +The custom provider calls Azure OpenAI deployments directly: + +```bash +LLM_SERVER_URL=https://.openai.azure.com # the Endpoint from the resource's Keys and Endpoint page, without an /openai path +LLM_SERVER_KEY=your_azure_openai_api_key # KEY 1 or KEY 2 from the same page +LLM_SERVER_API_TYPE=azure # the api-key header and deployment URLs +LLM_SERVER_API_VERSION=2024-10-21 # the default; a newer version works as well +LLM_SERVER_MODEL= # Leave empty, models are specified in the config +LLM_SERVER_CONFIG_PATH=/opt/pentagi/conf/azure-openai.provider.yml +``` + +Azure addresses a **deployment**, not a model: PentAGI calls `/openai/deployments//chat/completions?api-version=`, so every `model:` in the provider config must be the name of a deployment in your resource, and `LLM_SERVER_PROVIDER` must stay empty, because its prefix would become part of the deployment name. The bundled [`azure-openai.provider.yml`](examples/configs/azure-openai.provider.yml) expects deployments named after their models: `gpt-4.1`, `gpt-4.1-mini` and `o4-mini`. If yours are named differently, copy the file, change the `model:` values, and mount your copy: + +```bash +PENTAGI_LLM_SERVER_CONFIG_PATH=/path/on/host/my-azure.provider.yml # mounted at /opt/pentagi/conf/custom.provider.yml +LLM_SERVER_CONFIG_PATH=/opt/pentagi/conf/custom.provider.yml +``` + +After `docker compose up -d`, check the deployments before running a flow. Without `-config`, `ctester` tests the file `LLM_SERVER_CONFIG_PATH` names, the bundled one or your copy; the bundled configuration's own run is in [`examples/tests/azure-openai-report.md`](examples/tests/azure-openai-report.md): + +```bash +docker exec -it pentagi /opt/pentagi/bin/ctester +``` + +- PentAGI does not read a model catalogue from Azure, so it knows no context window for a deployment and does not compact agent chains to fit one. +- `LLM_SERVER_API_TYPE=azure_ad` sends `LLM_SERVER_KEY` as a Microsoft Entra ID bearer token instead of an API key. PentAGI does not refresh it, and Entra tokens expire after about an hour, so use an API key for anything longer. +- Azure's OpenAI-compatible [v1 API](https://learn.microsoft.com/en-us/azure/foundry/openai/api-version-lifecycle) works as a plain endpoint too: set `LLM_SERVER_URL=https://.openai.azure.com/openai/v1` and leave `LLM_SERVER_API_TYPE` empty; `model:` is still the deployment name. +- The installer's custom provider form has **API Type** and **API Version** fields for the same two settings. +- Embeddings are configured separately through `EMBEDDING_*` and have no Azure mode; `LLM_SERVER_API_TYPE` does not apply to them. + #### Troubleshooting: tool-call (function-call) parser errors PentAGI drives its agents with tool calls (also called function calls), so any custom OpenAI-compatible backend configured through `LLM_SERVER_*` must return valid tool-call JSON in the format the OpenAI Chat Completions API defines. When the backend emits malformed, truncated, or non-conforming tool-call arguments, the agent chain cannot continue. @@ -2788,10 +2823,14 @@ NEO4J_PASSWORD=replace_with_a_strong_password NEO4J_URI=bolt://neo4j:7687 ``` -Download the optional compose file when installing manually, then start both stacks: +Download the optional compose file and the Neo4j settings with the APOC plugin Graphiti needs when installing manually, then start both stacks: ```bash curl -O https://raw.githubusercontent.com/vxcontrol/pentagi/master/docker-compose-graphiti.yml +mkdir -p neo4j/conf neo4j/plugins +for f in conf/neo4j.conf conf/apoc.conf plugins/apoc-5.26.19-core.jar; do + curl -fsSL -o "neo4j/$f" "https://raw.githubusercontent.com/vxcontrol/pentagi/master/examples/neo4j/$f" +done docker compose -f docker-compose.yml -f docker-compose-graphiti.yml up -d ``` @@ -2914,7 +2953,7 @@ The installer copies [`examples/neo4j`](examples/neo4j) beside the installation NEO4J_DIR=./neo4j ``` -`NEO4J_DIR` may point directly to `./examples/neo4j` for development. Both `conf/` and `plugins/` are mounted read-only; the stack still starts on Neo4j's built-in defaults (without APOC) if the directory is absent, since Docker creates an empty one automatically. Do not duplicate any `NEO4J_*` variable from the table above inside `conf/neo4j.conf` — the Neo4j Docker entrypoint always strips a matching line from the mounted file and re-appends the environment variable's value, so a duplicated setting in the file would be silently ignored. +`NEO4J_DIR` may point directly to `./examples/neo4j` for development. Both `conf/` and `plugins/` are mounted read-only; the stack still starts on Neo4j's built-in defaults (without APOC) if the directory is absent, since Docker creates an empty one automatically — but Graphiti writes relationships through APOC, so its graph then stays empty while PentAGI logs no error. Do not duplicate any `NEO4J_*` variable from the table above inside `conf/neo4j.conf` — the Neo4j Docker entrypoint always strips a matching line from the mounted file and re-appends the environment variable's value, so a duplicated setting in the file would be silently ignored. The bundled stack currently wires Neo4j only. The Graphiti image contains FalkorDB support, but using it requires a separately configured deployment because the stock compose file does not expose `GRAPHITI_GRAPH_BACKEND` or `FALKORDB_*`. @@ -3374,6 +3413,9 @@ docker exec -it pentagi /opt/pentagi/bin/ctester -type bedrock # Test with Custom OpenAI configuration docker exec -it pentagi /opt/pentagi/bin/ctester -config /opt/pentagi/conf/custom-openai.provider.yml +# Test with Azure OpenAI configuration (needs LLM_SERVER_API_TYPE=azure, see Using Azure OpenAI) +docker exec -it pentagi /opt/pentagi/bin/ctester -config /opt/pentagi/conf/azure-openai.provider.yml + # Test with Ollama configuration (local inference) docker exec -it pentagi /opt/pentagi/bin/ctester -config /opt/pentagi/conf/ollama-llama318b.provider.yml diff --git a/backend/cmd/installer/wizard/models/main_menu.go b/backend/cmd/installer/wizard/models/main_menu.go index 2df63c6d..3d256753 100644 --- a/backend/cmd/installer/wizard/models/main_menu.go +++ b/backend/cmd/installer/wizard/models/main_menu.go @@ -30,6 +30,10 @@ func NewMainMenuHandler(c controller.Controller, s styles.Styles, w window.Windo // ListScreenHandler interface implementation func (h *MainMenuHandler) LoadItems() []ListItem { + checker := h.controller.GetChecker() + // Updates are offered only inside Maintenance, so its entry carries their highlight. + hasUpdates := checker.CanUpdateWorker() || checker.CanUpdateAll() || checker.CanUpdateInstaller() + items := []ListItem{ {ID: LLMProvidersScreen}, {ID: EmbedderFormScreen}, @@ -39,7 +43,7 @@ func (h *MainMenuHandler) LoadItems() []ListItem { {ID: ServerSettingsScreen}, {ID: ApplyChangesScreen, Highlighted: true}, {ID: InstallPentagiScreen, Highlighted: true}, - {ID: MaintenanceScreen}, + {ID: MaintenanceScreen, Highlighted: hasUpdates}, } // filter out disabled items diff --git a/backend/cmd/installer/wizard/models/main_menu_test.go b/backend/cmd/installer/wizard/models/main_menu_test.go new file mode 100644 index 00000000..3aee464d --- /dev/null +++ b/backend/cmd/installer/wizard/models/main_menu_test.go @@ -0,0 +1,58 @@ +package models + +import ( + "os" + "path/filepath" + "slices" + "testing" + + "pentagi/cmd/installer/checker" + "pentagi/cmd/installer/files" + "pentagi/cmd/installer/state" + "pentagi/cmd/installer/wizard/controller" + "pentagi/cmd/installer/wizard/styles" + "pentagi/cmd/installer/wizard/window" +) + +func TestMainMenu_LoadItems_HighlightsMaintenanceWhileAnUpdateWaits(t *testing.T) { + upToDate := checker.CheckResult{ + WorkerImageExists: true, WorkerIsUpToDate: true, + PentagiInstalled: true, PentagiIsUpToDate: true, + InstallerIsUpToDate: true, UpdateServerAccessible: true, + } + + for _, tc := range []struct { + name string + update func(*checker.CheckResult) + want bool + }{ + {"everything is up to date", func(*checker.CheckResult) {}, false}, + {"a newer worker image", func(c *checker.CheckResult) { c.WorkerIsUpToDate = false }, true}, + {"a newer stack", func(c *checker.CheckResult) { c.PentagiIsUpToDate = false }, true}, + {"a newer installer", func(c *checker.CheckResult) { c.InstallerIsUpToDate = false }, true}, + } { + t.Run(tc.name, func(t *testing.T) { + envPath := filepath.Join(t.TempDir(), ".env") + if err := os.WriteFile(envPath, nil, 0o600); err != nil { + t.Fatal(err) + } + st, err := state.NewState(envPath) + if err != nil { + t.Fatal(err) + } + result := upToDate + tc.update(&result) + c := controller.NewController(st, files.NewFiles(), result) + + items := NewMainMenuHandler(c, styles.New(), window.New()).LoadItems() + + i := slices.IndexFunc(items, func(item ListItem) bool { return item.ID == MaintenanceScreen }) + if i < 0 { + t.Fatal("maintenance is not offered") + } + if items[i].Highlighted != tc.want { + t.Errorf("maintenance highlighted = %v, want %v", items[i].Highlighted, tc.want) + } + }) + } +} diff --git a/backend/pkg/controller/prompter.go b/backend/pkg/controller/prompter.go index 34700105..d442b306 100644 --- a/backend/pkg/controller/prompter.go +++ b/backend/pkg/controller/prompter.go @@ -6,6 +6,9 @@ import ( "pentagi/pkg/database" "pentagi/pkg/templates" + "pentagi/pkg/templates/validator" + + "github.com/sirupsen/logrus" ) // newUserPrompter loads the user's custom prompts from the database and @@ -30,9 +33,9 @@ func newUserPrompter(ctx context.Context, db database.Querier, userID int64) (te // buildUserPrompter is the pure merge step extracted from newUserPrompter so // it can be unit-tested without a database fake or filesystem access. It // mutates the supplied defaults map by overlaying each non-empty user -// override on top, then returns a Prompter backed by that map. Callers must -// pass a fresh map (e.g., from templates.LoadDefaultPromptsMap) so the -// embedded defaults are not modified. +// override that still validates, then returns a Prompter backed by that map. +// Callers must pass a fresh map (e.g., from templates.LoadDefaultPromptsMap) +// so the embedded defaults are not modified. func buildUserPrompter(defaults templates.PromptsMap, userPrompts []database.Prompt) templates.Prompter { for _, p := range userPrompts { if p.Prompt == "" { @@ -43,7 +46,18 @@ func buildUserPrompter(defaults templates.PromptsMap, userPrompts []database.Pro // ErrTemplateNotFound deep inside agent rendering. continue } - defaults[templates.PromptType(p.Type)] = p.Prompt + + // Validation runs only on save, so an override written before a variable + // was removed still loads, and text/template renders the missing key as + // "" without an error — the agent would be pointed at a tool + // that does not exist. + promptType := templates.PromptType(p.Type) + if err := validator.ValidatePrompt(promptType, p.Prompt); err != nil { + logrus.WithError(err).WithField("prompt_type", p.Type). + Warn("custom prompt no longer validates, using the default") + continue + } + defaults[promptType] = p.Prompt } return templates.NewFlowPrompter(defaults) diff --git a/backend/pkg/controller/prompter_test.go b/backend/pkg/controller/prompter_test.go index a80804a1..6b2621a1 100644 --- a/backend/pkg/controller/prompter_test.go +++ b/backend/pkg/controller/prompter_test.go @@ -65,6 +65,15 @@ func TestPrompter_NewUserPrompter_OverlaysTheUsersPromptsOnTheDefaults(t *testin prompts: []database.Prompt{{Type: database.PromptTypePrimaryAgent, Prompt: ""}}, wantDefault: []templates.PromptType{templates.PromptTypePrimaryAgent}, }, + { + name: "an override using a variable the backend no longer provides", + prompts: []database.Prompt{ + {Type: database.PromptTypeSearcher, Prompt: "Search with {{.GoogleToolName}}"}, + {Type: database.PromptTypeCoder, Prompt: "custom coder"}, + }, + wantCustom: map[templates.PromptType]string{templates.PromptTypeCoder: "custom coder"}, + wantDefault: []templates.PromptType{templates.PromptTypeSearcher}, + }, {name: "a database that cannot be read", dbErr: dbErr}, } { t.Run(tc.name, func(t *testing.T) { diff --git a/backend/pkg/docker/client_test.go b/backend/pkg/docker/client_test.go index 63175e39..5b7d4bba 100644 --- a/backend/pkg/docker/client_test.go +++ b/backend/pkg/docker/client_test.go @@ -236,7 +236,7 @@ func listingFrame(streamID byte, payload string) []byte { func TestClient_IsContainerRunning_ReportsRunningUntilTheContainerIsRemoved(t *testing.T) { dc := newDaemonClient(t) - containerID := startProbeSandbox(t, dc, probeImage, 0) + containerID := startProbeSandbox(t, dc, probeImage) running, err := dc.IsContainerRunning(t.Context(), containerID) require.NoError(t, err) @@ -846,7 +846,7 @@ func TestClient_KillFlowCommands_SkipsASandboxThatIsNotRunning(t *testing.T) { for name, stop := range tests { t.Run(name, func(t *testing.T) { - containerID := startProbeSandbox(t, dc, probeImage, 0) + containerID := startProbeSandbox(t, dc, probeImage) require.NoError(t, stop(t.Context(), containerID)) require.NoError(t, dc.KillFlowCommands(t.Context(), containerID)) @@ -864,7 +864,7 @@ func TestClient_KillFlowCommands_FinishesTheSweepAfterTheCallerGivesUp(t *testin for name, giveUpAfter := range tests { t.Run(name, func(t *testing.T) { - containerID := startProbeSandbox(t, dc, probeImage, 0) + containerID := startProbeSandbox(t, dc, probeImage) startInSandbox(t, dc, containerID, FlowCommand(`trap "" TERM; while :; do sleep 1; done`), false) dockerWaitForCommand(t, dc, containerID, "do sleep 1") @@ -919,7 +919,7 @@ func TestClient_KillFlowCommands_SweepsAnUnhealthySandbox(t *testing.T) { func TestClient_KillFlowCommands_RunsTheShellFoundOnThePath(t *testing.T) { dc := newDaemonClient(t) - containerID := startProbeSandbox(t, dc, probeImage, 0) + containerID := startProbeSandbox(t, dc, probeImage) runInSandbox(t, dc, containerID, `mv /bin/sh /usr/local/bin/sh`) startInSandbox(t, dc, containerID, FlowCommand(`sleep 921`), false) @@ -932,7 +932,7 @@ func TestClient_KillFlowCommands_RunsTheShellFoundOnThePath(t *testing.T) { func TestClient_KillFlowCommands_ReportsASweepThatCannotRun(t *testing.T) { dc := newDaemonClient(t) - containerID := startProbeSandbox(t, dc, probeImage, 0) + containerID := startProbeSandbox(t, dc, probeImage) runInSandbox(t, dc, containerID, `rm /bin/sh`) diff --git a/backend/pkg/docker/fixtures_test.go b/backend/pkg/docker/fixtures_test.go index 6f7afe39..e00c5482 100644 --- a/backend/pkg/docker/fixtures_test.go +++ b/backend/pkg/docker/fixtures_test.go @@ -45,15 +45,10 @@ func newDaemonClient(t *testing.T) *dockerClient { return &dockerClient{client: cli, logger: logger} } -func startProbeSandbox(t *testing.T, dc *dockerClient, image string, pidsLimit int64) string { +func startProbeSandbox(t *testing.T, dc *dockerClient, image string) string { t.Helper() - hostConfig := &container.HostConfig{} - if pidsLimit > 0 { - hostConfig.PidsLimit = &pidsLimit - } - - return startSandbox(t, dc, &container.Config{Image: image}, hostConfig) + return startSandbox(t, dc, &container.Config{Image: image}, &container.HostConfig{}) } func startSandbox(t *testing.T, dc *dockerClient, config *container.Config, hostConfig *container.HostConfig) string { diff --git a/backend/pkg/docker/flow_commands_test.go b/backend/pkg/docker/flow_commands_test.go index b639800e..572149c8 100644 --- a/backend/pkg/docker/flow_commands_test.go +++ b/backend/pkg/docker/flow_commands_test.go @@ -10,6 +10,7 @@ import ( "testing" "time" + "github.com/moby/moby/api/types/container" "github.com/moby/moby/client" "github.com/stretchr/testify/require" ) @@ -92,7 +93,7 @@ func TestFlowCommands_FlowCommand_StopEndsItsCommandsAndNothingElse(t *testing.T for name, image := range images { t.Run(name, func(t *testing.T) { - containerID := startProbeSandbox(t, dc, image, 0) + containerID := startProbeSandbox(t, dc, image) for _, flow := range flowCommands { startInSandbox(t, dc, containerID, FlowCommand(flow.command), flow.tty) @@ -149,7 +150,7 @@ func TestFlowCommands_WrapperAndSweepRunAtThePidsLimit(t *testing.T) { const pidsLimit = 32 dc := newDaemonClient(t) - containerID := startProbeSandbox(t, dc, probeImage, pidsLimit) + containerID := startProbeSandbox(t, dc, probeImage) putSandboxFile(t, dc, containerID, "sweep.sh", killFlowCommandsScript) startInSandbox(t, dc, containerID, []string{"sh", "-c", "sleep 906"}, true) @@ -187,6 +188,13 @@ exec tail -f /dev/null`}, false) strings.Count(commands, "while [ ! -e /tmp/go-") == len(runners)+2 }, 15*time.Second, 100*time.Millisecond, "the probe commands never started") + // Set only now: each exec's runc init counts its threads against pids.max while it starts. + limit := int64(pidsLimit) + _, err := dc.client.ContainerUpdate(t.Context(), containerID, client.ContainerUpdateOptions{ + Resources: &container.Resources{PidsLimit: &limit}, + }) + require.NoError(t, err) + fire := func(trigger string, done func() bool, failure string) { putSandboxFile(t, dc, containerID, trigger, "") require.Eventually(t, done, 10*time.Second, 100*time.Millisecond, failure) @@ -236,7 +244,7 @@ func TestFlowCommands_BackgroundCommandsOutliveTheTerminalThatStartedThem(t *tes for name, tc := range tests { t.Run(name, func(t *testing.T) { - containerID := startProbeSandbox(t, dc, probeImage, 0) + containerID := startProbeSandbox(t, dc, probeImage) select { case err := <-startInSandbox(t, dc, containerID, tc.launch(tc.command+` >/dev/null 2>&1 &`), true): diff --git a/backend/pkg/providers/provider/failover.go b/backend/pkg/providers/provider/failover.go index 85959c77..1e3d79f4 100644 --- a/backend/pkg/providers/provider/failover.go +++ b/backend/pkg/providers/provider/failover.go @@ -5,6 +5,7 @@ import ( "errors" "pentagi/pkg/providers/pconfig" + "pentagi/pkg/templates" "github.com/vxcontrol/langchaingo/llms" "github.com/vxcontrol/langchaingo/llms/streaming" @@ -22,9 +23,9 @@ type failoverProvider struct { } // WithFailover returns a provider that repeats a failed call on backup. -// Everything the interface exposes beyond the four calling methods keeps -// answering for primary: a flow is still described by the provider it was -// created with. +// Everything the interface exposes beyond the calling methods and the tool +// call ID template keeps answering for primary: a flow is still described by +// the provider it was created with. func WithFailover(primary, backup Provider, notifier SwitchNotifier) Provider { if backup == nil || primary == nil || primary.Name() == backup.Name() { return primary @@ -113,3 +114,17 @@ func (f *failoverProvider) CallWithExtraOptions( return f.backup.CallWithExtraOptions(ctx, opt, chain, tools, streamCb, extra...) } + +// GetToolCallIDTemplate is detected by calling the model when a flow or an +// assistant is created, so without a switch here a primary that is down +// fails the creation before any agent call could fall over. +func (f *failoverProvider) GetToolCallIDTemplate(ctx context.Context, prompter templates.Prompter) (string, error) { + template, err := f.Provider.GetToolCallIDTemplate(ctx, prompter) + if !switchable(ctx, err) { + return template, err + } + + f.switched(ctx, err) + + return f.backup.GetToolCallIDTemplate(ctx, prompter) +} diff --git a/backend/pkg/providers/provider/failover_test.go b/backend/pkg/providers/provider/failover_test.go index b492238f..5591695f 100644 --- a/backend/pkg/providers/provider/failover_test.go +++ b/backend/pkg/providers/provider/failover_test.go @@ -7,6 +7,7 @@ import ( "testing" "pentagi/pkg/providers/pconfig" + "pentagi/pkg/templates" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -66,6 +67,10 @@ func (f *fakeProvider) CallWithExtraOptions( return f.chainAnswer(chain) } +func (f *fakeProvider) GetToolCallIDTemplate(ctx context.Context, _ templates.Prompter) (string, error) { + return f.Call(ctx, pconfig.OptionsTypeSimple, "hi") +} + func failoverContent(resp *llms.ContentResponse, err error) (string, error) { if err != nil { return "", err @@ -73,7 +78,7 @@ func failoverContent(resp *llms.ContentResponse, err error) (string, error) { return resp.Choices[0].Content, nil } -// failoverPaths drives the four calling methods with the same input. +// failoverPaths drives every method that reaches the model with the same input. var failoverPaths = []struct { name string call func(ctx context.Context, prv Provider) (string, error) @@ -93,6 +98,9 @@ var failoverPaths = []struct { return failoverContent(prv.CallWithExtraOptions(ctx, pconfig.OptionsTypeSimple, []llms.MessageContent{llms.TextParts(llms.ChatMessageTypeHuman, "hi")}, nil, nil)) }}, + {"the tool call id template", func(ctx context.Context, prv Provider) (string, error) { + return prv.GetToolCallIDTemplate(ctx, nil) + }}, } // TestFailover_SwitchesToTheBackupOnlyWhenThePrimaryFails runs every case through each calling method. diff --git a/backend/pkg/server/services/users.go b/backend/pkg/server/services/users.go index f069b505..3c51e5f1 100644 --- a/backend/pkg/server/services/users.go +++ b/backend/pkg/server/services/users.go @@ -686,7 +686,10 @@ func (s *UserService) PatchUser(c *gin.Context) { return db.Where("hash = ? AND id = ?", hash, uid) } } - if !slices.Contains(privs, "users.edit") && uhash != hash { + // Without users.edit only the caller's own record is editable, and not its + // password: /user/password asks for the current one, so a stolen cookie + // alone cannot take the account over. + if !slices.Contains(privs, "users.edit") && (uhash != hash || user.Password != "") { logger.FromContext(c).Errorf("error filtering user role permissions: permission not found") response.Error(c, response.ErrNotPermitted, nil) return diff --git a/backend/pkg/server/services/users_test.go b/backend/pkg/server/services/users_test.go index 558af169..dbc357c5 100644 --- a/backend/pkg/server/services/users_test.go +++ b/backend/pkg/server/services/users_test.go @@ -299,12 +299,12 @@ type passwordHarness struct { hash string } -func newPasswordHarness(t *testing.T) *passwordHarness { +func newPasswordHarness(t *testing.T, roleID int) *passwordHarness { t.Helper() db := setupTestDB(t) t.Cleanup(func() { db.Close() }) - seedLocalUser(t, db, "known@corp.com") + seedLocalAccount(t, db, "known@corp.com", roleID) refuse := false gin.SetMode(gin.TestMode) @@ -425,16 +425,17 @@ func (h *passwordHarness) changeThroughTheEditor(t *testing.T) *httptest.Respons // usersPasswordDoors are the handlers that reach changePassword for the account's own password. var usersPasswordDoors = []struct { name string + role int change func(*passwordHarness, *testing.T) *httptest.ResponseRecorder }{ - {"through the password form", (*passwordHarness).changeThroughTheForm}, - {"through the user editor", (*passwordHarness).changeThroughTheEditor}, + {"a user through the password form", 2, (*passwordHarness).changeThroughTheForm}, + {"an administrator through the user editor", 1, (*passwordHarness).changeThroughTheEditor}, } func TestUsers_ChangePassword_EndsOtherSessionsAndKeepsTheCaller(t *testing.T) { for _, door := range usersPasswordDoors { t.Run(door.name, func(t *testing.T) { - h := newPasswordHarness(t) + h := newPasswordHarness(t, door.role) before, _ := h.state(t) stolen := slices.Clone(h.cookies) @@ -451,20 +452,34 @@ func TestUsers_ChangePassword_EndsOtherSessionsAndKeepsTheCaller(t *testing.T) { } } -func TestUsers_ChangePassword_KeepsAnAdministratorResettingAnotherAccountSignedIn(t *testing.T) { - h := newPasswordHarness(t) - seedLocalAccount(t, h.db, "boss.admin@corp.com", 1) - before, _ := h.state(t) +func TestUsers_ChangePassword_ThroughTheEditorTakesUsersEdit(t *testing.T) { + t.Run("an administrator resetting another account stays signed in", func(t *testing.T) { + h := newPasswordHarness(t, 2) + seedLocalAccount(t, h.db, "boss.admin@corp.com", 1) + before, _ := h.state(t) - h.cookies = h.login(t, "boss.admin@corp.com") - rec := h.changeThroughTheEditor(t) - require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) + h.cookies = h.login(t, "boss.admin@corp.com") + rec := h.changeThroughTheEditor(t) + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) - after, _ := h.state(t) - assert.Equal(t, before+1, after, "the user's sessions end") - h.cookieGeneration(t, rec.Result().Cookies()) - assert.Equal(t, http.StatusOK, h.privateStatus(t, h.cookies), - "the administrator's cookie is not stamped with the other account's generation") + after, _ := h.state(t) + assert.Equal(t, before+1, after, "the user's sessions end") + h.cookieGeneration(t, rec.Result().Cookies()) + assert.Equal(t, http.StatusOK, h.privateStatus(t, h.cookies), + "the administrator's cookie is not stamped with the other account's generation") + }) + + t.Run("a user cannot set their own password without the current one", func(t *testing.T) { + h := newPasswordHarness(t, 2) + beforeGeneration, beforePassword := h.state(t) + + rec := h.changeThroughTheEditor(t) + + assert.Equal(t, http.StatusForbidden, rec.Code, rec.Body.String()) + afterGeneration, afterPassword := h.state(t) + assert.Equal(t, beforeGeneration, afterGeneration) + assert.Equal(t, beforePassword, afterPassword) + }) } func TestUsers_ChangePassword_ChangesNothingWhenAStepFails(t *testing.T) { @@ -500,7 +515,7 @@ func TestUsers_ChangePassword_ChangesNothingWhenAStepFails(t *testing.T) { } { for _, door := range usersPasswordDoors { t.Run(fault.name+", "+door.name, func(t *testing.T) { - h := newPasswordHarness(t) + h := newPasswordHarness(t, door.role) beforeGeneration, beforePassword := h.state(t) fault.inject(t, h) diff --git a/docker-compose-langfuse.yml b/docker-compose-langfuse.yml index 16f78af8..06a18bb4 100644 --- a/docker-compose-langfuse.yml +++ b/docker-compose-langfuse.yml @@ -177,10 +177,11 @@ services: - langfuse-network minio: - image: ${MINIO_IMAGE:-minio/minio:RELEASE.2025-07-23T15-54-02Z} + image: ${MINIO_IMAGE:-cgr.dev/chainguard/minio:latest} restart: unless-stopped container_name: langfuse-minio hostname: langfuse-minio + user: "0:0" command: server /data --console-address ":9001" --address ":9000" --json environment: MINIO_ROOT_USER: ${LANGFUSE_S3_ACCESS_KEY_ID:-minio} diff --git a/examples/tests/azure-openai-report.md b/examples/tests/azure-openai-report.md new file mode 100644 index 00000000..9f30af3b --- /dev/null +++ b/examples/tests/azure-openai-report.md @@ -0,0 +1,551 @@ +# LLM Agent Testing Report + +Generated: Wed, 30 Sep 2026 16:33:34 UTC + +## Overall Results + +| Agent | Model | Reasoning | Success Rate | Average Latency | +|-------|-------|-----------|--------------|-----------------| +| simple | gpt-4.1-mini | false | 25/25 (100.00%) | 1.680s | +| simple_json | gpt-4.1-mini | false | 8/8 (100.00%) | 1.039s | +| primary_agent | o4-mini | true | 24/25 (96.00%) | 2.618s | +| assistant | o4-mini | true | 24/25 (96.00%) | 2.603s | +| generator | o4-mini | true | 24/25 (96.00%) | 2.988s | +| refiner | gpt-4.1 | false | 24/25 (96.00%) | 1.811s | +| adviser | o4-mini | true | 15/16 (93.75%) | 1.982s | +| reflector | o4-mini | true | 16/16 (100.00%) | 2.225s | +| searcher | gpt-4.1-mini | false | 25/25 (100.00%) | 1.585s | +| enricher | gpt-4.1-mini | false | 25/25 (100.00%) | 1.737s | +| coder | gpt-4.1 | false | 25/25 (100.00%) | 1.348s | +| installer | gpt-4.1 | false | 25/25 (100.00%) | 2.029s | +| pentester | o4-mini | true | 25/25 (100.00%) | 2.610s | + +**Total**: 285/290 (98.28%) successful tests +**Overall average latency**: 2.072s + +## Detailed Results + +### simple (gpt-4.1-mini) + +#### Basic Tests + +| Test | Result | Latency | Error | +|------|--------|---------|-------| +| Streaming Count from 1 to 3 Streaming | ✅ Pass | 0.544s | | +| Simple Math | ✅ Pass | 0.996s | | +| Text Transform Uppercase | ✅ Pass | 1.060s | | +| Streaming Simple Math Streaming | ✅ Pass | 1.133s | | +| Basic Echo Function | ✅ Pass | 1.244s | | +| Count from 1 to 5 | ✅ Pass | 1.282s | | +| Math Calculation | ✅ Pass | 1.430s | | +| Streaming Basic Echo Function Streaming | ✅ Pass | 1.511s | | +| Answer Stops At The Output Limit | ✅ Pass | 15.265s | | + +#### Advanced Tests + +| Test | Result | Latency | Error | +|------|--------|---------|-------| +| JSON Response Function | ✅ Pass | 0.739s | | +| Search Query Function | ✅ Pass | 0.702s | | +| Ask Advice Function | ✅ Pass | 1.013s | | +| Streaming Search Query Function Streaming | ✅ Pass | 1.095s | | +| Basic Context Memory Test | ✅ Pass | 1.012s | | +| Function Argument Memory Test | ✅ Pass | 1.436s | | +| Function Response Memory Test | ✅ Pass | 0.823s | | +| Cybersecurity Workflow Memory Test | ✅ Pass | 0.657s | | +| Penetration Testing Memory with Tool Call | ✅ Pass | 1.590s | | +| Read a file, then edit it via unified diff | ✅ Pass | 3.133s | | +| SQL Injection Attack Type | ✅ Pass | 0.848s | | +| Penetration Testing Methodology | ✅ Pass | 1.044s | | +| Penetration Testing Framework | ✅ Pass | 0.701s | | +| Web Application Security Scanner | ✅ Pass | 0.583s | | +| Vulnerability Assessment Tools | ✅ Pass | 1.133s | | +| Penetration Testing Tool Selection | ✅ Pass | 1.021s | | + +**Summary**: 25/25 (100.00%) successful tests + +**Average latency**: 1.680s + +--- + +### simple_json (gpt-4.1-mini) + +#### Advanced Tests + +| Test | Result | Latency | Error | +|------|--------|---------|-------| +| Vulnerability Report Memory Test | ✅ Pass | 1.698s | | +| Person Information JSON | ✅ Pass | 0.861s | | +| User Profile JSON | ✅ Pass | 0.986s | | +| Project Information JSON | ✅ Pass | 1.410s | | +| Streaming Person Information JSON Streaming | ✅ Pass | 1.006s | | +| JSON Array Response Without Schema | ✅ Pass | 1.591s | | + +#### Capability Tests + +| Test | Capability | Result | Latency | Note | +|------|------------|--------|---------|------| +| Structured Output Refuses A Non-Object Schema | structured_output | ✅ Pass | 0.000s | | +| Structured Output With JSON Schema | structured_output | ✅ Pass | 0.751s | | + +**Summary**: 8/8 (100.00%) successful tests + +**Average latency**: 1.039s + +--- + +### primary_agent (o4-mini) + +#### Basic Tests + +| Test | Result | Latency | Error | +|------|--------|---------|-------| +| Count from 1 to 5 | ✅ Pass | 1.401s | | +| Text Transform Uppercase | ✅ Pass | 1.541s | | +| Simple Math | ✅ Pass | 1.734s | | +| Math Calculation | ✅ Pass | 1.552s | | +| Basic Echo Function | ✅ Pass | 2.705s | | +| Streaming Simple Math Streaming | ✅ Pass | 2.204s | | +| Streaming Count from 1 to 3 Streaming | ✅ Pass | 2.368s | | +| Answer Stops At The Output Limit | ❌ Fail | 4.833s | expected the answer to stop at the output limit, got stop reason "stop" | +| Streaming Basic Echo Function Streaming | ✅ Pass | 7.237s | | + +#### Advanced Tests + +| Test | Result | Latency | Error | +|------|--------|---------|-------| +| Search Query Function | ✅ Pass | 1.862s | | +| JSON Response Function | ✅ Pass | 2.115s | | +| Streaming Search Query Function Streaming | ✅ Pass | 1.807s | | +| Basic Context Memory Test | ✅ Pass | 1.636s | | +| Function Argument Memory Test | ✅ Pass | 1.660s | | +| Ask Advice Function | ✅ Pass | 2.890s | | +| Function Response Memory Test | ✅ Pass | 1.789s | | +| Penetration Testing Memory with Tool Call | ✅ Pass | 2.388s | | +| Cybersecurity Workflow Memory Test | ✅ Pass | 2.550s | | +| Read a file, then edit it via unified diff | ✅ Pass | 8.723s | | +| Penetration Testing Framework | ✅ Pass | 1.582s | | +| Penetration Testing Methodology | ✅ Pass | 2.130s | | +| SQL Injection Attack Type | ✅ Pass | 2.003s | | +| Web Application Security Scanner | ✅ Pass | 2.034s | | +| Vulnerability Assessment Tools | ✅ Pass | 2.322s | | +| Penetration Testing Tool Selection | ✅ Pass | 2.364s | | + +**Summary**: 24/25 (96.00%) successful tests + +**Average latency**: 2.618s + +--- + +### assistant (o4-mini) + +#### Basic Tests + +| Test | Result | Latency | Error | +|------|--------|---------|-------| +| Simple Math | ✅ Pass | 1.493s | | +| Math Calculation | ✅ Pass | 1.033s | | +| Count from 1 to 5 | ✅ Pass | 1.432s | | +| Text Transform Uppercase | ✅ Pass | 1.622s | | +| Streaming Simple Math Streaming | ✅ Pass | 1.777s | | +| Basic Echo Function | ✅ Pass | 2.297s | | +| Streaming Count from 1 to 3 Streaming | ✅ Pass | 1.770s | | +| Streaming Basic Echo Function Streaming | ✅ Pass | 5.221s | | +| Answer Stops At The Output Limit | ✅ Pass | 13.484s | | + +#### Advanced Tests + +| Test | Result | Latency | Error | +|------|--------|---------|-------| +| JSON Response Function | ✅ Pass | 1.911s | | +| Search Query Function | ✅ Pass | 2.081s | | +| Streaming Search Query Function Streaming | ✅ Pass | 1.587s | | +| Ask Advice Function | ✅ Pass | 2.801s | | +| Function Argument Memory Test | ✅ Pass | 1.676s | | +| Function Response Memory Test | ✅ Pass | 1.604s | | +| Basic Context Memory Test | ✅ Pass | 1.983s | | +| Penetration Testing Memory with Tool Call | ✅ Pass | 1.577s | | +| Cybersecurity Workflow Memory Test | ✅ Pass | 2.080s | | +| Read a file, then edit it via unified diff | ❌ Fail | 5.404s | edit\_file's diff did not apply: expected a hunk header \("@@ \-old +new @@"\) but found: "\*\*\* Begin Patch"; stop reason: tool\_calls | +| Penetration Testing Methodology | ✅ Pass | 1.584s | | +| SQL Injection Attack Type | ✅ Pass | 2.122s | | +| Penetration Testing Tool Selection | ✅ Pass | 1.739s | | +| Web Application Security Scanner | ✅ Pass | 1.890s | | +| Penetration Testing Framework | ✅ Pass | 1.991s | | +| Vulnerability Assessment Tools | ✅ Pass | 2.895s | | + +**Summary**: 24/25 (96.00%) successful tests + +**Average latency**: 2.603s + +--- + +### generator (o4-mini) + +#### Basic Tests + +| Test | Result | Latency | Error | +|------|--------|---------|-------| +| Text Transform Uppercase | ✅ Pass | 2.071s | | +| Count from 1 to 5 | ✅ Pass | 2.041s | | +| Simple Math | ✅ Pass | 2.620s | | +| Math Calculation | ✅ Pass | 2.069s | | +| Basic Echo Function | ✅ Pass | 2.137s | | +| Streaming Simple Math Streaming | ✅ Pass | 2.680s | | +| Streaming Count from 1 to 3 Streaming | ✅ Pass | 2.590s | | +| Streaming Basic Echo Function Streaming | ✅ Pass | 2.588s | | +| Answer Stops At The Output Limit | ✅ Pass | 14.565s | | + +#### Advanced Tests + +| Test | Result | Latency | Error | +|------|--------|---------|-------| +| Streaming Search Query Function Streaming | ✅ Pass | 1.265s | | +| Function Argument Memory Test | ✅ Pass | 1.405s | | +| Search Query Function | ✅ Pass | 2.323s | | +| Basic Context Memory Test | ✅ Pass | 2.330s | | +| Ask Advice Function | ✅ Pass | 2.543s | | +| Function Response Memory Test | ✅ Pass | 2.227s | | +| Cybersecurity Workflow Memory Test | ✅ Pass | 1.892s | | +| Penetration Testing Memory with Tool Call | ✅ Pass | 2.775s | | +| JSON Response Function | ✅ Pass | 7.147s | | +| Read a file, then edit it via unified diff | ❌ Fail | 5.006s | edit\_file's diff did not apply: expected a hunk header \("@@ \-old +new @@"\) but found: "\*\*\* Begin Patch"; stop reason: tool\_calls | +| Penetration Testing Methodology | ✅ Pass | 2.263s | | +| SQL Injection Attack Type | ✅ Pass | 1.747s | | +| Vulnerability Assessment Tools | ✅ Pass | 2.395s | | +| Web Application Security Scanner | ✅ Pass | 1.588s | | +| Penetration Testing Framework | ✅ Pass | 1.646s | | +| Penetration Testing Tool Selection | ✅ Pass | 2.770s | | + +**Summary**: 24/25 (96.00%) successful tests + +**Average latency**: 2.988s + +--- + +### refiner (gpt-4.1) + +#### Basic Tests + +| Test | Result | Latency | Error | +|------|--------|---------|-------| +| Simple Math | ✅ Pass | 0.905s | | +| Text Transform Uppercase | ✅ Pass | 0.748s | | +| Count from 1 to 5 | ✅ Pass | 0.947s | | +| Math Calculation | ✅ Pass | 1.253s | | +| Basic Echo Function | ✅ Pass | 0.922s | | +| Streaming Count from 1 to 3 Streaming | ✅ Pass | 1.098s | | +| Streaming Basic Echo Function Streaming | ✅ Pass | 1.258s | | +| Streaming Simple Math Streaming | ✅ Pass | 1.611s | | +| Answer Stops At The Output Limit | ❌ Fail | 14.009s | expected the answer to stop at the output limit, got stop reason "stop" | + +#### Advanced Tests + +| Test | Result | Latency | Error | +|------|--------|---------|-------| +| JSON Response Function | ✅ Pass | 1.443s | | +| Search Query Function | ✅ Pass | 1.229s | | +| Ask Advice Function | ✅ Pass | 1.521s | | +| Streaming Search Query Function Streaming | ✅ Pass | 1.123s | | +| Function Argument Memory Test | ✅ Pass | 0.670s | | +| Function Response Memory Test | ✅ Pass | 0.774s | | +| Basic Context Memory Test | ✅ Pass | 1.180s | | +| Cybersecurity Workflow Memory Test | ✅ Pass | 0.954s | | +| Penetration Testing Memory with Tool Call | ✅ Pass | 1.041s | | +| Read a file, then edit it via unified diff | ✅ Pass | 4.832s | | +| SQL Injection Attack Type | ✅ Pass | 1.047s | | +| Vulnerability Assessment Tools | ✅ Pass | 1.170s | | +| Penetration Testing Methodology | ✅ Pass | 1.696s | | +| Web Application Security Scanner | ✅ Pass | 0.650s | | +| Penetration Testing Framework | ✅ Pass | 1.444s | | +| Penetration Testing Tool Selection | ✅ Pass | 1.737s | | + +**Summary**: 24/25 (96.00%) successful tests + +**Average latency**: 1.811s + +--- + +### adviser (o4-mini) + +#### Basic Tests + +| Test | Result | Latency | Error | +|------|--------|---------|-------| +| Text Transform Uppercase | ✅ Pass | 1.435s | | +| Math Calculation | ✅ Pass | 1.337s | | +| Answer Stops At The Output Limit | ❌ Fail | 2.622s | expected the answer to stop at the output limit, got stop reason "stop" | +| Count from 1 to 5 | ✅ Pass | 1.813s | | +| Simple Math | ✅ Pass | 2.365s | | +| Streaming Simple Math Streaming | ✅ Pass | 1.324s | | +| Streaming Count from 1 to 3 Streaming | ✅ Pass | 1.171s | | + +#### Advanced Tests + +| Test | Result | Latency | Error | +|------|--------|---------|-------| +| Basic Context Memory Test | ✅ Pass | 1.999s | | +| Function Response Memory Test | ✅ Pass | 1.943s | | +| Function Argument Memory Test | ✅ Pass | 2.681s | | +| Cybersecurity Workflow Memory Test | ✅ Pass | 2.377s | | +| Penetration Testing Framework | ✅ Pass | 1.757s | | +| Web Application Security Scanner | ✅ Pass | 1.811s | | +| Penetration Testing Methodology | ✅ Pass | 2.048s | | +| SQL Injection Attack Type | ✅ Pass | 2.043s | | +| Vulnerability Assessment Tools | ✅ Pass | 2.971s | | + +**Summary**: 15/16 (93.75%) successful tests + +**Average latency**: 1.982s + +--- + +### reflector (o4-mini) + +#### Basic Tests + +| Test | Result | Latency | Error | +|------|--------|---------|-------| +| Text Transform Uppercase | ✅ Pass | 1.478s | | +| Count from 1 to 5 | ✅ Pass | 1.652s | | +| Streaming Simple Math Streaming | ✅ Pass | 1.121s | | +| Math Calculation | ✅ Pass | 1.232s | | +| Simple Math | ✅ Pass | 2.111s | | +| Streaming Count from 1 to 3 Streaming | ✅ Pass | 1.700s | | +| Answer Stops At The Output Limit | ✅ Pass | 9.038s | | + +#### Advanced Tests + +| Test | Result | Latency | Error | +|------|--------|---------|-------| +| Function Argument Memory Test | ✅ Pass | 1.333s | | +| Basic Context Memory Test | ✅ Pass | 1.992s | | +| Cybersecurity Workflow Memory Test | ✅ Pass | 2.230s | | +| Function Response Memory Test | ✅ Pass | 2.552s | | +| Penetration Testing Methodology | ✅ Pass | 1.489s | | +| Vulnerability Assessment Tools | ✅ Pass | 2.083s | | +| SQL Injection Attack Type | ✅ Pass | 1.749s | | +| Penetration Testing Framework | ✅ Pass | 1.812s | | +| Web Application Security Scanner | ✅ Pass | 2.024s | | + +**Summary**: 16/16 (100.00%) successful tests + +**Average latency**: 2.225s + +--- + +### searcher (gpt-4.1-mini) + +#### Basic Tests + +| Test | Result | Latency | Error | +|------|--------|---------|-------| +| Count from 1 to 5 | ✅ Pass | 0.627s | | +| Text Transform Uppercase | ✅ Pass | 0.734s | | +| Simple Math | ✅ Pass | 0.801s | | +| Streaming Simple Math Streaming | ✅ Pass | 0.608s | | +| Math Calculation | ✅ Pass | 0.842s | | +| Basic Echo Function | ✅ Pass | 0.921s | | +| Streaming Count from 1 to 3 Streaming | ✅ Pass | 0.685s | | +| Streaming Basic Echo Function Streaming | ✅ Pass | 0.846s | | +| Answer Stops At The Output Limit | ✅ Pass | 16.953s | | + +#### Advanced Tests + +| Test | Result | Latency | Error | +|------|--------|---------|-------| +| Search Query Function | ✅ Pass | 0.804s | | +| JSON Response Function | ✅ Pass | 0.980s | | +| Basic Context Memory Test | ✅ Pass | 0.763s | | +| Streaming Search Query Function Streaming | ✅ Pass | 0.915s | | +| Function Argument Memory Test | ✅ Pass | 0.652s | | +| Ask Advice Function | ✅ Pass | 1.017s | | +| Function Response Memory Test | ✅ Pass | 0.815s | | +| Penetration Testing Memory with Tool Call | ✅ Pass | 1.286s | | +| Cybersecurity Workflow Memory Test | ✅ Pass | 1.580s | | +| Read a file, then edit it via unified diff | ✅ Pass | 2.436s | | +| Penetration Testing Methodology | ✅ Pass | 0.671s | | +| Vulnerability Assessment Tools | ✅ Pass | 0.814s | | +| Penetration Testing Framework | ✅ Pass | 0.701s | | +| Web Application Security Scanner | ✅ Pass | 0.544s | | +| SQL Injection Attack Type | ✅ Pass | 1.647s | | +| Penetration Testing Tool Selection | ✅ Pass | 0.981s | | + +**Summary**: 25/25 (100.00%) successful tests + +**Average latency**: 1.585s + +--- + +### enricher (gpt-4.1-mini) + +#### Basic Tests + +| Test | Result | Latency | Error | +|------|--------|---------|-------| +| Text Transform Uppercase | ✅ Pass | 0.553s | | +| Simple Math | ✅ Pass | 0.739s | | +| Count from 1 to 5 | ✅ Pass | 0.754s | | +| Math Calculation | ✅ Pass | 0.592s | | +| Streaming Simple Math Streaming | ✅ Pass | 0.577s | | +| Basic Echo Function | ✅ Pass | 0.775s | | +| Streaming Count from 1 to 3 Streaming | ✅ Pass | 0.541s | | +| Streaming Basic Echo Function Streaming | ✅ Pass | 0.736s | | +| Answer Stops At The Output Limit | ✅ Pass | 16.991s | | + +#### Advanced Tests + +| Test | Result | Latency | Error | +|------|--------|---------|-------| +| Search Query Function | ✅ Pass | 0.684s | | +| Streaming Search Query Function Streaming | ✅ Pass | 0.811s | | +| JSON Response Function | ✅ Pass | 1.059s | | +| Ask Advice Function | ✅ Pass | 1.121s | | +| Function Response Memory Test | ✅ Pass | 0.641s | | +| Function Argument Memory Test | ✅ Pass | 0.943s | | +| Cybersecurity Workflow Memory Test | ✅ Pass | 1.372s | | +| Penetration Testing Memory with Tool Call | ✅ Pass | 1.815s | | +| Basic Context Memory Test | ✅ Pass | 4.891s | | +| Read a file, then edit it via unified diff | ✅ Pass | 2.303s | | +| SQL Injection Attack Type | ✅ Pass | 0.623s | | +| Penetration Testing Methodology | ✅ Pass | 0.831s | | +| Penetration Testing Framework | ✅ Pass | 0.624s | | +| Web Application Security Scanner | ✅ Pass | 0.643s | | +| Vulnerability Assessment Tools | ✅ Pass | 1.176s | | +| Penetration Testing Tool Selection | ✅ Pass | 1.611s | | + +**Summary**: 25/25 (100.00%) successful tests + +**Average latency**: 1.737s + +--- + +### coder (gpt-4.1) + +#### Basic Tests + +| Test | Result | Latency | Error | +|------|--------|---------|-------| +| Simple Math | ✅ Pass | 1.052s | | +| Text Transform Uppercase | ✅ Pass | 1.382s | | +| Count from 1 to 5 | ✅ Pass | 1.079s | | +| Math Calculation | ✅ Pass | 1.055s | | +| Streaming Simple Math Streaming | ✅ Pass | 0.875s | | +| Basic Echo Function | ✅ Pass | 0.927s | | +| Streaming Count from 1 to 3 Streaming | ✅ Pass | 0.758s | | +| Streaming Basic Echo Function Streaming | ✅ Pass | 0.852s | | +| Answer Stops At The Output Limit | ✅ Pass | 6.824s | | + +#### Advanced Tests + +| Test | Result | Latency | Error | +|------|--------|---------|-------| +| Search Query Function | ✅ Pass | 0.832s | | +| Ask Advice Function | ✅ Pass | 0.780s | | +| Streaming Search Query Function Streaming | ✅ Pass | 0.861s | | +| Function Argument Memory Test | ✅ Pass | 0.644s | | +| Function Response Memory Test | ✅ Pass | 0.861s | | +| Cybersecurity Workflow Memory Test | ✅ Pass | 0.759s | | +| JSON Response Function | ✅ Pass | 2.149s | | +| Basic Context Memory Test | ✅ Pass | 1.593s | | +| Penetration Testing Memory with Tool Call | ✅ Pass | 1.103s | | +| Read a file, then edit it via unified diff | ✅ Pass | 2.438s | | +| Penetration Testing Framework | ✅ Pass | 0.605s | | +| SQL Injection Attack Type | ✅ Pass | 0.691s | | +| Vulnerability Assessment Tools | ✅ Pass | 0.966s | | +| Web Application Security Scanner | ✅ Pass | 1.036s | | +| Penetration Testing Methodology | ✅ Pass | 1.498s | | +| Penetration Testing Tool Selection | ✅ Pass | 2.065s | | + +**Summary**: 25/25 (100.00%) successful tests + +**Average latency**: 1.348s + +--- + +### installer (gpt-4.1) + +#### Basic Tests + +| Test | Result | Latency | Error | +|------|--------|---------|-------| +| Text Transform Uppercase | ✅ Pass | 0.925s | | +| Math Calculation | ✅ Pass | 0.655s | | +| Simple Math | ✅ Pass | 1.744s | | +| Count from 1 to 5 | ✅ Pass | 1.038s | | +| Streaming Count from 1 to 3 Streaming | ✅ Pass | 0.696s | | +| Basic Echo Function | ✅ Pass | 1.044s | | +| Streaming Simple Math Streaming | ✅ Pass | 1.113s | | +| Streaming Basic Echo Function Streaming | ✅ Pass | 1.136s | | +| Answer Stops At The Output Limit | ✅ Pass | 23.267s | | + +#### Advanced Tests + +| Test | Result | Latency | Error | +|------|--------|---------|-------| +| JSON Response Function | ✅ Pass | 1.044s | | +| Search Query Function | ✅ Pass | 1.193s | | +| Ask Advice Function | ✅ Pass | 0.833s | | +| Streaming Search Query Function Streaming | ✅ Pass | 0.803s | | +| Function Argument Memory Test | ✅ Pass | 0.993s | | +| Function Response Memory Test | ✅ Pass | 1.012s | | +| Basic Context Memory Test | ✅ Pass | 1.376s | | +| Penetration Testing Memory with Tool Call | ✅ Pass | 1.004s | | +| Cybersecurity Workflow Memory Test | ✅ Pass | 0.869s | | +| Read a file, then edit it via unified diff | ✅ Pass | 3.138s | | +| Penetration Testing Methodology | ✅ Pass | 1.111s | | +| SQL Injection Attack Type | ✅ Pass | 0.999s | | +| Vulnerability Assessment Tools | ✅ Pass | 1.218s | | +| Web Application Security Scanner | ✅ Pass | 0.703s | | +| Penetration Testing Framework | ✅ Pass | 0.987s | | +| Penetration Testing Tool Selection | ✅ Pass | 1.812s | | + +**Summary**: 25/25 (100.00%) successful tests + +**Average latency**: 2.029s + +--- + +### pentester (o4-mini) + +#### Basic Tests + +| Test | Result | Latency | Error | +|------|--------|---------|-------| +| Simple Math | ✅ Pass | 1.250s | | +| Text Transform Uppercase | ✅ Pass | 1.692s | | +| Math Calculation | ✅ Pass | 1.231s | | +| Count from 1 to 5 | ✅ Pass | 2.156s | | +| Basic Echo Function | ✅ Pass | 1.795s | | +| Streaming Simple Math Streaming | ✅ Pass | 1.449s | | +| Streaming Count from 1 to 3 Streaming | ✅ Pass | 1.366s | | +| Streaming Basic Echo Function Streaming | ✅ Pass | 1.493s | | +| Answer Stops At The Output Limit | ✅ Pass | 12.448s | | + +#### Advanced Tests + +| Test | Result | Latency | Error | +|------|--------|---------|-------| +| Streaming Search Query Function Streaming | ✅ Pass | 1.074s | | +| Search Query Function | ✅ Pass | 1.687s | | +| Function Response Memory Test | ✅ Pass | 1.267s | | +| Function Argument Memory Test | ✅ Pass | 1.482s | | +| Ask Advice Function | ✅ Pass | 2.506s | | +| Basic Context Memory Test | ✅ Pass | 2.473s | | +| Cybersecurity Workflow Memory Test | ✅ Pass | 1.883s | | +| JSON Response Function | ✅ Pass | 4.265s | | +| Penetration Testing Memory with Tool Call | ✅ Pass | 5.350s | | +| Read a file, then edit it via unified diff | ✅ Pass | 4.847s | | +| Penetration Testing Methodology | ✅ Pass | 1.986s | | +| SQL Injection Attack Type | ✅ Pass | 1.970s | | +| Vulnerability Assessment Tools | ✅ Pass | 2.253s | | +| Penetration Testing Framework | ✅ Pass | 2.191s | | +| Web Application Security Scanner | ✅ Pass | 2.192s | | +| Penetration Testing Tool Selection | ✅ Pass | 2.938s | | + +**Summary**: 25/25 (100.00%) successful tests + +**Average latency**: 2.610s + +--- + diff --git a/examples/tests/vllm-mixed.report.md b/examples/tests/vllm-mixed.report.md index eb1d65ba..45329c87 100644 --- a/examples/tests/vllm-mixed.report.md +++ b/examples/tests/vllm-mixed.report.md @@ -1,94 +1,96 @@ # LLM Agent Testing Report -Generated: Thu, 23 Jul 2026 11:03:48 UTC +Generated: Wed, 30 Sep 2026 15:55:05 UTC ## Overall Results | Agent | Model | Reasoning | Success Rate | Average Latency | |-------|-------|-----------|--------------|-----------------| -| simple | Qwen/Qwen3.6-27B-FP8 | false | 24/24 (100.00%) | 0.244s | -| simple_json | Qwen/Qwen3.6-27B-FP8 | false | 7/7 (100.00%) | 0.224s | -| primary_agent | DeepSeek-V4-Flash | true | 24/24 (100.00%) | 1.671s | -| assistant | DeepSeek-V4-Flash | true | 24/24 (100.00%) | 1.627s | -| generator | DeepSeek-V4-Flash | true | 24/24 (100.00%) | 1.526s | -| refiner | DeepSeek-V4-Flash | true | 24/24 (100.00%) | 1.381s | -| adviser | DeepSeek-V4-Flash | true | 24/24 (100.00%) | 1.255s | -| reflector | Qwen/Qwen3.6-27B-FP8 | true | 24/24 (100.00%) | 0.243s | -| searcher | Qwen/Qwen3.6-27B-FP8 | true | 24/24 (100.00%) | 0.238s | -| enricher | Qwen/Qwen3.6-27B-FP8 | true | 24/24 (100.00%) | 0.231s | -| coder | Qwen/Qwen3.6-27B-FP8 | true | 24/24 (100.00%) | 0.233s | -| installer | Qwen/Qwen3.6-27B-FP8 | true | 24/24 (100.00%) | 0.238s | -| pentester | Qwen/Qwen3.6-27B-FP8 | true | 24/24 (100.00%) | 0.239s | +| simple | Qwen/Qwen3.8-27B-FP8 | false | 25/25 (100.00%) | 2.660s | +| simple_json | Qwen/Qwen3.8-27B-FP8 | false | 8/8 (100.00%) | 1.269s | +| primary_agent | DeepSeek-V4-Flash | true | 25/25 (100.00%) | 0.926s | +| assistant | DeepSeek-V4-Flash | true | 25/25 (100.00%) | 1.070s | +| generator | DeepSeek-V4-Flash | true | 25/25 (100.00%) | 1.045s | +| refiner | DeepSeek-V4-Flash | true | 25/25 (100.00%) | 0.907s | +| adviser | DeepSeek-V4-Flash | true | 16/16 (100.00%) | 1.014s | +| reflector | Qwen/Qwen3.8-27B-FP8 | false | 15/16 (93.75%) | 2.427s | +| searcher | Qwen/Qwen3.8-27B-FP8 | false | 25/25 (100.00%) | 2.041s | +| enricher | Qwen/Qwen3.8-27B-FP8 | false | 25/25 (100.00%) | 2.134s | +| coder | Qwen/Qwen3.8-27B-FP8 | true | 25/25 (100.00%) | 2.966s | +| installer | Qwen/Qwen3.8-27B-FP8 | true | 25/25 (100.00%) | 2.979s | +| pentester | DeepSeek-V4-Flash | true | 25/25 (100.00%) | 0.807s | -**Total**: 295/295 (100.00%) successful tests -**Overall average latency**: 0.748s +**Total**: 289/290 (99.66%) successful tests +**Overall average latency**: 1.737s ## Detailed Results -### simple (Qwen/Qwen3.6-27B-FP8) +### simple (Qwen/Qwen3.8-27B-FP8) #### Basic Tests | Test | Result | Latency | Error | |------|--------|---------|-------| -| Simple Math | ✅ Pass | 0.265s | | -| Text Transform Uppercase | ✅ Pass | 0.229s | | -| Count from 1 to 5 | ✅ Pass | 0.207s | | -| Math Calculation | ✅ Pass | 0.221s | | -| Basic Echo Function | ✅ Pass | 0.251s | | -| Streaming Simple Math Streaming | ✅ Pass | 0.224s | | -| Streaming Count from 1 to 3 Streaming | ✅ Pass | 0.233s | | -| Streaming Basic Echo Function Streaming | ✅ Pass | 0.215s | | +| Simple Math | ✅ Pass | 0.877s | | +| Streaming Simple Math Streaming | ✅ Pass | 1.113s | | +| Math Calculation | ✅ Pass | 1.152s | | +| Count from 1 to 5 | ✅ Pass | 1.181s | | +| Streaming Count from 1 to 3 Streaming | ✅ Pass | 1.327s | | +| Basic Echo Function | ✅ Pass | 1.364s | | +| Text Transform Uppercase | ✅ Pass | 1.488s | | +| Streaming Basic Echo Function Streaming | ✅ Pass | 0.956s | | +| Answer Stops At The Output Limit | ✅ Pass | 25.684s | | #### Advanced Tests | Test | Result | Latency | Error | |------|--------|---------|-------| -| JSON Response Function | ✅ Pass | 0.218s | | -| Search Query Function | ✅ Pass | 0.224s | | -| Ask Advice Function | ✅ Pass | 0.308s | | -| Streaming Search Query Function Streaming | ✅ Pass | 0.234s | | -| Basic Context Memory Test | ✅ Pass | 0.207s | | -| Function Argument Memory Test | ✅ Pass | 0.295s | | -| Function Response Memory Test | ✅ Pass | 0.337s | | -| Penetration Testing Memory with Tool Call | ✅ Pass | 0.224s | | -| Cybersecurity Workflow Memory Test | ✅ Pass | 0.214s | | -| Read a file, then edit it via unified diff | ✅ Pass | 0.451s | | -| Penetration Testing Methodology | ✅ Pass | 0.222s | | -| Vulnerability Assessment Tools | ✅ Pass | 0.214s | | -| SQL Injection Attack Type | ✅ Pass | 0.220s | | -| Penetration Testing Framework | ✅ Pass | 0.206s | | -| Web Application Security Scanner | ✅ Pass | 0.216s | | -| Penetration Testing Tool Selection | ✅ Pass | 0.219s | | +| Search Query Function | ✅ Pass | 0.859s | | +| JSON Response Function | ✅ Pass | 1.128s | | +| Streaming Search Query Function Streaming | ✅ Pass | 1.460s | | +| Basic Context Memory Test | ✅ Pass | 0.855s | | +| Ask Advice Function | ✅ Pass | 1.806s | | +| Function Argument Memory Test | ✅ Pass | 0.970s | | +| Function Response Memory Test | ✅ Pass | 0.775s | | +| Cybersecurity Workflow Memory Test | ✅ Pass | 1.440s | | +| Penetration Testing Memory with Tool Call | ✅ Pass | 1.887s | | +| Read a file, then edit it via unified diff | ✅ Pass | 6.458s | | +| Penetration Testing Methodology | ✅ Pass | 2.078s | | +| SQL Injection Attack Type | ✅ Pass | 0.882s | | +| Penetration Testing Tool Selection | ✅ Pass | 1.574s | | +| Web Application Security Scanner | ✅ Pass | 2.437s | | +| Penetration Testing Framework | ✅ Pass | 2.941s | | +| Vulnerability Assessment Tools | ✅ Pass | 3.807s | | -**Summary**: 24/24 (100.00%) successful tests +**Summary**: 25/25 (100.00%) successful tests -**Average latency**: 0.244s +**Average latency**: 2.660s --- -### simple_json (Qwen/Qwen3.6-27B-FP8) +### simple_json (Qwen/Qwen3.8-27B-FP8) #### Advanced Tests | Test | Result | Latency | Error | |------|--------|---------|-------| -| Vulnerability Report Memory Test | ✅ Pass | 0.213s | | -| Person Information JSON | ✅ Pass | 0.208s | | -| Project Information JSON | ✅ Pass | 0.213s | | -| User Profile JSON | ✅ Pass | 0.223s | | -| JSON Array Response Without Schema | ✅ Pass | 0.217s | | -| Streaming Person Information JSON Streaming | ✅ Pass | 0.274s | | +| Vulnerability Report Memory Test | ✅ Pass | 2.105s | | +| Person Information JSON | ✅ Pass | 1.025s | | +| Project Information JSON | ✅ Pass | 1.282s | | +| User Profile JSON | ✅ Pass | 1.190s | | +| Streaming Person Information JSON Streaming | ✅ Pass | 1.392s | | +| JSON Array Response Without Schema | ✅ Pass | 2.205s | | #### Capability Tests | Test | Capability | Result | Latency | Note | |------|------------|--------|---------|------| -| Structured Output With JSON Schema | structured_output | ✅ Pass | 0.214s | | +| Structured Output Refuses A Non-Object Schema | structured_output | ✅ Pass | 0.000s | | +| Structured Output With JSON Schema | structured_output | ✅ Pass | 0.947s | | -**Summary**: 7/7 (100.00%) successful tests +**Summary**: 8/8 (100.00%) successful tests -**Average latency**: 0.224s +**Average latency**: 1.269s --- @@ -98,39 +100,40 @@ Generated: Thu, 23 Jul 2026 11:03:48 UTC | Test | Result | Latency | Error | |------|--------|---------|-------| -| Simple Math | ✅ Pass | 1.238s | | -| Text Transform Uppercase | ✅ Pass | 1.238s | | -| Count from 1 to 5 | ✅ Pass | 1.104s | | -| Math Calculation | ✅ Pass | 1.233s | | -| Basic Echo Function | ✅ Pass | 1.589s | | -| Streaming Simple Math Streaming | ✅ Pass | 0.987s | | -| Streaming Count from 1 to 3 Streaming | ✅ Pass | 1.189s | | -| Streaming Basic Echo Function Streaming | ✅ Pass | 1.393s | | +| Text Transform Uppercase | ✅ Pass | 0.340s | | +| Simple Math | ✅ Pass | 0.388s | | +| Math Calculation | ✅ Pass | 0.316s | | +| Count from 1 to 5 | ✅ Pass | 0.532s | | +| Streaming Simple Math Streaming | ✅ Pass | 0.421s | | +| Streaming Count from 1 to 3 Streaming | ✅ Pass | 0.450s | | +| Basic Echo Function | ✅ Pass | 0.526s | | +| Streaming Basic Echo Function Streaming | ✅ Pass | 0.562s | | +| Answer Stops At The Output Limit | ✅ Pass | 8.126s | | #### Advanced Tests | Test | Result | Latency | Error | |------|--------|---------|-------| -| JSON Response Function | ✅ Pass | 1.553s | | -| Search Query Function | ✅ Pass | 1.386s | | -| Ask Advice Function | ✅ Pass | 2.022s | | -| Streaming Search Query Function Streaming | ✅ Pass | 1.370s | | -| Basic Context Memory Test | ✅ Pass | 1.393s | | -| Function Argument Memory Test | ✅ Pass | 1.112s | | -| Function Response Memory Test | ✅ Pass | 1.150s | | -| Cybersecurity Workflow Memory Test | ✅ Pass | 1.412s | | -| Penetration Testing Memory with Tool Call | ✅ Pass | 2.437s | | -| Penetration Testing Methodology | ✅ Pass | 1.728s | | -| Read a file, then edit it via unified diff | ✅ Pass | 3.727s | | -| SQL Injection Attack Type | ✅ Pass | 1.461s | | -| Vulnerability Assessment Tools | ✅ Pass | 3.942s | | -| Penetration Testing Framework | ✅ Pass | 2.063s | | -| Web Application Security Scanner | ✅ Pass | 1.749s | | -| Penetration Testing Tool Selection | ✅ Pass | 1.612s | | +| JSON Response Function | ✅ Pass | 0.536s | | +| Search Query Function | ✅ Pass | 0.457s | | +| Ask Advice Function | ✅ Pass | 0.608s | | +| Streaming Search Query Function Streaming | ✅ Pass | 0.408s | | +| Function Response Memory Test | ✅ Pass | 0.740s | | +| Function Argument Memory Test | ✅ Pass | 1.000s | | +| Basic Context Memory Test | ✅ Pass | 1.449s | | +| Penetration Testing Memory with Tool Call | ✅ Pass | 0.902s | | +| Cybersecurity Workflow Memory Test | ✅ Pass | 0.577s | | +| Read a file, then edit it via unified diff | ✅ Pass | 2.069s | | +| Penetration Testing Methodology | ✅ Pass | 0.308s | | +| SQL Injection Attack Type | ✅ Pass | 0.324s | | +| Web Application Security Scanner | ✅ Pass | 0.308s | | +| Penetration Testing Framework | ✅ Pass | 0.372s | | +| Vulnerability Assessment Tools | ✅ Pass | 0.700s | | +| Penetration Testing Tool Selection | ✅ Pass | 0.717s | | -**Summary**: 24/24 (100.00%) successful tests +**Summary**: 25/25 (100.00%) successful tests -**Average latency**: 1.671s +**Average latency**: 0.926s --- @@ -140,39 +143,40 @@ Generated: Thu, 23 Jul 2026 11:03:48 UTC | Test | Result | Latency | Error | |------|--------|---------|-------| -| Simple Math | ✅ Pass | 0.920s | | -| Text Transform Uppercase | ✅ Pass | 1.028s | | -| Math Calculation | ✅ Pass | 1.201s | | -| Count from 1 to 5 | ✅ Pass | 1.935s | | -| Basic Echo Function | ✅ Pass | 1.375s | | -| Streaming Simple Math Streaming | ✅ Pass | 1.001s | | -| Streaming Count from 1 to 3 Streaming | ✅ Pass | 1.119s | | -| Streaming Basic Echo Function Streaming | ✅ Pass | 1.356s | | +| Simple Math | ✅ Pass | 0.417s | | +| Text Transform Uppercase | ✅ Pass | 0.444s | | +| Math Calculation | ✅ Pass | 0.505s | | +| Streaming Simple Math Streaming | ✅ Pass | 0.400s | | +| Streaming Count from 1 to 3 Streaming | ✅ Pass | 0.409s | | +| Basic Echo Function | ✅ Pass | 0.582s | | +| Count from 1 to 5 | ✅ Pass | 0.865s | | +| Streaming Basic Echo Function Streaming | ✅ Pass | 0.570s | | +| Answer Stops At The Output Limit | ✅ Pass | 8.308s | | #### Advanced Tests | Test | Result | Latency | Error | |------|--------|---------|-------| -| JSON Response Function | ✅ Pass | 1.582s | | -| Search Query Function | ✅ Pass | 1.358s | | -| Ask Advice Function | ✅ Pass | 1.966s | | -| Streaming Search Query Function Streaming | ✅ Pass | 1.351s | | -| Basic Context Memory Test | ✅ Pass | 1.114s | | -| Function Argument Memory Test | ✅ Pass | 1.052s | | -| Function Response Memory Test | ✅ Pass | 1.238s | | -| Cybersecurity Workflow Memory Test | ✅ Pass | 1.392s | | -| Penetration Testing Memory with Tool Call | ✅ Pass | 2.506s | | -| Penetration Testing Methodology | ✅ Pass | 1.316s | | -| Read a file, then edit it via unified diff | ✅ Pass | 3.774s | | -| SQL Injection Attack Type | ✅ Pass | 1.488s | | -| Vulnerability Assessment Tools | ✅ Pass | 2.653s | | -| Web Application Security Scanner | ✅ Pass | 1.626s | | -| Penetration Testing Framework | ✅ Pass | 3.101s | | -| Penetration Testing Tool Selection | ✅ Pass | 1.589s | | +| JSON Response Function | ✅ Pass | 0.590s | | +| Search Query Function | ✅ Pass | 0.585s | | +| Ask Advice Function | ✅ Pass | 0.664s | | +| Streaming Search Query Function Streaming | ✅ Pass | 0.647s | | +| Function Argument Memory Test | ✅ Pass | 0.627s | | +| Function Response Memory Test | ✅ Pass | 0.927s | | +| Basic Context Memory Test | ✅ Pass | 1.178s | | +| Cybersecurity Workflow Memory Test | ✅ Pass | 0.586s | | +| Penetration Testing Memory with Tool Call | ✅ Pass | 1.270s | | +| Read a file, then edit it via unified diff | ✅ Pass | 1.836s | | +| SQL Injection Attack Type | ✅ Pass | 0.618s | | +| Penetration Testing Methodology | ✅ Pass | 0.795s | | +| Penetration Testing Framework | ✅ Pass | 0.401s | | +| Web Application Security Scanner | ✅ Pass | 0.417s | | +| Penetration Testing Tool Selection | ✅ Pass | 0.734s | | +| Vulnerability Assessment Tools | ✅ Pass | 2.371s | | -**Summary**: 24/24 (100.00%) successful tests +**Summary**: 25/25 (100.00%) successful tests -**Average latency**: 1.627s +**Average latency**: 1.070s --- @@ -182,39 +186,40 @@ Generated: Thu, 23 Jul 2026 11:03:48 UTC | Test | Result | Latency | Error | |------|--------|---------|-------| -| Simple Math | ✅ Pass | 0.828s | | -| Text Transform Uppercase | ✅ Pass | 1.088s | | -| Count from 1 to 5 | ✅ Pass | 1.407s | | -| Math Calculation | ✅ Pass | 0.976s | | -| Basic Echo Function | ✅ Pass | 1.545s | | -| Streaming Simple Math Streaming | ✅ Pass | 0.875s | | -| Streaming Count from 1 to 3 Streaming | ✅ Pass | 1.162s | | -| Streaming Basic Echo Function Streaming | ✅ Pass | 1.274s | | +| Text Transform Uppercase | ✅ Pass | 0.368s | | +| Simple Math | ✅ Pass | 0.459s | | +| Count from 1 to 5 | ✅ Pass | 0.469s | | +| Math Calculation | ✅ Pass | 0.443s | | +| Streaming Simple Math Streaming | ✅ Pass | 0.383s | | +| Basic Echo Function | ✅ Pass | 0.619s | | +| Streaming Count from 1 to 3 Streaming | ✅ Pass | 0.510s | | +| Streaming Basic Echo Function Streaming | ✅ Pass | 0.618s | | +| Answer Stops At The Output Limit | ✅ Pass | 9.096s | | #### Advanced Tests | Test | Result | Latency | Error | |------|--------|---------|-------| -| JSON Response Function | ✅ Pass | 1.400s | | -| Search Query Function | ✅ Pass | 1.342s | | -| Ask Advice Function | ✅ Pass | 1.860s | | -| Streaming Search Query Function Streaming | ✅ Pass | 1.190s | | -| Basic Context Memory Test | ✅ Pass | 1.152s | | -| Function Argument Memory Test | ✅ Pass | 0.959s | | -| Function Response Memory Test | ✅ Pass | 0.881s | | -| Cybersecurity Workflow Memory Test | ✅ Pass | 1.242s | | -| Penetration Testing Memory with Tool Call | ✅ Pass | 2.492s | | -| Penetration Testing Methodology | ✅ Pass | 1.616s | | -| Read a file, then edit it via unified diff | ✅ Pass | 3.804s | | -| SQL Injection Attack Type | ✅ Pass | 0.966s | | -| Vulnerability Assessment Tools | ✅ Pass | 3.905s | | -| Penetration Testing Framework | ✅ Pass | 1.841s | | -| Web Application Security Scanner | ✅ Pass | 1.273s | | -| Penetration Testing Tool Selection | ✅ Pass | 1.538s | | +| Search Query Function | ✅ Pass | 0.600s | | +| JSON Response Function | ✅ Pass | 0.667s | | +| Ask Advice Function | ✅ Pass | 0.645s | | +| Streaming Search Query Function Streaming | ✅ Pass | 0.625s | | +| Function Argument Memory Test | ✅ Pass | 0.654s | | +| Function Response Memory Test | ✅ Pass | 0.623s | | +| Cybersecurity Workflow Memory Test | ✅ Pass | 0.603s | | +| Basic Context Memory Test | ✅ Pass | 1.508s | | +| Penetration Testing Memory with Tool Call | ✅ Pass | 1.606s | | +| Read a file, then edit it via unified diff | ✅ Pass | 2.044s | | +| Penetration Testing Methodology | ✅ Pass | 0.407s | | +| SQL Injection Attack Type | ✅ Pass | 0.531s | | +| Penetration Testing Framework | ✅ Pass | 0.385s | | +| Vulnerability Assessment Tools | ✅ Pass | 0.855s | | +| Web Application Security Scanner | ✅ Pass | 0.320s | | +| Penetration Testing Tool Selection | ✅ Pass | 1.077s | | -**Summary**: 24/24 (100.00%) successful tests +**Summary**: 25/25 (100.00%) successful tests -**Average latency**: 1.526s +**Average latency**: 1.045s --- @@ -224,39 +229,40 @@ Generated: Thu, 23 Jul 2026 11:03:48 UTC | Test | Result | Latency | Error | |------|--------|---------|-------| -| Simple Math | ✅ Pass | 1.224s | | -| Text Transform Uppercase | ✅ Pass | 0.901s | | -| Count from 1 to 5 | ✅ Pass | 1.372s | | -| Math Calculation | ✅ Pass | 0.874s | | -| Basic Echo Function | ✅ Pass | 1.513s | | -| Streaming Simple Math Streaming | ✅ Pass | 0.725s | | -| Streaming Count from 1 to 3 Streaming | ✅ Pass | 1.123s | | -| Streaming Basic Echo Function Streaming | ✅ Pass | 1.377s | | +| Simple Math | ✅ Pass | 0.338s | | +| Text Transform Uppercase | ✅ Pass | 0.463s | | +| Math Calculation | ✅ Pass | 0.508s | | +| Count from 1 to 5 | ✅ Pass | 0.568s | | +| Basic Echo Function | ✅ Pass | 0.579s | | +| Streaming Simple Math Streaming | ✅ Pass | 0.379s | | +| Streaming Count from 1 to 3 Streaming | ✅ Pass | 0.431s | | +| Streaming Basic Echo Function Streaming | ✅ Pass | 0.510s | | +| Answer Stops At The Output Limit | ✅ Pass | 6.741s | | #### Advanced Tests | Test | Result | Latency | Error | |------|--------|---------|-------| -| JSON Response Function | ✅ Pass | 1.472s | | -| Search Query Function | ✅ Pass | 1.154s | | -| Ask Advice Function | ✅ Pass | 1.727s | | -| Streaming Search Query Function Streaming | ✅ Pass | 1.455s | | -| Basic Context Memory Test | ✅ Pass | 1.065s | | -| Function Argument Memory Test | ✅ Pass | 0.940s | | -| Function Response Memory Test | ✅ Pass | 0.840s | | -| Cybersecurity Workflow Memory Test | ✅ Pass | 0.239s | | -| Penetration Testing Memory with Tool Call | ✅ Pass | 2.333s | | -| Read a file, then edit it via unified diff | ✅ Pass | 3.754s | | -| Penetration Testing Methodology | ✅ Pass | 1.504s | | -| SQL Injection Attack Type | ✅ Pass | 0.221s | | -| Vulnerability Assessment Tools | ✅ Pass | 2.540s | | -| Penetration Testing Framework | ✅ Pass | 2.092s | | -| Web Application Security Scanner | ✅ Pass | 1.218s | | -| Penetration Testing Tool Selection | ✅ Pass | 1.460s | | +| Search Query Function | ✅ Pass | 0.617s | | +| JSON Response Function | ✅ Pass | 0.721s | | +| Ask Advice Function | ✅ Pass | 0.626s | | +| Streaming Search Query Function Streaming | ✅ Pass | 0.597s | | +| Basic Context Memory Test | ✅ Pass | 0.516s | | +| Function Response Memory Test | ✅ Pass | 0.640s | | +| Function Argument Memory Test | ✅ Pass | 0.680s | | +| Cybersecurity Workflow Memory Test | ✅ Pass | 0.529s | | +| Penetration Testing Memory with Tool Call | ✅ Pass | 1.261s | | +| Read a file, then edit it via unified diff | ✅ Pass | 1.732s | | +| Penetration Testing Methodology | ✅ Pass | 0.511s | | +| Penetration Testing Framework | ✅ Pass | 0.415s | | +| Vulnerability Assessment Tools | ✅ Pass | 0.752s | | +| Web Application Security Scanner | ✅ Pass | 0.549s | | +| SQL Injection Attack Type | ✅ Pass | 1.130s | | +| Penetration Testing Tool Selection | ✅ Pass | 0.865s | | -**Summary**: 24/24 (100.00%) successful tests +**Summary**: 25/25 (100.00%) successful tests -**Average latency**: 1.381s +**Average latency**: 0.907s --- @@ -266,291 +272,280 @@ Generated: Thu, 23 Jul 2026 11:03:48 UTC | Test | Result | Latency | Error | |------|--------|---------|-------| -| Simple Math | ✅ Pass | 0.874s | | -| Text Transform Uppercase | ✅ Pass | 0.964s | | -| Count from 1 to 5 | ✅ Pass | 1.183s | | -| Math Calculation | ✅ Pass | 0.690s | | -| Basic Echo Function | ✅ Pass | 1.489s | | -| Streaming Simple Math Streaming | ✅ Pass | 0.249s | | -| Streaming Count from 1 to 3 Streaming | ✅ Pass | 1.094s | | -| Streaming Basic Echo Function Streaming | ✅ Pass | 1.640s | | +| Simple Math | ✅ Pass | 0.363s | | +| Text Transform Uppercase | ✅ Pass | 0.430s | | +| Count from 1 to 5 | ✅ Pass | 0.593s | | +| Math Calculation | ✅ Pass | 0.345s | | +| Streaming Simple Math Streaming | ✅ Pass | 0.386s | | +| Streaming Count from 1 to 3 Streaming | ✅ Pass | 0.403s | | +| Answer Stops At The Output Limit | ✅ Pass | 7.857s | | #### Advanced Tests | Test | Result | Latency | Error | |------|--------|---------|-------| -| JSON Response Function | ✅ Pass | 1.782s | | -| Search Query Function | ✅ Pass | 1.120s | | -| Ask Advice Function | ✅ Pass | 1.737s | | -| Streaming Search Query Function Streaming | ✅ Pass | 0.208s | | -| Basic Context Memory Test | ✅ Pass | 1.212s | | -| Function Argument Memory Test | ✅ Pass | 0.972s | | -| Function Response Memory Test | ✅ Pass | 0.770s | | -| Cybersecurity Workflow Memory Test | ✅ Pass | 0.241s | | -| Penetration Testing Memory with Tool Call | ✅ Pass | 2.306s | | -| Penetration Testing Methodology | ✅ Pass | 0.221s | | -| Read a file, then edit it via unified diff | ✅ Pass | 3.772s | | -| SQL Injection Attack Type | ✅ Pass | 0.212s | | -| Vulnerability Assessment Tools | ✅ Pass | 2.754s | | -| Penetration Testing Framework | ✅ Pass | 2.173s | | -| Web Application Security Scanner | ✅ Pass | 1.074s | | -| Penetration Testing Tool Selection | ✅ Pass | 1.367s | | +| Cybersecurity Workflow Memory Test | ✅ Pass | 0.502s | | +| Function Response Memory Test | ✅ Pass | 0.529s | | +| Function Argument Memory Test | ✅ Pass | 0.606s | | +| Basic Context Memory Test | ✅ Pass | 1.222s | | +| Penetration Testing Framework | ✅ Pass | 0.341s | | +| Penetration Testing Methodology | ✅ Pass | 0.764s | | +| SQL Injection Attack Type | ✅ Pass | 0.642s | | +| Vulnerability Assessment Tools | ✅ Pass | 0.806s | | +| Web Application Security Scanner | ✅ Pass | 0.426s | | -**Summary**: 24/24 (100.00%) successful tests +**Summary**: 16/16 (100.00%) successful tests -**Average latency**: 1.255s +**Average latency**: 1.014s --- -### reflector (Qwen/Qwen3.6-27B-FP8) +### reflector (Qwen/Qwen3.8-27B-FP8) #### Basic Tests | Test | Result | Latency | Error | |------|--------|---------|-------| -| Simple Math | ✅ Pass | 0.257s | | -| Text Transform Uppercase | ✅ Pass | 0.211s | | -| Count from 1 to 5 | ✅ Pass | 0.331s | | -| Math Calculation | ✅ Pass | 0.231s | | -| Basic Echo Function | ✅ Pass | 0.219s | | -| Streaming Simple Math Streaming | ✅ Pass | 0.250s | | -| Streaming Count from 1 to 3 Streaming | ✅ Pass | 0.219s | | -| Streaming Basic Echo Function Streaming | ✅ Pass | 0.279s | | +| Simple Math | ✅ Pass | 1.123s | | +| Text Transform Uppercase | ✅ Pass | 0.812s | | +| Math Calculation | ✅ Pass | 0.630s | | +| Count from 1 to 5 | ✅ Pass | 1.419s | | +| Streaming Simple Math Streaming | ✅ Pass | 0.832s | | +| Streaming Count from 1 to 3 Streaming | ✅ Pass | 0.948s | | +| Answer Stops At The Output Limit | ✅ Pass | 20.378s | | #### Advanced Tests | Test | Result | Latency | Error | |------|--------|---------|-------| -| JSON Response Function | ✅ Pass | 0.291s | | -| Search Query Function | ✅ Pass | 0.216s | | -| Ask Advice Function | ✅ Pass | 0.212s | | -| Streaming Search Query Function Streaming | ✅ Pass | 0.208s | | -| Basic Context Memory Test | ✅ Pass | 0.215s | | -| Function Argument Memory Test | ✅ Pass | 0.207s | | -| Function Response Memory Test | ✅ Pass | 0.224s | | -| Penetration Testing Memory with Tool Call | ✅ Pass | 0.228s | | -| Cybersecurity Workflow Memory Test | ✅ Pass | 0.219s | | -| Read a file, then edit it via unified diff | ✅ Pass | 0.468s | | -| Penetration Testing Methodology | ✅ Pass | 0.218s | | -| Vulnerability Assessment Tools | ✅ Pass | 0.204s | | -| SQL Injection Attack Type | ✅ Pass | 0.209s | | -| Penetration Testing Framework | ✅ Pass | 0.224s | | -| Web Application Security Scanner | ✅ Pass | 0.214s | | -| Penetration Testing Tool Selection | ✅ Pass | 0.267s | | +| Basic Context Memory Test | ✅ Pass | 0.615s | | +| Function Response Memory Test | ✅ Pass | 0.571s | | +| Cybersecurity Workflow Memory Test | ✅ Pass | 0.600s | | +| Function Argument Memory Test | ❌ Fail | 1.188s | expected text 'Go programming language' not found; stop reason: stop | +| SQL Injection Attack Type | ✅ Pass | 0.700s | | +| Penetration Testing Framework | ✅ Pass | 1.591s | | +| Web Application Security Scanner | ✅ Pass | 1.752s | | +| Vulnerability Assessment Tools | ✅ Pass | 2.773s | | +| Penetration Testing Methodology | ✅ Pass | 2.897s | | -**Summary**: 24/24 (100.00%) successful tests +**Summary**: 15/16 (93.75%) successful tests -**Average latency**: 0.243s +**Average latency**: 2.427s --- -### searcher (Qwen/Qwen3.6-27B-FP8) +### searcher (Qwen/Qwen3.8-27B-FP8) #### Basic Tests | Test | Result | Latency | Error | |------|--------|---------|-------| -| Simple Math | ✅ Pass | 0.262s | | -| Text Transform Uppercase | ✅ Pass | 0.216s | | -| Count from 1 to 5 | ✅ Pass | 0.289s | | -| Math Calculation | ✅ Pass | 0.217s | | -| Basic Echo Function | ✅ Pass | 0.214s | | -| Streaming Simple Math Streaming | ✅ Pass | 0.223s | | -| Streaming Count from 1 to 3 Streaming | ✅ Pass | 0.229s | | -| Streaming Basic Echo Function Streaming | ✅ Pass | 0.287s | | +| Simple Math | ✅ Pass | 0.877s | | +| Text Transform Uppercase | ✅ Pass | 0.752s | | +| Count from 1 to 5 | ✅ Pass | 0.747s | | +| Math Calculation | ✅ Pass | 0.637s | | +| Streaming Simple Math Streaming | ✅ Pass | 0.843s | | +| Streaming Count from 1 to 3 Streaming | ✅ Pass | 1.230s | | +| Basic Echo Function | ✅ Pass | 1.550s | | +| Streaming Basic Echo Function Streaming | ✅ Pass | 1.175s | | +| Answer Stops At The Output Limit | ✅ Pass | 17.939s | | #### Advanced Tests | Test | Result | Latency | Error | |------|--------|---------|-------| -| JSON Response Function | ✅ Pass | 0.224s | | -| Search Query Function | ✅ Pass | 0.237s | | -| Ask Advice Function | ✅ Pass | 0.232s | | -| Streaming Search Query Function Streaming | ✅ Pass | 0.230s | | -| Basic Context Memory Test | ✅ Pass | 0.213s | | -| Function Argument Memory Test | ✅ Pass | 0.210s | | -| Function Response Memory Test | ✅ Pass | 0.217s | | -| Penetration Testing Memory with Tool Call | ✅ Pass | 0.225s | | -| Cybersecurity Workflow Memory Test | ✅ Pass | 0.224s | | -| Read a file, then edit it via unified diff | ✅ Pass | 0.436s | | -| Penetration Testing Methodology | ✅ Pass | 0.218s | | -| Vulnerability Assessment Tools | ✅ Pass | 0.207s | | -| SQL Injection Attack Type | ✅ Pass | 0.212s | | -| Penetration Testing Framework | ✅ Pass | 0.215s | | -| Web Application Security Scanner | ✅ Pass | 0.230s | | -| Penetration Testing Tool Selection | ✅ Pass | 0.244s | | +| JSON Response Function | ✅ Pass | 1.071s | | +| Search Query Function | ✅ Pass | 0.892s | | +| Function Argument Memory Test | ✅ Pass | 0.721s | | +| Basic Context Memory Test | ✅ Pass | 0.830s | | +| Streaming Search Query Function Streaming | ✅ Pass | 0.871s | | +| Ask Advice Function | ✅ Pass | 1.197s | | +| Function Response Memory Test | ✅ Pass | 0.940s | | +| Cybersecurity Workflow Memory Test | ✅ Pass | 0.668s | | +| Penetration Testing Memory with Tool Call | ✅ Pass | 1.880s | | +| Read a file, then edit it via unified diff | ✅ Pass | 5.749s | | +| SQL Injection Attack Type | ✅ Pass | 0.675s | | +| Penetration Testing Methodology | ✅ Pass | 1.930s | | +| Penetration Testing Framework | ✅ Pass | 1.777s | | +| Web Application Security Scanner | ✅ Pass | 1.710s | | +| Penetration Testing Tool Selection | ✅ Pass | 1.133s | | +| Vulnerability Assessment Tools | ✅ Pass | 3.226s | | -**Summary**: 24/24 (100.00%) successful tests +**Summary**: 25/25 (100.00%) successful tests -**Average latency**: 0.238s +**Average latency**: 2.041s --- -### enricher (Qwen/Qwen3.6-27B-FP8) +### enricher (Qwen/Qwen3.8-27B-FP8) #### Basic Tests | Test | Result | Latency | Error | |------|--------|---------|-------| -| Simple Math | ✅ Pass | 0.217s | | -| Text Transform Uppercase | ✅ Pass | 0.212s | | -| Count from 1 to 5 | ✅ Pass | 0.248s | | -| Math Calculation | ✅ Pass | 0.249s | | -| Basic Echo Function | ✅ Pass | 0.211s | | -| Streaming Simple Math Streaming | ✅ Pass | 0.240s | | -| Streaming Count from 1 to 3 Streaming | ✅ Pass | 0.225s | | -| Streaming Basic Echo Function Streaming | ✅ Pass | 0.280s | | +| Simple Math | ✅ Pass | 0.616s | | +| Count from 1 to 5 | ✅ Pass | 0.682s | | +| Text Transform Uppercase | ✅ Pass | 0.725s | | +| Math Calculation | ✅ Pass | 0.694s | | +| Streaming Simple Math Streaming | ✅ Pass | 0.605s | | +| Basic Echo Function | ✅ Pass | 1.007s | | +| Streaming Count from 1 to 3 Streaming | ✅ Pass | 1.401s | | +| Streaming Basic Echo Function Streaming | ✅ Pass | 1.608s | | +| Answer Stops At The Output Limit | ✅ Pass | 18.074s | | #### Advanced Tests | Test | Result | Latency | Error | |------|--------|---------|-------| -| JSON Response Function | ✅ Pass | 0.250s | | -| Search Query Function | ✅ Pass | 0.209s | | -| Ask Advice Function | ✅ Pass | 0.214s | | -| Streaming Search Query Function Streaming | ✅ Pass | 0.215s | | -| Basic Context Memory Test | ✅ Pass | 0.206s | | -| Function Argument Memory Test | ✅ Pass | 0.205s | | -| Function Response Memory Test | ✅ Pass | 0.204s | | -| Penetration Testing Memory with Tool Call | ✅ Pass | 0.210s | | -| Cybersecurity Workflow Memory Test | ✅ Pass | 0.212s | | -| Read a file, then edit it via unified diff | ✅ Pass | 0.429s | | -| Penetration Testing Methodology | ✅ Pass | 0.209s | | -| Vulnerability Assessment Tools | ✅ Pass | 0.207s | | -| SQL Injection Attack Type | ✅ Pass | 0.209s | | -| Penetration Testing Framework | ✅ Pass | 0.218s | | -| Web Application Security Scanner | ✅ Pass | 0.216s | | -| Penetration Testing Tool Selection | ✅ Pass | 0.242s | | +| Function Argument Memory Test | ✅ Pass | 0.623s | | +| JSON Response Function | ✅ Pass | 1.110s | | +| Basic Context Memory Test | ✅ Pass | 0.649s | | +| Streaming Search Query Function Streaming | ✅ Pass | 1.039s | | +| Search Query Function | ✅ Pass | 1.151s | | +| Ask Advice Function | ✅ Pass | 1.367s | | +| Function Response Memory Test | ✅ Pass | 0.554s | | +| Cybersecurity Workflow Memory Test | ✅ Pass | 1.726s | | +| Penetration Testing Memory with Tool Call | ✅ Pass | 3.037s | | +| Read a file, then edit it via unified diff | ✅ Pass | 4.871s | | +| SQL Injection Attack Type | ✅ Pass | 0.927s | | +| Penetration Testing Methodology | ✅ Pass | 2.076s | | +| Penetration Testing Tool Selection | ✅ Pass | 1.288s | | +| Web Application Security Scanner | ✅ Pass | 1.457s | | +| Penetration Testing Framework | ✅ Pass | 1.922s | | +| Vulnerability Assessment Tools | ✅ Pass | 4.124s | | -**Summary**: 24/24 (100.00%) successful tests +**Summary**: 25/25 (100.00%) successful tests -**Average latency**: 0.231s +**Average latency**: 2.134s --- -### coder (Qwen/Qwen3.6-27B-FP8) +### coder (Qwen/Qwen3.8-27B-FP8) #### Basic Tests | Test | Result | Latency | Error | |------|--------|---------|-------| -| Simple Math | ✅ Pass | 0.215s | | -| Text Transform Uppercase | ✅ Pass | 0.210s | | -| Count from 1 to 5 | ✅ Pass | 0.229s | | -| Math Calculation | ✅ Pass | 0.241s | | -| Basic Echo Function | ✅ Pass | 0.219s | | -| Streaming Simple Math Streaming | ✅ Pass | 0.243s | | -| Streaming Count from 1 to 3 Streaming | ✅ Pass | 0.212s | | -| Streaming Basic Echo Function Streaming | ✅ Pass | 0.265s | | +| Simple Math | ✅ Pass | 0.977s | | +| Text Transform Uppercase | ✅ Pass | 1.107s | | +| Math Calculation | ✅ Pass | 1.026s | | +| Count from 1 to 5 | ✅ Pass | 1.237s | | +| Basic Echo Function | ✅ Pass | 1.649s | | +| Streaming Simple Math Streaming | ✅ Pass | 1.559s | | +| Streaming Count from 1 to 3 Streaming | ✅ Pass | 1.477s | | +| Streaming Basic Echo Function Streaming | ✅ Pass | 1.346s | | +| Answer Stops At The Output Limit | ✅ Pass | 26.162s | | #### Advanced Tests | Test | Result | Latency | Error | |------|--------|---------|-------| -| JSON Response Function | ✅ Pass | 0.225s | | -| Search Query Function | ✅ Pass | 0.219s | | -| Ask Advice Function | ✅ Pass | 0.211s | | -| Streaming Search Query Function Streaming | ✅ Pass | 0.209s | | -| Basic Context Memory Test | ✅ Pass | 0.211s | | -| Function Argument Memory Test | ✅ Pass | 0.210s | | -| Function Response Memory Test | ✅ Pass | 0.210s | | -| Penetration Testing Memory with Tool Call | ✅ Pass | 0.267s | | -| Cybersecurity Workflow Memory Test | ✅ Pass | 0.226s | | -| Read a file, then edit it via unified diff | ✅ Pass | 0.430s | | -| Penetration Testing Methodology | ✅ Pass | 0.212s | | -| Vulnerability Assessment Tools | ✅ Pass | 0.214s | | -| SQL Injection Attack Type | ✅ Pass | 0.213s | | -| Penetration Testing Framework | ✅ Pass | 0.256s | | -| Web Application Security Scanner | ✅ Pass | 0.217s | | -| Penetration Testing Tool Selection | ✅ Pass | 0.222s | | +| Streaming Search Query Function Streaming | ✅ Pass | 1.766s | | +| Search Query Function | ✅ Pass | 2.263s | | +| Basic Context Memory Test | ✅ Pass | 2.019s | | +| JSON Response Function | ✅ Pass | 2.585s | | +| Ask Advice Function | ✅ Pass | 2.462s | | +| Function Argument Memory Test | ✅ Pass | 1.874s | | +| Function Response Memory Test | ✅ Pass | 1.283s | | +| Cybersecurity Workflow Memory Test | ✅ Pass | 2.481s | | +| Penetration Testing Memory with Tool Call | ✅ Pass | 4.669s | | +| Read a file, then edit it via unified diff | ✅ Pass | 4.858s | | +| Penetration Testing Methodology | ✅ Pass | 1.243s | | +| SQL Injection Attack Type | ✅ Pass | 1.810s | | +| Web Application Security Scanner | ✅ Pass | 1.516s | | +| Vulnerability Assessment Tools | ✅ Pass | 2.377s | | +| Penetration Testing Tool Selection | ✅ Pass | 1.995s | | +| Penetration Testing Framework | ✅ Pass | 2.401s | | -**Summary**: 24/24 (100.00%) successful tests +**Summary**: 25/25 (100.00%) successful tests -**Average latency**: 0.233s +**Average latency**: 2.966s --- -### installer (Qwen/Qwen3.6-27B-FP8) +### installer (Qwen/Qwen3.8-27B-FP8) #### Basic Tests | Test | Result | Latency | Error | |------|--------|---------|-------| -| Simple Math | ✅ Pass | 0.225s | | -| Text Transform Uppercase | ✅ Pass | 0.212s | | -| Count from 1 to 5 | ✅ Pass | 0.232s | | -| Math Calculation | ✅ Pass | 0.215s | | -| Basic Echo Function | ✅ Pass | 0.207s | | -| Streaming Simple Math Streaming | ✅ Pass | 0.217s | | -| Streaming Count from 1 to 3 Streaming | ✅ Pass | 0.230s | | -| Streaming Basic Echo Function Streaming | ✅ Pass | 0.269s | | +| Simple Math | ✅ Pass | 1.335s | | +| Text Transform Uppercase | ✅ Pass | 3.156s | | +| Count from 1 to 5 | ✅ Pass | 4.504s | | +| Math Calculation | ✅ Pass | 4.474s | | +| Streaming Simple Math Streaming | ✅ Pass | 1.184s | | +| Basic Echo Function | ✅ Pass | 2.944s | | +| Streaming Basic Echo Function Streaming | ✅ Pass | 1.145s | | +| Streaming Count from 1 to 3 Streaming | ✅ Pass | 1.793s | | +| Answer Stops At The Output Limit | ✅ Pass | 18.439s | | #### Advanced Tests | Test | Result | Latency | Error | |------|--------|---------|-------| -| JSON Response Function | ✅ Pass | 0.222s | | -| Search Query Function | ✅ Pass | 0.208s | | -| Ask Advice Function | ✅ Pass | 0.204s | | -| Streaming Search Query Function Streaming | ✅ Pass | 0.235s | | -| Basic Context Memory Test | ✅ Pass | 0.429s | | -| Function Argument Memory Test | ✅ Pass | 0.215s | | -| Function Response Memory Test | ✅ Pass | 0.207s | | -| Penetration Testing Memory with Tool Call | ✅ Pass | 0.217s | | -| Cybersecurity Workflow Memory Test | ✅ Pass | 0.226s | | -| Read a file, then edit it via unified diff | ✅ Pass | 0.423s | | -| Penetration Testing Methodology | ✅ Pass | 0.217s | | -| Vulnerability Assessment Tools | ✅ Pass | 0.212s | | -| SQL Injection Attack Type | ✅ Pass | 0.214s | | -| Penetration Testing Framework | ✅ Pass | 0.217s | | -| Web Application Security Scanner | ✅ Pass | 0.222s | | -| Penetration Testing Tool Selection | ✅ Pass | 0.234s | | +| Streaming Search Query Function Streaming | ✅ Pass | 1.158s | | +| Search Query Function | ✅ Pass | 1.575s | | +| JSON Response Function | ✅ Pass | 1.767s | | +| Ask Advice Function | ✅ Pass | 1.812s | | +| Function Response Memory Test | ✅ Pass | 1.251s | | +| Basic Context Memory Test | ✅ Pass | 2.580s | | +| Cybersecurity Workflow Memory Test | ✅ Pass | 1.543s | | +| Penetration Testing Memory with Tool Call | ✅ Pass | 3.639s | | +| Function Argument Memory Test | ✅ Pass | 4.864s | | +| Read a file, then edit it via unified diff | ✅ Pass | 4.589s | | +| Penetration Testing Methodology | ✅ Pass | 1.344s | | +| SQL Injection Attack Type | ✅ Pass | 1.632s | | +| Web Application Security Scanner | ✅ Pass | 1.454s | | +| Penetration Testing Framework | ✅ Pass | 1.655s | | +| Vulnerability Assessment Tools | ✅ Pass | 2.692s | | +| Penetration Testing Tool Selection | ✅ Pass | 1.941s | | -**Summary**: 24/24 (100.00%) successful tests +**Summary**: 25/25 (100.00%) successful tests -**Average latency**: 0.238s +**Average latency**: 2.979s --- -### pentester (Qwen/Qwen3.6-27B-FP8) +### pentester (DeepSeek-V4-Flash) #### Basic Tests | Test | Result | Latency | Error | |------|--------|---------|-------| -| Simple Math | ✅ Pass | 0.212s | | -| Text Transform Uppercase | ✅ Pass | 0.207s | | -| Count from 1 to 5 | ✅ Pass | 0.218s | | -| Math Calculation | ✅ Pass | 0.235s | | -| Basic Echo Function | ✅ Pass | 0.217s | | -| Streaming Simple Math Streaming | ✅ Pass | 0.220s | | -| Streaming Count from 1 to 3 Streaming | ✅ Pass | 0.208s | | -| Streaming Basic Echo Function Streaming | ✅ Pass | 0.268s | | +| Simple Math | ✅ Pass | 0.336s | | +| Text Transform Uppercase | ✅ Pass | 0.324s | | +| Count from 1 to 5 | ✅ Pass | 0.401s | | +| Math Calculation | ✅ Pass | 0.304s | | +| Streaming Simple Math Streaming | ✅ Pass | 0.325s | | +| Basic Echo Function | ✅ Pass | 0.393s | | +| Streaming Count from 1 to 3 Streaming | ✅ Pass | 0.418s | | +| Streaming Basic Echo Function Streaming | ✅ Pass | 0.412s | | +| Answer Stops At The Output Limit | ✅ Pass | 6.242s | | #### Advanced Tests | Test | Result | Latency | Error | |------|--------|---------|-------| -| JSON Response Function | ✅ Pass | 0.214s | | -| Search Query Function | ✅ Pass | 0.293s | | -| Ask Advice Function | ✅ Pass | 0.219s | | -| Streaming Search Query Function Streaming | ✅ Pass | 0.222s | | -| Basic Context Memory Test | ✅ Pass | 0.417s | | -| Function Argument Memory Test | ✅ Pass | 0.209s | | -| Function Response Memory Test | ✅ Pass | 0.210s | | -| Penetration Testing Memory with Tool Call | ✅ Pass | 0.214s | | -| Cybersecurity Workflow Memory Test | ✅ Pass | 0.219s | | -| Read a file, then edit it via unified diff | ✅ Pass | 0.406s | | -| Penetration Testing Methodology | ✅ Pass | 0.222s | | -| Vulnerability Assessment Tools | ✅ Pass | 0.215s | | -| SQL Injection Attack Type | ✅ Pass | 0.215s | | -| Penetration Testing Framework | ✅ Pass | 0.217s | | -| Web Application Security Scanner | ✅ Pass | 0.220s | | -| Penetration Testing Tool Selection | ✅ Pass | 0.233s | | +| JSON Response Function | ✅ Pass | 0.486s | | +| Search Query Function | ✅ Pass | 0.448s | | +| Ask Advice Function | ✅ Pass | 0.483s | | +| Streaming Search Query Function Streaming | ✅ Pass | 0.425s | | +| Function Argument Memory Test | ✅ Pass | 0.681s | | +| Function Response Memory Test | ✅ Pass | 0.696s | | +| Basic Context Memory Test | ✅ Pass | 0.979s | | +| Cybersecurity Workflow Memory Test | ✅ Pass | 0.512s | | +| Penetration Testing Memory with Tool Call | ✅ Pass | 1.127s | | +| Read a file, then edit it via unified diff | ✅ Pass | 1.576s | | +| Penetration Testing Methodology | ✅ Pass | 0.612s | | +| SQL Injection Attack Type | ✅ Pass | 0.461s | | +| Penetration Testing Framework | ✅ Pass | 0.361s | | +| Web Application Security Scanner | ✅ Pass | 0.392s | | +| Vulnerability Assessment Tools | ✅ Pass | 1.194s | | +| Penetration Testing Tool Selection | ✅ Pass | 0.572s | | -**Summary**: 24/24 (100.00%) successful tests +**Summary**: 25/25 (100.00%) successful tests -**Average latency**: 0.239s +**Average latency**: 0.807s --- diff --git a/scripts/version.ps1 b/scripts/version.ps1 index 52e5ce2a..875354c5 100644 --- a/scripts/version.ps1 +++ b/scripts/version.ps1 @@ -28,7 +28,8 @@ $tagCommit = git rev-list -n 1 $latestTag 2>$null # Set revision only if current commit differs from tag commit if ($currentCommit -and ($currentCommit -ne $tagCommit)) { - $env:PACKAGE_REV = git rev-parse --short HEAD + # A fixed width, not --short: git widens the abbreviation with the size of the clone. + $env:PACKAGE_REV = $currentCommit.Substring(0, 7) $buildType = "development" } else { $env:PACKAGE_REV = "" diff --git a/scripts/version.sh b/scripts/version.sh index ead7752d..572b7036 100755 --- a/scripts/version.sh +++ b/scripts/version.sh @@ -26,7 +26,9 @@ TAG_COMMIT=$(git rev-list -n 1 "$LATEST_TAG" 2>/dev/null || echo "") # Set revision only if current commit differs from tag commit if [ -n "$CURRENT_COMMIT" ] && [ "$CURRENT_COMMIT" != "$TAG_COMMIT" ]; then - PACKAGE_REV=$(git rev-parse --short HEAD) + # A fixed width, not --short: git widens the abbreviation with the size of the clone, + # so CI and a developer checkout would stamp the same commit with different revisions. + PACKAGE_REV=$(printf '%s' "$CURRENT_COMMIT" | cut -c1-7) else PACKAGE_REV="" fi