test(e2e): cover the two detail routes whose editor loads server content

/settings/prompts/:promptId and /templates/:templateId had no test on any tier,
and they are where MarkdownEditorField loads content from the server — the
prompts list spec only expands a row into a <pre>, and the templates spec only
exercises create mode. The editor's one shipped crash reproduced solely in a
production build, which is exactly what the mock tier runs.

Each route now loads a non-trivial body (headings, list, fenced command, table,
and the {{.Var}} / {{PLACEHOLDER}} atoms the backend parses) and asserts both
halves: the raw view matches the loaded source byte-exact, and after an edit in
the rich editor every atom survives its serialization.

Also close the hole that let them stay uncovered: route builders are functions,
so the manifest's static path walk never saw them. Every builder must now
declare where it is covered or why it is not, and the check fails when a new one
appears undeclared.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Sergey Kozyrenko
2026-07-22 13:18:20 +07:00
co-authored by Claude Opus 4.8
parent f3311ecadf
commit ce1fc7049d
5 changed files with 259 additions and 6 deletions
@@ -19,7 +19,40 @@ const agentPrompt = (system: PromptType) => entity('AgentPrompt', { system: prom
const agentPromptPair = (system: PromptType, human: PromptType) =>
entity('AgentPrompts', { human: prompt(human), system: prompt(system) });
const settingsPrompts: ResultOf<typeof SettingsPromptsDocument> = {
/**
* A realistic Go `text/template` for the prompt detail route: the backend parses these with
* text/template, so the `{{.Var}}` atoms and the fenced command must survive the editor's
* parse/serialize round-trip. Deliberately has no list-nested fence — that is a separate,
* already-tracked editor defect and would make this spec assert a known bug.
*/
export const RICH_PROMPT_TEMPLATE = [
'# Pentester',
'',
'You are assessing **{{.Target}}** within {{.Scope}}.',
'',
'## Rules',
'',
'- stay inside the agreed scope',
'- report every finding with evidence',
'',
'```bash',
'nmap -sV {{.Target}}',
'```',
'',
'| Field | Value |',
'| --- | --- |',
'| Scope | {{.Scope}} |',
].join('\n');
const richPentesterSystem = entity('DefaultPrompt', {
template: RICH_PROMPT_TEMPLATE,
type: PromptType.Pentester,
variables: ['Target', 'Scope'],
});
const makeSettingsPrompts = (
pentesterSystem: DefaultPromptFragmentFragment = prompt(PromptType.Pentester),
): ResultOf<typeof SettingsPromptsDocument> => ({
settingsPrompts: entity('PromptsConfig', {
default: entity('DefaultPrompts', {
agents: entity('AgentsPrompts', {
@@ -30,7 +63,10 @@ const settingsPrompts: ResultOf<typeof SettingsPromptsDocument> = {
generator: agentPromptPair(PromptType.Generator, PromptType.SubtasksGenerator),
installer: agentPromptPair(PromptType.Installer, PromptType.QuestionInstaller),
memorist: agentPromptPair(PromptType.Memorist, PromptType.QuestionMemorist),
pentester: agentPromptPair(PromptType.Pentester, PromptType.QuestionPentester),
pentester: entity('AgentPrompts', {
human: prompt(PromptType.QuestionPentester),
system: pentesterSystem,
}),
primaryAgent: agentPrompt(PromptType.PrimaryAgent),
refiner: agentPromptPair(PromptType.Refiner, PromptType.SubtasksRefiner),
reflector: agentPromptPair(PromptType.Reflector, PromptType.QuestionReflector),
@@ -56,7 +92,9 @@ const settingsPrompts: ResultOf<typeof SettingsPromptsDocument> = {
}),
userDefined: [],
}),
};
});
const settingsPrompts = makeSettingsPrompts();
export const settingsPromptsCassette = (override: Cassette = {}): Cassette =>
mergeCassettes(
@@ -69,3 +107,19 @@ export const settingsPromptsCassette = (override: Cassette = {}): Cassette =>
},
override,
);
/** The agent key the prompt detail route resolves from its `:promptId` param. */
export const PROMPT_DETAIL_AGENT = 'pentester';
/** Same config, but that agent's system prompt carries RICH_PROMPT_TEMPLATE. */
export const promptDetailCassette = (override: Cassette = {}): Cassette =>
mergeCassettes(
{
queries: {
...baseQueries(),
settingsPrompts: [{ data: makeSettingsPrompts(richPentesterSystem) }],
},
rest: baseRest(),
},
override,
);
+43 -1
View File
@@ -1,6 +1,6 @@
import type { ResultOf } from '@graphql-typed-document-node/core';
import type { FlowTemplateFragmentFragment, FlowTemplatesDocument } from '@/graphql/types';
import type { FlowTemplateDocument, FlowTemplateFragmentFragment, FlowTemplatesDocument } from '@/graphql/types';
import type { Cassette } from '../cassette.ts';
@@ -23,6 +23,35 @@ export const TEMPLATE_SEED = makeTemplate('11', 'E2E Seed Template', 'Scan the t
const flowTemplates: ResultOf<typeof FlowTemplatesDocument> = { flowTemplates: [TEMPLATE_SEED] };
/**
* Non-trivial body for the template detail route, whose editor loads it from the server.
* Carries the `{{PLACEHOLDER}}` atoms a flow template is built from, so a parse/serialize
* round-trip that drops them fails the spec. No list-nested fence — that is a separate,
* already-tracked editor defect.
*/
export const RICH_TEMPLATE_TEXT = [
'# Recon',
'',
'Target: **{{TARGET}}** (scope {{SCOPE}}).',
'',
'## Steps',
'',
'- enumerate services',
'- capture evidence for each finding',
'',
'```bash',
'nmap -sV {{TARGET}}',
'```',
'',
'| Field | Value |',
'| --- | --- |',
'| Scope | {{SCOPE}} |',
].join('\n');
export const TEMPLATE_DETAIL = makeTemplate('11', 'E2E Seed Template', RICH_TEMPLATE_TEXT);
const flowTemplate: ResultOf<typeof FlowTemplateDocument> = { flowTemplate: TEMPLATE_DETAIL };
export const templatesCassette = (override: Cassette = {}): Cassette =>
mergeCassettes(
{
@@ -34,3 +63,16 @@ export const templatesCassette = (override: Cassette = {}): Cassette =>
},
override,
);
/** Adds the single-template query the detail route issues for TEMPLATE_DETAIL. */
export const templateDetailCassette = (override: Cassette = {}): Cassette =>
templatesCassette(
mergeCassettes(
{
queries: {
flowTemplate: [{ data: flowTemplate, variables: { templateId: TEMPLATE_DETAIL.id } }],
},
},
override,
),
);
+35 -2
View File
@@ -8,8 +8,7 @@ import { ROUTE_MANIFEST } from './routes.ts';
* Routes deliberately outside the manifest sweeps (nav, visual, a11y,
* diff-scoping), each with the reason. Adding a route to lib/routes forces a
* decision here: give it a manifest entry or list it with a reason — it cannot
* silently stay out of every sweep. Dynamic builders (`routes.flow(id)`, …)
* are functions and out of this static check's scope.
* silently stay out of every sweep.
*/
const EXCLUDED: Record<string, string> = {
'/': 'redirects to /dashboard',
@@ -21,6 +20,22 @@ const EXCLUDED: Record<string, string> = {
'/templates/new': 'create-mode variant of the template detail page',
};
/**
* Dynamic route builders are functions, so the static walk below cannot see them — a new detail
* route could otherwise stay outside every sweep with nothing failing. Each builder must say where
* it is covered, or why it is not; the test asserts this list matches the builders that exist.
*/
const DYNAMIC_ROUTES: Record<string, string> = {
flow: 'manifest entry (routes.flow("5"))',
flowReport: 'not swept: needs a finished-flow report cassette',
knowledge: 'specs/crud/knowledges.spec.ts — detail page after create',
login: 'specs/smoke.spec.ts + the /login a11y scan',
'settings.newProvider': 'specs/settings/providers.spec.ts — opened from the empty state',
'settings.prompt': 'specs/settings/prompt-detail.spec.ts',
'settings.provider': 'not swept: provider form; unit-covered by settings-provider.test.tsx',
template: 'specs/crud/template-detail.spec.ts',
};
const staticPaths = (node: unknown): string[] => {
if (typeof node === 'string') {
return [node];
@@ -33,6 +48,20 @@ const staticPaths = (node: unknown): string[] => {
return [];
};
const dynamicRouteKeys = (node: unknown, prefix = ''): string[] => {
if (typeof node === 'function') {
return [prefix];
}
if (node && typeof node === 'object') {
return Object.entries(node).flatMap(([key, value]) =>
dynamicRouteKeys(value, prefix ? `${prefix}.${key}` : key),
);
}
return [];
};
describe('ROUTE_MANIFEST completeness', () => {
const manifestPaths = new Set(ROUTE_MANIFEST.map((entry) => entry.path));
@@ -45,4 +74,8 @@ describe('ROUTE_MANIFEST completeness', () => {
it('keeps the exclusion list free of routes the manifest already covers', () => {
expect(Object.keys(EXCLUDED).filter((path) => manifestPaths.has(path))).toEqual([]);
});
it('forces a coverage decision for every dynamic route builder', () => {
expect(dynamicRouteKeys(routes).sort()).toEqual(Object.keys(DYNAMIC_ROUTES).sort());
});
});
@@ -0,0 +1,58 @@
import type { Page } from '@playwright/test';
import { expect, test } from '../../fixtures/test.ts';
import { expectCleanPage } from '../../helpers/errors.ts';
import { RICH_TEMPLATE_TEXT, TEMPLATE_DETAIL, templateDetailCassette } from '../../mocks/cassettes/templates.ts';
// The other route whose editor loads server content: the templates list spec only exercises
// create mode, so nothing covered the load path in the production bundle this tier runs.
test.describe('template detail', { tag: '@coverage' }, () => {
test.use({ cassette: templateDetailCassette() });
const EDITOR = 'Template content';
// The raw/rich switch lives inside the actions menu, which stays open on select.
const switchToRaw = async (page: Page) => {
await page.getByRole('button', { name: 'Template actions' }).click();
await page.getByLabel('Raw source').click();
await page.keyboard.press('Escape');
};
test('loads the template body into the editor byte-exact', async ({ page, pageErrorLog }) => {
await page.goto(`/templates/${TEMPLATE_DETAIL.id}`);
const editor = page.getByRole('textbox', { name: EDITOR });
await expect(editor).toBeVisible();
await expect(editor.getByRole('heading', { name: 'Recon' })).toBeVisible();
await expect(editor.getByText('nmap -sV {{TARGET}}')).toBeVisible();
await switchToRaw(page);
await expect(page.getByRole('textbox', { name: EDITOR })).toHaveValue(RICH_TEMPLATE_TEXT);
expectCleanPage(pageErrorLog);
});
test('carries the placeholders through an edit in the rich editor', async ({ page, pageErrorLog }) => {
await page.goto(`/templates/${TEMPLATE_DETAIL.id}`);
const editor = page.getByRole('textbox', { name: EDITOR });
await expect(editor.getByRole('heading', { name: 'Recon' })).toBeVisible();
await editor.click();
await page.keyboard.press('ControlOrMeta+End');
await editor.pressSequentially(' E2E-MARK');
await switchToRaw(page);
const raw = await page.getByRole('textbox', { name: EDITOR }).inputValue();
for (const atom of ['Recon', '{{TARGET}}', '{{SCOPE}}', 'nmap -sV', 'capture evidence for each finding']) {
expect(raw, `"${atom}" survived the editor round-trip`).toContain(atom);
}
expect(raw).toContain('E2E-MARK');
expectCleanPage(pageErrorLog);
});
});
@@ -0,0 +1,66 @@
import type { Page } from '@playwright/test';
import { expect, test } from '../../fixtures/test.ts';
import { expectCleanPage } from '../../helpers/errors.ts';
import {
PROMPT_DETAIL_AGENT,
promptDetailCassette,
RICH_PROMPT_TEMPLATE,
} from '../../mocks/cassettes/settings-prompts.ts';
// The route the editor loads a real Go text/template into. Covered by nothing else: the prompts
// list spec expands the row in place, which renders a <pre>, never the editor — and the editor's
// one shipped crash reproduced only in a production build, which is what this tier runs.
test.describe('settings prompt detail', { tag: '@coverage' }, () => {
test.use({ cassette: promptDetailCassette() });
const EDITOR = 'System prompt template';
// The raw/rich switch lives inside the actions menu, which stays open on select.
const switchToRaw = async (page: Page) => {
await page.getByRole('button', { name: 'Prompt actions' }).click();
await page.getByLabel('Raw source').click();
await page.keyboard.press('Escape');
};
test('loads the template into the editor byte-exact', async ({ page, pageErrorLog }) => {
await page.goto(`/settings/prompts/${PROMPT_DETAIL_AGENT}`);
const editor = page.getByRole('textbox', { name: EDITOR });
await expect(editor).toBeVisible();
// The rich editor parsed the markdown rather than showing source.
await expect(editor.getByRole('heading', { name: 'Pentester' })).toBeVisible();
await expect(editor.getByText('nmap -sV {{.Target}}')).toBeVisible();
await switchToRaw(page);
await expect(page.getByRole('textbox', { name: EDITOR })).toHaveValue(RICH_PROMPT_TEMPLATE);
expectCleanPage(pageErrorLog);
});
test('carries the template atoms through an edit in the rich editor', async ({ page, pageErrorLog }) => {
await page.goto(`/settings/prompts/${PROMPT_DETAIL_AGENT}`);
const editor = page.getByRole('textbox', { name: EDITOR });
await expect(editor.getByRole('heading', { name: 'Pentester' })).toBeVisible();
// Typing makes the rich editor emit its own serialization — the half a load-only
// assertion cannot see, and where a parse/serialize defect would drop content.
await editor.click();
await page.keyboard.press('ControlOrMeta+End');
await editor.pressSequentially(' E2E-MARK');
await switchToRaw(page);
const raw = await page.getByRole('textbox', { name: EDITOR }).inputValue();
for (const atom of ['Pentester', '{{.Target}}', '{{.Scope}}', 'nmap -sV', 'stay inside the agreed scope']) {
expect(raw, `"${atom}" survived the editor round-trip`).toContain(atom);
}
expect(raw).toContain('E2E-MARK');
expectCleanPage(pageErrorLog);
});
});