feat(markdown-editor): decode named HTML entities outside code instead of freezing them

The `literalAmpersand` extension pre-encoded every `&` so @tiptap/markdown's
decodeHtmlEntities netted to a no-op, keeping `<`/`>`/`&`/`"`
byte-verbatim. But bare-prose entities in the knowledge corpus are HTML-encoding
artifacts from ingestion (e.g. `Time difference > 5 min` meaning `>`), and
freezing them as `>` is not what the author wrote. Drop the extension so
marked decodes the 4 named entities in prose (`&lt;`->`<`, etc.).

Verified scoped and safe: numeric refs (`&#123;`, `&#40;`) and any entity inside
code / inline-code are NOT decoded (code content bypasses inline tokenizers), a
bare `&` survives as `&` (only valid entities decode), raw `<tags>` unaffected,
and the identity serializer keeps the decoded `<` from being re-encoded on save.
Full corpus (378 real knowledge docs): zero new non-convergence, zero
bare-&->&amp; regressions. Live on :8000: `&gt; 5 min` -> `> 5 min` on load+save
while numeric/code entities stay put.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

frontend/src/components/shared/markdown-editor/markdown-editor-content-integrity.test.ts

@@ -13,8 +13,9 @@ const mulberry32 = (seed: number) => () => {
 };
 
 // Content atoms that MUST survive load→serialize verbatim: identifiers/dunders, Go-template variables,
-// xml-like tags, regex/path backslashes, HTML entities, C++. Whitespace/formatting may reflow; these
-// bytes may not disappear or mutate.
+// xml-like tags, regex/path backslashes, numeric HTML entities + a bare `&`, C++. Whitespace/formatting may
+// reflow; these bytes may not disappear or mutate. (Named entities like `&lt;` decode to `<` by design —
+// see markdown-editor-marked.ts — so they are NOT survive-verbatim atoms.)
 const ATOMS = [
     '__init__',
     '__call__',
@@ -32,9 +33,9 @@ const ATOMS = [
     'regex \\.php files',
     'glob \\* and \\?',
     'escaped \\[ \\| \\+ here',
-    '&lt;script&gt;',
+    '&#40;paren&#41;',
     'AT&T',
-    'x &amp; y',
+    '2>&1 redirect',
     'C++ then C++',
     '~10% left',
 ];

frontend/src/components/shared/markdown-editor/markdown-editor-extensions.test.ts

@@ -109,16 +109,27 @@ describe('backslash before punctuation survives — escape tokenizer neutralized
     });
 });
 
-describe('HTML entities stay literal — decode neutralized (a pentest doc teaching <script> keeps its source)', () => {
+describe('named HTML entities decode outside code; numeric refs, bare & and raw <tags> survive', () => {
     it.each([
-        'encode &lt;script&gt; as text',
-        'ampersand AT&T and a &amp; b',
-        'quote &quot;value&quot; and &gt; alone and &lt; too',
+        ['encode &lt;script&gt; as text', 'encode <script> as text'],
+        ['ampersand AT&amp;T and a &amp; b', 'ampersand AT&T and a & b'],
+        ['quote &quot;value&quot; and &gt; alone and &lt; too', 'quote "value" and > alone and < too'],
+    ])('decodes %s to its characters', (src, expected) => {
+        expect(roundTrip(src)).toBe(expected);
+    });
+
+    it.each([
+        'numeric &#123; and &#40; and &#x25; stay', // numeric refs are not decoded
+        'bare AT&T and a & b and 2>&1 survive', // a lone & is left literal
     ])('keeps %s byte-identical', (s) => {
         expect(roundTrip(s)).toBe(s);
     });
 
-    it('literal <tags> (no entity) still stay literal, never entity-encoded', () => {
+    it('entities inside inline code are preserved (code content is not decoded)', () => {
+        expect(roundTrip('`&lt;script&gt;`')).toBe('`&lt;script&gt;`');
+    });
+
+    it('literal <tags> (raw, no entity) still stay literal, never entity-encoded', () => {
         expect(roundTrip('use <input> and <container_environment> here')).toBe(
             'use <input> and <container_environment> here',
         );

frontend/src/components/shared/markdown-editor/markdown-editor-marked.ts

@@ -14,24 +14,6 @@ const createFaithfulMarked = () => {
     const instance = new Marked();
 
     instance.use({
-        extensions: [
-            // @tiptap/markdown runs decodeHtmlEntities (`&lt;`→`<`, `&amp;`→`&`, `&gt;`→`>`, `&quot;`→`"`) on
-            // every text token during parse — a module-scope call its `.lexer()`/`.inlineTokens()` path never
-            // routes through marked's walkTokens/hooks, so those can't intercept it. Emit each literal `&` as
-            // its own text token pre-encoded to `&amp;`; the decode then nets back to the original byte
-            // (`&amp;`→`&`), leaving a source `&lt;`/`&amp;quot;` intact instead of collapsing it to `<`/`"`.
-            {
-                level: 'inline',
-                name: 'literalAmpersand',
-                start: (src: string) => {
-                    const index = src.indexOf('&');
-
-                    return index < 0 ? undefined : index;
-                },
-                tokenizer: (src: string) =>
-                    src[0] === '&' ? { raw: '&', text: '&amp;', type: 'text' as const } : undefined,
-            },
-        ],
         tokenizer: {
             // These marked tokenizers auto-convert literal text into markup, mangling Go-template / pentest
             // prose on round-trip. Returning `undefined` forces the char to stay literal text; `false` defers
@@ -43,6 +25,10 @@ const createFaithfulMarked = () => {
             //   • html/tag — keep `<xml-like>` tags literal (marked swallows real-HTML-element names)
             // NB: autolink/url are intentionally NOT neutralised — a bare `https://…`, `<url>` or email is
             // meant to become a link (see markdown-editor-extensions.ts link config, kept symmetric with typing).
+            // NB: named HTML entities (`&lt; &gt; &amp; &quot;`) are LEFT to marked's decoder — a bare-prose
+            // `&lt;` decodes to `<` (fixes HTML-encoding artifacts from ingestion). Numeric refs (`&#123;`) and
+            // anything inside code are untouched; a bare `&` survives as `&`. Don't re-add a literalAmpersand
+            // token to "preserve" `&lt;` — that re-freezes the artifacts.
             del: (src: string) => (/^~~(?!~)/.test(src) ? false : undefined),
             emStrong: (src: string) => (/^_/.test(src) ? undefined : false),
             escape: () => undefined,
@@ -57,10 +43,10 @@ const createFaithfulMarked = () => {
 
 // The serialize-side counterpart to createFaithfulMarked. @tiptap/markdown's MarkdownManager
 // .encodeTextForMarkdown HTML-entity-encodes text (`<` → `&lt;`) and backslash-escapes ``` ` * _ [ ] ~ \ ```.
-// Both are wrong here: the load side neutralises marked's escape/html/tag tokenizers, so a `\`-escape or
-// `&lt;`-entity is NEVER decoded on parse — anything we encode now resurfaces as a literal backslash / entity
-// on the next load, the exact corruption this editor avoids. Serialization is hard-coded in the manager (no
-// per-extension hook), so replace that one method with identity, keeping load and save byte-symmetric.
+// Both are wrong here: the load side keeps `\`+punct literal (escape tokenizer off), so re-escaping on save
+// would double every backslash; and named HTML entities are DECODED on load (`&lt;`→`<`), so re-encoding
+// would freeze them back into `&lt;`. Serialization is hard-coded in the manager (no per-extension hook), so
+// replace that one method with identity — save emits exactly the text the doc holds.
 type ManagerWithEncode = { encodeTextForMarkdown: (text: string) => string };
 
 const FaithfulMarkdownText = Extension.create({