From e4f80badd4bbdcfd52436ac2fc543bc21d68439b Mon Sep 17 00:00:00 2001
From: Sergey Kozyrenko <sirozha_blr@tut.by>
Date: Sat, 4 Jul 2026 00:34:47 +0700
Subject: [PATCH] feat(markdown-editor): decode named HTML entities outside
 code instead of freezing them

The `literalAmpersand` extension pre-encoded every `&` so @tiptap/markdown's
decodeHtmlEntities netted to a no-op, keeping `&lt;`/`&gt;`/`&amp;`/`&quot;`
byte-verbatim. But bare-prose entities in the knowledge corpus are HTML-encoding
artifacts from ingestion (e.g. `Time difference &gt; 5 min` meaning `>`), and
freezing them as `&gt;` is not what the author wrote. Drop the extension so
marked decodes the 4 named entities in prose (`&lt;`->`<`, etc.).

Verified scoped and safe: numeric refs (`&#123;`, `&#40;`) and any entity inside
code / inline-code are NOT decoded (code content bypasses inline tokenizers), a
bare `&` survives as `&` (only valid entities decode), raw `<tags>` unaffected,
and the identity serializer keeps the decoded `<` from being re-encoded on save.
Full corpus (378 real knowledge docs): zero new non-convergence, zero
bare-&->&amp; regressions. Live on :8000: `&gt; 5 min` -> `> 5 min` on load+save
while numeric/code entities stay put.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 .../markdown-editor-content-integrity.test.ts |  9 +++---
 .../markdown-editor-extensions.test.ts        | 21 +++++++++----
 .../markdown-editor/markdown-editor-marked.ts | 30 +++++--------------
 3 files changed, 29 insertions(+), 31 deletions(-)

diff --git a/frontend/src/components/shared/markdown-editor/markdown-editor-content-integrity.test.ts b/frontend/src/components/shared/markdown-editor/markdown-editor-content-integrity.test.ts
index 1da12c70..532c1584 100644
--- a/frontend/src/components/shared/markdown-editor/markdown-editor-content-integrity.test.ts
+++ b/frontend/src/components/shared/markdown-editor/markdown-editor-content-integrity.test.ts
@@ -13,8 +13,9 @@ const mulberry32 = (seed: number) => () => {
 };
 
 // Content atoms that MUST survive load→serialize verbatim: identifiers/dunders, Go-template variables,
-// xml-like tags, regex/path backslashes, HTML entities, C++. Whitespace/formatting may reflow; these
-// bytes may not disappear or mutate.
+// xml-like tags, regex/path backslashes, numeric HTML entities + a bare `&`, C++. Whitespace/formatting may
+// reflow; these bytes may not disappear or mutate. (Named entities like `&lt;` decode to `<` by design —
+// see markdown-editor-marked.ts — so they are NOT survive-verbatim atoms.)
 const ATOMS = [
     '__init__',
     '__call__',
@@ -32,9 +33,9 @@ const ATOMS = [
     'regex \\.php files',
     'glob \\* and \\?',
     'escaped \\[ \\| \\+ here',
-    '&lt;script&gt;',
+    '&#40;paren&#41;',
     'AT&T',
-    'x &amp; y',
+    '2>&1 redirect',
     'C++ then C++',
     '~10% left',
 ];
diff --git a/frontend/src/components/shared/markdown-editor/markdown-editor-extensions.test.ts b/frontend/src/components/shared/markdown-editor/markdown-editor-extensions.test.ts
index e6b5a6e1..3717c7aa 100644
--- a/frontend/src/components/shared/markdown-editor/markdown-editor-extensions.test.ts
+++ b/frontend/src/components/shared/markdown-editor/markdown-editor-extensions.test.ts
@@ -109,16 +109,27 @@ describe('backslash before punctuation survives — escape tokenizer neutralized
     });
 });
 
-describe('HTML entities stay literal — decode neutralized (a pentest doc teaching &lt;script&gt; keeps its source)', () => {
+describe('named HTML entities decode outside code; numeric refs, bare & and raw <tags> survive', () => {
     it.each([
-        'encode &lt;script&gt; as text',
-        'ampersand AT&T and a &amp; b',
-        'quote &quot;value&quot; and &gt; alone and &lt; too',
+        ['encode &lt;script&gt; as text', 'encode <script> as text'],
+        ['ampersand AT&amp;T and a &amp; b', 'ampersand AT&T and a & b'],
+        ['quote &quot;value&quot; and &gt; alone and &lt; too', 'quote "value" and > alone and < too'],
+    ])('decodes %s to its characters', (src, expected) => {
+        expect(roundTrip(src)).toBe(expected);
+    });
+
+    it.each([
+        'numeric &#123; and &#40; and &#x25; stay', // numeric refs are not decoded
+        'bare AT&T and a & b and 2>&1 survive', // a lone & is left literal
     ])('keeps %s byte-identical', (s) => {
         expect(roundTrip(s)).toBe(s);
     });
 
-    it('literal <tags> (no entity) still stay literal, never entity-encoded', () => {
+    it('entities inside inline code are preserved (code content is not decoded)', () => {
+        expect(roundTrip('`&lt;script&gt;`')).toBe('`&lt;script&gt;`');
+    });
+
+    it('literal <tags> (raw, no entity) still stay literal, never entity-encoded', () => {
         expect(roundTrip('use <input> and <container_environment> here')).toBe(
             'use <input> and <container_environment> here',
         );
diff --git a/frontend/src/components/shared/markdown-editor/markdown-editor-marked.ts b/frontend/src/components/shared/markdown-editor/markdown-editor-marked.ts
index 1a21c9cb..232c7d9a 100644
--- a/frontend/src/components/shared/markdown-editor/markdown-editor-marked.ts
+++ b/frontend/src/components/shared/markdown-editor/markdown-editor-marked.ts
@@ -14,24 +14,6 @@ const createFaithfulMarked = () => {
     const instance = new Marked();
 
     instance.use({
-        extensions: [
-            // @tiptap/markdown runs decodeHtmlEntities (`&lt;`→`<`, `&amp;`→`&`, `&gt;`→`>`, `&quot;`→`"`) on
-            // every text token during parse — a module-scope call its `.lexer()`/`.inlineTokens()` path never
-            // routes through marked's walkTokens/hooks, so those can't intercept it. Emit each literal `&` as
-            // its own text token pre-encoded to `&amp;`; the decode then nets back to the original byte
-            // (`&amp;`→`&`), leaving a source `&lt;`/`&amp;quot;` intact instead of collapsing it to `<`/`"`.
-            {
-                level: 'inline',
-                name: 'literalAmpersand',
-                start: (src: string) => {
-                    const index = src.indexOf('&');
-
-                    return index < 0 ? undefined : index;
-                },
-                tokenizer: (src: string) =>
-                    src[0] === '&' ? { raw: '&', text: '&amp;', type: 'text' as const } : undefined,
-            },
-        ],
         tokenizer: {
             // These marked tokenizers auto-convert literal text into markup, mangling Go-template / pentest
             // prose on round-trip. Returning `undefined` forces the char to stay literal text; `false` defers
@@ -43,6 +25,10 @@ const createFaithfulMarked = () => {
             //   • html/tag — keep `<xml-like>` tags literal (marked swallows real-HTML-element names)
             // NB: autolink/url are intentionally NOT neutralised — a bare `https://…`, `<url>` or email is
             // meant to become a link (see markdown-editor-extensions.ts link config, kept symmetric with typing).
+            // NB: named HTML entities (`&lt; &gt; &amp; &quot;`) are LEFT to marked's decoder — a bare-prose
+            // `&lt;` decodes to `<` (fixes HTML-encoding artifacts from ingestion). Numeric refs (`&#123;`) and
+            // anything inside code are untouched; a bare `&` survives as `&`. Don't re-add a literalAmpersand
+            // token to "preserve" `&lt;` — that re-freezes the artifacts.
             del: (src: string) => (/^~~(?!~)/.test(src) ? false : undefined),
             emStrong: (src: string) => (/^_/.test(src) ? undefined : false),
             escape: () => undefined,
@@ -57,10 +43,10 @@ const createFaithfulMarked = () => {
 
 // The serialize-side counterpart to createFaithfulMarked. @tiptap/markdown's MarkdownManager
 // .encodeTextForMarkdown HTML-entity-encodes text (`<` → `&lt;`) and backslash-escapes ``` ` * _ [ ] ~ \ ```.
-// Both are wrong here: the load side neutralises marked's escape/html/tag tokenizers, so a `\`-escape or
-// `&lt;`-entity is NEVER decoded on parse — anything we encode now resurfaces as a literal backslash / entity
-// on the next load, the exact corruption this editor avoids. Serialization is hard-coded in the manager (no
-// per-extension hook), so replace that one method with identity, keeping load and save byte-symmetric.
+// Both are wrong here: the load side keeps `\`+punct literal (escape tokenizer off), so re-escaping on save
+// would double every backslash; and named HTML entities are DECODED on load (`&lt;`→`<`), so re-encoding
+// would freeze them back into `&lt;`. Serialization is hard-coded in the manager (no per-extension hook), so
+// replace that one method with identity — save emits exactly the text the doc holds.
 type ManagerWithEncode = { encodeTextForMarkdown: (text: string) => string };
 
 const FaithfulMarkdownText = Extension.create({