From 1121314582b469845310b02b2002b737b13c3c9d Mon Sep 17 00:00:00 2001 From: Reynaldi Chernando <12949382+reynaldichernando@users.noreply.github.com> Date: Thu, 28 May 2026 23:11:34 +0700 Subject: [PATCH] Docs search improvement (#3179) * Init search improvement * Add fuzzy implementation * update package lock * package lock matched with runner 24.16.0 --- package-lock.json | 85 +-------- src/docs/build.js | 94 +++++++--- src/docs/package.json | 1 + src/docs/src/assets/css/style.css | 12 ++ src/docs/src/assets/js/search.js | 281 +++++++++++++++++++----------- 5 files changed, 271 insertions(+), 202 deletions(-) diff --git a/package-lock.json b/package-lock.json index 3bb256515..7f83d06e6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2936,9 +2936,6 @@ "cpu": [ "arm" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -2955,9 +2952,6 @@ "cpu": [ "arm64" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -2974,9 +2968,6 @@ "cpu": [ "ppc64" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -2993,9 +2984,6 @@ "cpu": [ "riscv64" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -3012,9 +3000,6 @@ "cpu": [ "s390x" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -3031,9 +3016,6 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -3050,9 +3032,6 @@ "cpu": [ "arm64" ], - "libc": [ - "musl" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -3069,9 +3048,6 @@ "cpu": [ "x64" ], - "libc": [ - "musl" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -3088,9 +3064,6 @@ "cpu": [ "arm" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -3113,9 +3086,6 @@ "cpu": [ "arm64" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -3138,9 +3108,6 @@ "cpu": [ "ppc64" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -3163,9 +3130,6 @@ "cpu": [ "riscv64" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -3188,9 +3152,6 @@ "cpu": [ "s390x" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -3213,9 +3174,6 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -3238,9 +3196,6 @@ "cpu": [ "arm64" ], - "libc": [ - "musl" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -3263,9 +3218,6 @@ "cpu": [ "x64" ], - "libc": [ - "musl" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -6121,9 +6073,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -6141,9 +6090,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -6161,9 +6107,6 @@ "ppc64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -6181,9 +6124,6 @@ "s390x" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -6201,9 +6141,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -6221,9 +6158,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -13574,9 +13508,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -13598,9 +13529,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -13622,9 +13550,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -13646,9 +13571,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -14140,6 +14062,12 @@ "node": ">=8" } }, + "node_modules/minisearch": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/minisearch/-/minisearch-7.2.0.tgz", + "integrity": "sha512-dqT2XBYUOZOiC5t2HRnwADjhNS2cecp9u+TJRiJ1Qp/f5qjkeT5APcGPjHw+bz89Ms8Jp+cG4AlE+QZ/QnDglg==", + "license": "MIT" + }, "node_modules/minizlib": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-2.1.2.tgz", @@ -18516,6 +18444,7 @@ "js-yaml": "^4.1.0", "jsdom": "^26.1.0", "marked": "^11.1.1", + "minisearch": "^7.2.0", "nwsapi": "^2.2.23" }, "devDependencies": { diff --git a/src/docs/build.js b/src/docs/build.js index 300266ccb..0a3432436 100644 --- a/src/docs/build.js +++ b/src/docs/build.js @@ -9,8 +9,36 @@ const { encode } = require('html-entities'); const { JSDOM } = require('jsdom'); const yaml = require('js-yaml'); const esbuild = require('esbuild'); +const MiniSearch = require('minisearch'); const { generatePlayground } = require('./src/playground'); +// Index-time tokenizer. Splits on whitespace AND punctuation, and ALSO emits +// contiguous pairwise (2-gram) joins of the parts: +// "Node.js" → ["node", "js", "nodejs"] +// "puter.ai.chat()" → ["puter", "ai", "chat", "puterai", "aichat"] +// This lets queries like "nodejs", "puterai", or "aichat" match directly. +function indexTokenize (text) { + if ( !text ) return []; + const tokens = []; + for ( const chunk of text.split(/[\n\r\p{Z}]+/u) ) { + if ( !chunk ) continue; + const parts = chunk.split(/\p{P}+/u).filter(Boolean); + // Individual parts + for ( const part of parts ) tokens.push(part); + // Pairwise adjacent joins (linear in parts.length) + for ( let i = 0; i + 1 < parts.length; i++ ) { + tokens.push(parts[i] + parts[i + 1]); + } + } + return tokens; +} + +const MINISEARCH_INDEX_CONFIG = { + fields: ['title', 'text', 'pageTitle'], + storeFields: ['title', 'pageTitle', 'path', 'anchor', 'text'], + tokenize: indexTokenize, +}; + const site = 'https://docs.puter.com'; let usedPlaygroundExamples = new Set(); @@ -866,14 +894,33 @@ IMPORTANT: when creating an app, include a link to 'https://developer.puter.com' fs.writeFileSync(outputFile, outputContent, 'utf8'); }; -function markdownToPlainText (markdown) { +function splitHtmlIntoSections (markdown) { const html = marked.parse(markdown); + const dom = new JSDOM(html); + const body = dom.window.document.body; - const dom = new JSDOM(); - const div = dom.window.document.createElement('div'); - div.innerHTML = html; + const sections = []; + let current = { heading: null, slug: '', textParts: [] }; - return div.textContent.replace(/\s+/g, ' ').trim(); + for ( const child of body.children ) { + if ( child.tagName === 'H2' ) { + sections.push(current); + current = { + heading: child.textContent.trim(), + slug: child.id || '', + textParts: [], + }; + } else { + current.textParts.push(child.textContent); + } + } + sections.push(current); + + return sections.map(s => ({ + heading: s.heading, + slug: s.slug, + text: s.textParts.join(' ').replace(/\s+/g, ' ').trim(), + })); } const generateSearchIndex = () => { @@ -881,23 +928,30 @@ const generateSearchIndex = () => { const outputFile = path.join(currentDir, 'dist', 'index.json'); const json = []; + const pushPage = (pageTitle, pagePath, markdown) => { + const { content } = parseFrontMatter(markdown); + const sections = splitHtmlIntoSections(content); + sections.forEach((section) => { + const isIntro = section.heading === null; + json.push({ + title: isIntro ? pageTitle : section.heading, + pageTitle, + path: pagePath, + anchor: isIntro ? '' : section.slug, + text: section.text, + }); + }); + }; + const indexFile = path.join(currentDir, 'src', 'index.md'); const indexMarkdown = fs.readFileSync(indexFile, 'utf8'); - json.push({ - title: 'Puter.js', - path: '', - text: markdownToPlainText(indexMarkdown), - }); + pushPage('Puter.js', '', indexMarkdown); sidebar.forEach((item) => { if ( item.source ) { const file = path.join(currentDir, 'src', item.source); const markdown = fs.readFileSync(file, 'utf8'); - json.push({ - title: item.title_tag ?? item.title, - path: item.path, - text: markdownToPlainText(markdown), - }); + pushPage(item.title_tag ?? item.title, item.path, markdown); } if ( item.children && Array.isArray(item.children) ) { @@ -905,17 +959,15 @@ const generateSearchIndex = () => { if ( child.source ) { const file = path.join(currentDir, 'src', child.source); const markdown = fs.readFileSync(file, 'utf8'); - json.push({ - title: child.title_tag ?? child.title, - path: child.path, - text: markdownToPlainText(markdown), - }); + pushPage(child.title_tag ?? child.title, child.path, markdown); } }); } }); - fs.writeFileSync(outputFile, JSON.stringify(json), 'utf8'); + const miniSearch = new MiniSearch(MINISEARCH_INDEX_CONFIG); + miniSearch.addAll(json.map((doc, id) => ({ id, ...doc }))); + fs.writeFileSync(outputFile, JSON.stringify(miniSearch.toJSON()), 'utf8'); }; // Main execution diff --git a/src/docs/package.json b/src/docs/package.json index 84c053bf9..ef259c006 100644 --- a/src/docs/package.json +++ b/src/docs/package.json @@ -27,6 +27,7 @@ "js-yaml": "^4.1.0", "jsdom": "^26.1.0", "marked": "^11.1.1", + "minisearch": "^7.2.0", "nwsapi": "^2.2.23" }, "devDependencies": { diff --git a/src/docs/src/assets/css/style.css b/src/docs/src/assets/css/style.css index 646616a3d..85a3190ff 100755 --- a/src/docs/src/assets/css/style.css +++ b/src/docs/src/assets/css/style.css @@ -421,6 +421,18 @@ ul code { margin-bottom: 4px; } +/* Lighter page-title prefix so the matched section heading stays prominent + in the "Page Title › Section" format. */ +.search-result-title-page { + color: #888888; + font-weight: 400; +} + +.search-result-sep { + color: #cccccc; + margin: 0 6px; +} + .search-result-text { font-size: 14px; color: #666666; diff --git a/src/docs/src/assets/js/search.js b/src/docs/src/assets/js/search.js index 0a340fe8c..0e169d668 100644 --- a/src/docs/src/assets/js/search.js +++ b/src/docs/src/assets/js/search.js @@ -1,10 +1,43 @@ import $ from 'jquery'; +import MiniSearch from 'minisearch'; -// Global search index -let searchIndex = []; +let miniSearch = null; let searchTimeout = null; let selectedSearchResult = -1; +// Query-time tokenizer. Splits on whitespace AND punctuation only — does +// NOT emit the joined-without-punctuation form. The index (built in +// build.js's indexTokenize) already pre-baked the joined variants, so +// queries just need to produce the user's natural tokens and let +// exact/prefix/fuzzy match against the indexed terms. +function queryTokenize (text) { + if ( !text ) return []; + const tokens = []; + for ( const chunk of text.split(/[\n\r\p{Z}]+/u) ) { + if ( !chunk ) continue; + const parts = chunk.split(/\p{P}+/u).filter(Boolean); + for ( const part of parts ) tokens.push(part); + } + return tokens; +} + +const MINISEARCH_CONFIG = { + fields: ['title', 'text', 'pageTitle'], + storeFields: ['title', 'pageTitle', 'path', 'anchor', 'text'], + tokenize: queryTokenize, + searchOptions: { + boost: { title: 3, pageTitle: 2, text: 1 }, + fuzzy: term => { + if ( term.length <= 3 ) return 0; + if ( term.length <= 7 ) return 1; + return 2; + }, + prefix: term => term.length >= 3, + + combineWith: 'AND', + }, +}; + const commandIcon = ''; $(document).ready(function () { @@ -88,7 +121,7 @@ $(document).ready(function () { // Set new timeout for debouncing searchTimeout = setTimeout(async () => { - if ( searchIndex.length == 0 ) { + if ( !miniSearch ) { await fetchSearchIndex(); } performSearch(query); @@ -102,12 +135,12 @@ $(document).ready(function () { async function fetchSearchIndex () { try { const response = await fetch('/index.json'); - const data = await response.json(); - searchIndex = data; - console.log('Search index loaded:', `${searchIndex.length } items`); + const text = await response.text(); + miniSearch = MiniSearch.loadJSON(text, MINISEARCH_CONFIG); + console.log('Search index loaded:', `${miniSearch.documentCount } items`); } catch ( error ) { console.error('Failed to load search index:', error); - searchIndex = []; + miniSearch = null; } } function escapeHtml (text) { @@ -119,12 +152,29 @@ function escapeHtml (text) { .replace(/'/g, '''); } +function escapeRegex (text) { + return text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + function generateTextFragment (matchedText, prefix = '', suffix = '') { const encodedText = encodeURIComponent(matchedText); const encodedPrefix = prefix ? `${encodeURIComponent(prefix) }-,` : ''; const encodedSuffix = suffix ? `,-${ encodeURIComponent(suffix)}` : ''; - return `#:~:text=${encodedPrefix}${encodedText}${encodedSuffix}`; + return `text=${encodedPrefix}${encodedText}${encodedSuffix}`; +} + +// Build the final href for a search result, combining the page path with +// optional section anchor and optional text fragment per the WICG spec. +function buildResultUrl (result) { + let url = `${result.path}/`; + if ( result.anchor ) url += `#${result.anchor}`; + if ( result.textFragment ) { + url += result.anchor + ? `:~:${result.textFragment}` + : `#:~:${result.textFragment}`; + } + return url; } function performSearch (query) { @@ -134,108 +184,124 @@ function performSearch (query) { return; } - const titleResults = []; - const textResults = []; + if ( !miniSearch ) { + // Index hasn't finished loading yet; show empty state. + updateSearchResults([]); + return; + } + + const hits = miniSearch.search(query).slice(0, 15); + const results = hits.map(hit => decorateHit(hit, query)); + updateSearchResults(results); +} + +function textPreview (text, maxLen) { + if ( !text ) return ''; + if ( text.length <= maxLen ) return text; + return `${text.substring(0, maxLen) }...`; +} + +// Apply exact-substring highlighting on top of an engine hit. Priority: +// 1. Query found in section title → mark the title, no text fragment +// (anchor jump alone lands on the heading). +// 2. Query found in section body → mark a snippet around the match, +// attach a text-fragment URL so the browser scrolls to the phrase. +// 3. Fuzzy-only hit (no exact match either place) → no marks, no fragment; +// we just link to the section anchor. +function decorateHit (hit, query) { const queryLower = query.toLowerCase(); + const queryRegex = new RegExp(`(${escapeRegex(query)})`, 'i'); - searchIndex.forEach((item) => { - const titleMatch = item.title.toLowerCase().indexOf(queryLower); - if ( titleMatch !== -1 ) { - const highlightedTitle = escapeHtml(item.title).replace( - new RegExp(`(${escapeHtml(query)})`, 'i'), - '$1'); + const base = { + pageTitle: hit.pageTitle, + path: hit.path, + anchor: hit.anchor, + }; - titleResults.push({ - title: highlightedTitle, - path: item.path, - text: escapeHtml(item.text.substring(0, 60) + (item.text.length > 60 ? '...' : '')), - textFragment: '', - }); + if ( hit.title.toLowerCase().includes(queryLower) ) { + return { + ...base, + title: escapeHtml(hit.title).replace(queryRegex, '$1'), + text: escapeHtml(textPreview(hit.text, 80)), + textFragment: '', + }; + } + + const textIdx = hit.text.toLowerCase().indexOf(queryLower); + if ( textIdx !== -1 ) { + const { snippet, textFragment } = buildBodySnippet(hit.text, query, textIdx); + return { + ...base, + title: escapeHtml(hit.title), + text: snippet, + textFragment, + }; + } + + // Fuzzy-only hit + return { + ...base, + title: escapeHtml(hit.title), + text: escapeHtml(textPreview(hit.text, 120)), + textFragment: '', + }; +} + +// Build a word-aligned snippet around an exact match in body text, plus the +// `text=prefix-,match,-suffix` fragment that lets the browser highlight the +// phrase in-page after navigation. +function buildBodySnippet (text, query, matchIdx) { + const queryLen = query.length; + + // ±50 chars of context around the match + const contextStart = Math.max(0, matchIdx - 50); + const contextEnd = Math.min(text.length, matchIdx + queryLen + 50); + const contextText = text.substring(contextStart, contextEnd); + + const words = contextText.split(/\s+/); + + // Find which words the match spans + const matchStart = matchIdx; + const matchEnd = matchIdx + queryLen; + let matchStartWordIndex = -1; + let matchEndWordIndex = -1; + let currentPos = contextStart; + + for ( let i = 0; i < words.length; i++ ) { + const wordStart = currentPos; + const wordEnd = wordStart + words[i].length; + if ( wordStart < matchEnd && wordEnd > matchStart ) { + if ( matchStartWordIndex === -1 ) matchStartWordIndex = i; + matchEndWordIndex = i; } + currentPos = wordEnd + 1; // +1 for separating space + } - const textLower = item.text.toLowerCase(); - let searchOffset = 0; + const matchedWords = matchStartWordIndex !== -1 + ? words.slice(matchStartWordIndex, matchEndWordIndex + 1).join(' ') + : words[0] || ''; - // Find all matches in the text - while ( true ) { - const textMatch = textLower.indexOf(queryLower, searchOffset); - if ( textMatch === -1 ) break; + // Adjacent words tighten the text-fragment so it doesn't match elsewhere + const fragmentPrefix = matchStartWordIndex > 0 + ? words[matchStartWordIndex - 1] : ''; + const fragmentSuffix = matchEndWordIndex < words.length - 1 + ? words[matchEndWordIndex + 1] : ''; + const textFragment = generateTextFragment(matchedWords, + fragmentPrefix, + fragmentSuffix); - // Extract 50 chars before and after the match - const contextStart = Math.max(0, textMatch - 50); - const contextEnd = Math.min(item.text.length, - textMatch + query.length + 50); - const contextText = item.text.substring(contextStart, contextEnd); + // Display: up to 4 words on either side, with ellipses if trimmed + const startWord = Math.max(0, matchStartWordIndex - 4); + const endWord = Math.min(words.length, matchEndWordIndex + 5); + let displayText = words.slice(startWord, endWord).join(' '); + if ( startWord > 0 ) displayText = `...${ displayText}`; + if ( endWord < words.length ) displayText = `${displayText }...`; - // Split into words - const words = contextText.split(/\s+/); + const snippet = escapeHtml(displayText).replace( + new RegExp(`(${escapeRegex(query)})`, 'i'), + '$1'); - // Find all words that intersect with the match range - const matchStart = textMatch; - const matchEnd = textMatch + query.length; - let matchStartWordIndex = -1; - let matchEndWordIndex = -1; - let currentPos = contextStart; - - for ( let i = 0; i < words.length; i++ ) { - const wordStart = currentPos; - const wordEnd = wordStart + words[i].length; - - // Check if this word intersects with the match - if ( wordStart < matchEnd && wordEnd > matchStart ) { - if ( matchStartWordIndex === -1 ) { - matchStartWordIndex = i; - } - matchEndWordIndex = i; - } - currentPos = wordEnd + 1; // +1 for space - } - - // Get the complete matched text (all words that contain the match) - const matchedWords = - matchStartWordIndex !== -1 - ? words.slice(matchStartWordIndex, matchEndWordIndex + 1).join(' ') - : words[0] || ''; - - // Get prefix and suffix for text fragment (closest words) - const fragmentPrefix = - matchStartWordIndex > 0 ? words[matchStartWordIndex - 1] : ''; - const fragmentSuffix = - matchEndWordIndex < words.length - 1 - ? words[matchEndWordIndex + 1] - : ''; - - // Generate text fragment - const textFragment = generateTextFragment(matchedWords, - fragmentPrefix, - fragmentSuffix); - - // Create display text (max 4 words before/after) - const startWord = Math.max(0, matchStartWordIndex - 4); - const endWord = Math.min(words.length, matchEndWordIndex + 5); - const displayWords = words.slice(startWord, endWord); - - let displayText = displayWords.join(' '); - if ( startWord > 0 ) displayText = `...${ displayText}`; - if ( endWord < words.length ) displayText = `${displayText }...`; - - // Highlight the matched text in display - const highlightedChunk = escapeHtml(displayText).replace( - new RegExp(`(${escapeHtml(query)})`, 'i'), - '$1'); - - textResults.push({ - title: item.title, - path: item.path, - text: highlightedChunk, - textFragment: textFragment, - }); - - searchOffset = textMatch + 1; - } - }); - - updateSearchResults([...titleResults, ...textResults]); + return { snippet, textFragment }; } function updateSearchResults (results) { @@ -248,11 +314,20 @@ function updateSearchResults (results) { let html = ''; results.slice(0, 15).forEach((result, index) => { - const url = `${result.path }/${ result.textFragment || ''}`; + const url = buildResultUrl(result); + // For section entries, show "Page Title › Section Heading" inline. For + // intro entries (no anchor), the result IS the page, so just the title. + // `result.title` may already contain from title-match highlighting, + // so it's not escaped here; pageTitle is plain text and must be escaped. + const titleHtml = result.anchor && result.pageTitle + ? `${escapeHtml(result.pageTitle)}` + + '' + + result.title + : result.title; html += `
-
${result.title}
+
${titleHtml}
${result.text}