Docs search improvement (#3179)

* Init search improvement

* Add fuzzy implementation

* update package lock

* package lock matched with runner 24.16.0
This commit is contained in:
Reynaldi Chernando
2026-05-28 23:11:34 +07:00
committed by GitHub
parent dda6b82f0f
commit 1121314582
5 changed files with 271 additions and 202 deletions
+7 -78
View File
@@ -2936,9 +2936,6 @@
"cpu": [
"arm"
],
"libc": [
"glibc"
],
"license": "LGPL-3.0-or-later",
"optional": true,
"os": [
@@ -2955,9 +2952,6 @@
"cpu": [
"arm64"
],
"libc": [
"glibc"
],
"license": "LGPL-3.0-or-later",
"optional": true,
"os": [
@@ -2974,9 +2968,6 @@
"cpu": [
"ppc64"
],
"libc": [
"glibc"
],
"license": "LGPL-3.0-or-later",
"optional": true,
"os": [
@@ -2993,9 +2984,6 @@
"cpu": [
"riscv64"
],
"libc": [
"glibc"
],
"license": "LGPL-3.0-or-later",
"optional": true,
"os": [
@@ -3012,9 +3000,6 @@
"cpu": [
"s390x"
],
"libc": [
"glibc"
],
"license": "LGPL-3.0-or-later",
"optional": true,
"os": [
@@ -3031,9 +3016,6 @@
"cpu": [
"x64"
],
"libc": [
"glibc"
],
"license": "LGPL-3.0-or-later",
"optional": true,
"os": [
@@ -3050,9 +3032,6 @@
"cpu": [
"arm64"
],
"libc": [
"musl"
],
"license": "LGPL-3.0-or-later",
"optional": true,
"os": [
@@ -3069,9 +3048,6 @@
"cpu": [
"x64"
],
"libc": [
"musl"
],
"license": "LGPL-3.0-or-later",
"optional": true,
"os": [
@@ -3088,9 +3064,6 @@
"cpu": [
"arm"
],
"libc": [
"glibc"
],
"license": "Apache-2.0",
"optional": true,
"os": [
@@ -3113,9 +3086,6 @@
"cpu": [
"arm64"
],
"libc": [
"glibc"
],
"license": "Apache-2.0",
"optional": true,
"os": [
@@ -3138,9 +3108,6 @@
"cpu": [
"ppc64"
],
"libc": [
"glibc"
],
"license": "Apache-2.0",
"optional": true,
"os": [
@@ -3163,9 +3130,6 @@
"cpu": [
"riscv64"
],
"libc": [
"glibc"
],
"license": "Apache-2.0",
"optional": true,
"os": [
@@ -3188,9 +3152,6 @@
"cpu": [
"s390x"
],
"libc": [
"glibc"
],
"license": "Apache-2.0",
"optional": true,
"os": [
@@ -3213,9 +3174,6 @@
"cpu": [
"x64"
],
"libc": [
"glibc"
],
"license": "Apache-2.0",
"optional": true,
"os": [
@@ -3238,9 +3196,6 @@
"cpu": [
"arm64"
],
"libc": [
"musl"
],
"license": "Apache-2.0",
"optional": true,
"os": [
@@ -3263,9 +3218,6 @@
"cpu": [
"x64"
],
"libc": [
"musl"
],
"license": "Apache-2.0",
"optional": true,
"os": [
@@ -6121,9 +6073,6 @@
"arm64"
],
"dev": true,
"libc": [
"glibc"
],
"license": "MIT",
"optional": true,
"os": [
@@ -6141,9 +6090,6 @@
"arm64"
],
"dev": true,
"libc": [
"musl"
],
"license": "MIT",
"optional": true,
"os": [
@@ -6161,9 +6107,6 @@
"ppc64"
],
"dev": true,
"libc": [
"glibc"
],
"license": "MIT",
"optional": true,
"os": [
@@ -6181,9 +6124,6 @@
"s390x"
],
"dev": true,
"libc": [
"glibc"
],
"license": "MIT",
"optional": true,
"os": [
@@ -6201,9 +6141,6 @@
"x64"
],
"dev": true,
"libc": [
"glibc"
],
"license": "MIT",
"optional": true,
"os": [
@@ -6221,9 +6158,6 @@
"x64"
],
"dev": true,
"libc": [
"musl"
],
"license": "MIT",
"optional": true,
"os": [
@@ -13574,9 +13508,6 @@
"arm64"
],
"dev": true,
"libc": [
"glibc"
],
"license": "MPL-2.0",
"optional": true,
"os": [
@@ -13598,9 +13529,6 @@
"arm64"
],
"dev": true,
"libc": [
"musl"
],
"license": "MPL-2.0",
"optional": true,
"os": [
@@ -13622,9 +13550,6 @@
"x64"
],
"dev": true,
"libc": [
"glibc"
],
"license": "MPL-2.0",
"optional": true,
"os": [
@@ -13646,9 +13571,6 @@
"x64"
],
"dev": true,
"libc": [
"musl"
],
"license": "MPL-2.0",
"optional": true,
"os": [
@@ -14140,6 +14062,12 @@
"node": ">=8"
}
},
"node_modules/minisearch": {
"version": "7.2.0",
"resolved": "https://registry.npmjs.org/minisearch/-/minisearch-7.2.0.tgz",
"integrity": "sha512-dqT2XBYUOZOiC5t2HRnwADjhNS2cecp9u+TJRiJ1Qp/f5qjkeT5APcGPjHw+bz89Ms8Jp+cG4AlE+QZ/QnDglg==",
"license": "MIT"
},
"node_modules/minizlib": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/minizlib/-/minizlib-2.1.2.tgz",
@@ -18516,6 +18444,7 @@
"js-yaml": "^4.1.0",
"jsdom": "^26.1.0",
"marked": "^11.1.1",
"minisearch": "^7.2.0",
"nwsapi": "^2.2.23"
},
"devDependencies": {
+73 -21
View File
@@ -9,8 +9,36 @@ const { encode } = require('html-entities');
const { JSDOM } = require('jsdom');
const yaml = require('js-yaml');
const esbuild = require('esbuild');
const MiniSearch = require('minisearch');
const { generatePlayground } = require('./src/playground');
// Index-time tokenizer. Splits on whitespace AND punctuation, and ALSO emits
// contiguous pairwise (2-gram) joins of the parts:
// "Node.js" → ["node", "js", "nodejs"]
// "puter.ai.chat()" → ["puter", "ai", "chat", "puterai", "aichat"]
// This lets queries like "nodejs", "puterai", or "aichat" match directly.
function indexTokenize (text) {
if ( !text ) return [];
const tokens = [];
for ( const chunk of text.split(/[\n\r\p{Z}]+/u) ) {
if ( !chunk ) continue;
const parts = chunk.split(/\p{P}+/u).filter(Boolean);
// Individual parts
for ( const part of parts ) tokens.push(part);
// Pairwise adjacent joins (linear in parts.length)
for ( let i = 0; i + 1 < parts.length; i++ ) {
tokens.push(parts[i] + parts[i + 1]);
}
}
return tokens;
}
const MINISEARCH_INDEX_CONFIG = {
fields: ['title', 'text', 'pageTitle'],
storeFields: ['title', 'pageTitle', 'path', 'anchor', 'text'],
tokenize: indexTokenize,
};
const site = 'https://docs.puter.com';
let usedPlaygroundExamples = new Set();
@@ -866,14 +894,33 @@ IMPORTANT: when creating an app, include a link to 'https://developer.puter.com'
fs.writeFileSync(outputFile, outputContent, 'utf8');
};
function markdownToPlainText (markdown) {
function splitHtmlIntoSections (markdown) {
const html = marked.parse(markdown);
const dom = new JSDOM(html);
const body = dom.window.document.body;
const dom = new JSDOM();
const div = dom.window.document.createElement('div');
div.innerHTML = html;
const sections = [];
let current = { heading: null, slug: '', textParts: [] };
return div.textContent.replace(/\s+/g, ' ').trim();
for ( const child of body.children ) {
if ( child.tagName === 'H2' ) {
sections.push(current);
current = {
heading: child.textContent.trim(),
slug: child.id || '',
textParts: [],
};
} else {
current.textParts.push(child.textContent);
}
}
sections.push(current);
return sections.map(s => ({
heading: s.heading,
slug: s.slug,
text: s.textParts.join(' ').replace(/\s+/g, ' ').trim(),
}));
}
const generateSearchIndex = () => {
@@ -881,23 +928,30 @@ const generateSearchIndex = () => {
const outputFile = path.join(currentDir, 'dist', 'index.json');
const json = [];
const pushPage = (pageTitle, pagePath, markdown) => {
const { content } = parseFrontMatter(markdown);
const sections = splitHtmlIntoSections(content);
sections.forEach((section) => {
const isIntro = section.heading === null;
json.push({
title: isIntro ? pageTitle : section.heading,
pageTitle,
path: pagePath,
anchor: isIntro ? '' : section.slug,
text: section.text,
});
});
};
const indexFile = path.join(currentDir, 'src', 'index.md');
const indexMarkdown = fs.readFileSync(indexFile, 'utf8');
json.push({
title: 'Puter.js',
path: '',
text: markdownToPlainText(indexMarkdown),
});
pushPage('Puter.js', '', indexMarkdown);
sidebar.forEach((item) => {
if ( item.source ) {
const file = path.join(currentDir, 'src', item.source);
const markdown = fs.readFileSync(file, 'utf8');
json.push({
title: item.title_tag ?? item.title,
path: item.path,
text: markdownToPlainText(markdown),
});
pushPage(item.title_tag ?? item.title, item.path, markdown);
}
if ( item.children && Array.isArray(item.children) ) {
@@ -905,17 +959,15 @@ const generateSearchIndex = () => {
if ( child.source ) {
const file = path.join(currentDir, 'src', child.source);
const markdown = fs.readFileSync(file, 'utf8');
json.push({
title: child.title_tag ?? child.title,
path: child.path,
text: markdownToPlainText(markdown),
});
pushPage(child.title_tag ?? child.title, child.path, markdown);
}
});
}
});
fs.writeFileSync(outputFile, JSON.stringify(json), 'utf8');
const miniSearch = new MiniSearch(MINISEARCH_INDEX_CONFIG);
miniSearch.addAll(json.map((doc, id) => ({ id, ...doc })));
fs.writeFileSync(outputFile, JSON.stringify(miniSearch.toJSON()), 'utf8');
};
// Main execution
+1
View File
@@ -27,6 +27,7 @@
"js-yaml": "^4.1.0",
"jsdom": "^26.1.0",
"marked": "^11.1.1",
"minisearch": "^7.2.0",
"nwsapi": "^2.2.23"
},
"devDependencies": {
+12
View File
@@ -421,6 +421,18 @@ ul code {
margin-bottom: 4px;
}
/* Lighter page-title prefix so the matched section heading stays prominent
in the "Page Title Section" format. */
.search-result-title-page {
color: #888888;
font-weight: 400;
}
.search-result-sep {
color: #cccccc;
margin: 0 6px;
}
.search-result-text {
font-size: 14px;
color: #666666;
+178 -103
View File
@@ -1,10 +1,43 @@
import $ from 'jquery';
import MiniSearch from 'minisearch';
// Global search index
let searchIndex = [];
let miniSearch = null;
let searchTimeout = null;
let selectedSearchResult = -1;
// Query-time tokenizer. Splits on whitespace AND punctuation only — does
// NOT emit the joined-without-punctuation form. The index (built in
// build.js's indexTokenize) already pre-baked the joined variants, so
// queries just need to produce the user's natural tokens and let
// exact/prefix/fuzzy match against the indexed terms.
function queryTokenize (text) {
if ( !text ) return [];
const tokens = [];
for ( const chunk of text.split(/[\n\r\p{Z}]+/u) ) {
if ( !chunk ) continue;
const parts = chunk.split(/\p{P}+/u).filter(Boolean);
for ( const part of parts ) tokens.push(part);
}
return tokens;
}
const MINISEARCH_CONFIG = {
fields: ['title', 'text', 'pageTitle'],
storeFields: ['title', 'pageTitle', 'path', 'anchor', 'text'],
tokenize: queryTokenize,
searchOptions: {
boost: { title: 3, pageTitle: 2, text: 1 },
fuzzy: term => {
if ( term.length <= 3 ) return 0;
if ( term.length <= 7 ) return 1;
return 2;
},
prefix: term => term.length >= 3,
combineWith: 'AND',
},
};
const commandIcon = '<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-command-icon lucide-command"><path d="M15 6v12a3 3 0 1 0 3-3H6a3 3 0 1 0 3 3V6a3 3 0 1 0-3 3h12a3 3 0 1 0-3-3"/></svg>';
$(document).ready(function () {
@@ -88,7 +121,7 @@ $(document).ready(function () {
// Set new timeout for debouncing
searchTimeout = setTimeout(async () => {
if ( searchIndex.length == 0 ) {
if ( !miniSearch ) {
await fetchSearchIndex();
}
performSearch(query);
@@ -102,12 +135,12 @@ $(document).ready(function () {
async function fetchSearchIndex () {
try {
const response = await fetch('/index.json');
const data = await response.json();
searchIndex = data;
console.log('Search index loaded:', `${searchIndex.length } items`);
const text = await response.text();
miniSearch = MiniSearch.loadJSON(text, MINISEARCH_CONFIG);
console.log('Search index loaded:', `${miniSearch.documentCount } items`);
} catch ( error ) {
console.error('Failed to load search index:', error);
searchIndex = [];
miniSearch = null;
}
}
function escapeHtml (text) {
@@ -119,12 +152,29 @@ function escapeHtml (text) {
.replace(/'/g, '&#39;');
}
function escapeRegex (text) {
return text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
}
function generateTextFragment (matchedText, prefix = '', suffix = '') {
const encodedText = encodeURIComponent(matchedText);
const encodedPrefix = prefix ? `${encodeURIComponent(prefix) }-,` : '';
const encodedSuffix = suffix ? `,-${ encodeURIComponent(suffix)}` : '';
return `#:~:text=${encodedPrefix}${encodedText}${encodedSuffix}`;
return `text=${encodedPrefix}${encodedText}${encodedSuffix}`;
}
// Build the final href for a search result, combining the page path with
// optional section anchor and optional text fragment per the WICG spec.
function buildResultUrl (result) {
let url = `${result.path}/`;
if ( result.anchor ) url += `#${result.anchor}`;
if ( result.textFragment ) {
url += result.anchor
? `:~:${result.textFragment}`
: `#:~:${result.textFragment}`;
}
return url;
}
function performSearch (query) {
@@ -134,108 +184,124 @@ function performSearch (query) {
return;
}
const titleResults = [];
const textResults = [];
if ( !miniSearch ) {
// Index hasn't finished loading yet; show empty state.
updateSearchResults([]);
return;
}
const hits = miniSearch.search(query).slice(0, 15);
const results = hits.map(hit => decorateHit(hit, query));
updateSearchResults(results);
}
function textPreview (text, maxLen) {
if ( !text ) return '';
if ( text.length <= maxLen ) return text;
return `${text.substring(0, maxLen) }...`;
}
// Apply exact-substring highlighting on top of an engine hit. Priority:
// 1. Query found in section title → mark the title, no text fragment
// (anchor jump alone lands on the heading).
// 2. Query found in section body → mark a snippet around the match,
// attach a text-fragment URL so the browser scrolls to the phrase.
// 3. Fuzzy-only hit (no exact match either place) → no marks, no fragment;
// we just link to the section anchor.
function decorateHit (hit, query) {
const queryLower = query.toLowerCase();
const queryRegex = new RegExp(`(${escapeRegex(query)})`, 'i');
searchIndex.forEach((item) => {
const titleMatch = item.title.toLowerCase().indexOf(queryLower);
if ( titleMatch !== -1 ) {
const highlightedTitle = escapeHtml(item.title).replace(
new RegExp(`(${escapeHtml(query)})`, 'i'),
'<mark>$1</mark>');
const base = {
pageTitle: hit.pageTitle,
path: hit.path,
anchor: hit.anchor,
};
titleResults.push({
title: highlightedTitle,
path: item.path,
text: escapeHtml(item.text.substring(0, 60) + (item.text.length > 60 ? '...' : '')),
textFragment: '',
});
if ( hit.title.toLowerCase().includes(queryLower) ) {
return {
...base,
title: escapeHtml(hit.title).replace(queryRegex, '<mark>$1</mark>'),
text: escapeHtml(textPreview(hit.text, 80)),
textFragment: '',
};
}
const textIdx = hit.text.toLowerCase().indexOf(queryLower);
if ( textIdx !== -1 ) {
const { snippet, textFragment } = buildBodySnippet(hit.text, query, textIdx);
return {
...base,
title: escapeHtml(hit.title),
text: snippet,
textFragment,
};
}
// Fuzzy-only hit
return {
...base,
title: escapeHtml(hit.title),
text: escapeHtml(textPreview(hit.text, 120)),
textFragment: '',
};
}
// Build a word-aligned snippet around an exact match in body text, plus the
// `text=prefix-,match,-suffix` fragment that lets the browser highlight the
// phrase in-page after navigation.
function buildBodySnippet (text, query, matchIdx) {
const queryLen = query.length;
// ±50 chars of context around the match
const contextStart = Math.max(0, matchIdx - 50);
const contextEnd = Math.min(text.length, matchIdx + queryLen + 50);
const contextText = text.substring(contextStart, contextEnd);
const words = contextText.split(/\s+/);
// Find which words the match spans
const matchStart = matchIdx;
const matchEnd = matchIdx + queryLen;
let matchStartWordIndex = -1;
let matchEndWordIndex = -1;
let currentPos = contextStart;
for ( let i = 0; i < words.length; i++ ) {
const wordStart = currentPos;
const wordEnd = wordStart + words[i].length;
if ( wordStart < matchEnd && wordEnd > matchStart ) {
if ( matchStartWordIndex === -1 ) matchStartWordIndex = i;
matchEndWordIndex = i;
}
currentPos = wordEnd + 1; // +1 for separating space
}
const textLower = item.text.toLowerCase();
let searchOffset = 0;
const matchedWords = matchStartWordIndex !== -1
? words.slice(matchStartWordIndex, matchEndWordIndex + 1).join(' ')
: words[0] || '';
// Find all matches in the text
while ( true ) {
const textMatch = textLower.indexOf(queryLower, searchOffset);
if ( textMatch === -1 ) break;
// Adjacent words tighten the text-fragment so it doesn't match elsewhere
const fragmentPrefix = matchStartWordIndex > 0
? words[matchStartWordIndex - 1] : '';
const fragmentSuffix = matchEndWordIndex < words.length - 1
? words[matchEndWordIndex + 1] : '';
const textFragment = generateTextFragment(matchedWords,
fragmentPrefix,
fragmentSuffix);
// Extract 50 chars before and after the match
const contextStart = Math.max(0, textMatch - 50);
const contextEnd = Math.min(item.text.length,
textMatch + query.length + 50);
const contextText = item.text.substring(contextStart, contextEnd);
// Display: up to 4 words on either side, with ellipses if trimmed
const startWord = Math.max(0, matchStartWordIndex - 4);
const endWord = Math.min(words.length, matchEndWordIndex + 5);
let displayText = words.slice(startWord, endWord).join(' ');
if ( startWord > 0 ) displayText = `...${ displayText}`;
if ( endWord < words.length ) displayText = `${displayText }...`;
// Split into words
const words = contextText.split(/\s+/);
const snippet = escapeHtml(displayText).replace(
new RegExp(`(${escapeRegex(query)})`, 'i'),
'<mark>$1</mark>');
// Find all words that intersect with the match range
const matchStart = textMatch;
const matchEnd = textMatch + query.length;
let matchStartWordIndex = -1;
let matchEndWordIndex = -1;
let currentPos = contextStart;
for ( let i = 0; i < words.length; i++ ) {
const wordStart = currentPos;
const wordEnd = wordStart + words[i].length;
// Check if this word intersects with the match
if ( wordStart < matchEnd && wordEnd > matchStart ) {
if ( matchStartWordIndex === -1 ) {
matchStartWordIndex = i;
}
matchEndWordIndex = i;
}
currentPos = wordEnd + 1; // +1 for space
}
// Get the complete matched text (all words that contain the match)
const matchedWords =
matchStartWordIndex !== -1
? words.slice(matchStartWordIndex, matchEndWordIndex + 1).join(' ')
: words[0] || '';
// Get prefix and suffix for text fragment (closest words)
const fragmentPrefix =
matchStartWordIndex > 0 ? words[matchStartWordIndex - 1] : '';
const fragmentSuffix =
matchEndWordIndex < words.length - 1
? words[matchEndWordIndex + 1]
: '';
// Generate text fragment
const textFragment = generateTextFragment(matchedWords,
fragmentPrefix,
fragmentSuffix);
// Create display text (max 4 words before/after)
const startWord = Math.max(0, matchStartWordIndex - 4);
const endWord = Math.min(words.length, matchEndWordIndex + 5);
const displayWords = words.slice(startWord, endWord);
let displayText = displayWords.join(' ');
if ( startWord > 0 ) displayText = `...${ displayText}`;
if ( endWord < words.length ) displayText = `${displayText }...`;
// Highlight the matched text in display
const highlightedChunk = escapeHtml(displayText).replace(
new RegExp(`(${escapeHtml(query)})`, 'i'),
'<mark>$1</mark>');
textResults.push({
title: item.title,
path: item.path,
text: highlightedChunk,
textFragment: textFragment,
});
searchOffset = textMatch + 1;
}
});
updateSearchResults([...titleResults, ...textResults]);
return { snippet, textFragment };
}
function updateSearchResults (results) {
@@ -248,11 +314,20 @@ function updateSearchResults (results) {
let html = '';
results.slice(0, 15).forEach((result, index) => {
const url = `${result.path }/${ result.textFragment || ''}`;
const url = buildResultUrl(result);
// For section entries, show "Page Title Section Heading" inline. For
// intro entries (no anchor), the result IS the page, so just the title.
// `result.title` may already contain <mark> from title-match highlighting,
// so it's not escaped here; pageTitle is plain text and must be escaped.
const titleHtml = result.anchor && result.pageTitle
? `<span class="search-result-title-page">${escapeHtml(result.pageTitle)}</span>`
+ '<span class="search-result-sep"></span>'
+ result.title
: result.title;
html += `
<div class="search-result" data-index="${index}">
<a href="${url}" class="search-result-link skip-insta-load">
<div class="search-result-title">${result.title}</div>
<div class="search-result-title">${titleHtml}</div>
<div class="search-result-text">${result.text}</div>
</a>
</div>