From 3fe470493fd66f8d4207dae0fa00a442a20a9052 Mon Sep 17 00:00:00 2001 From: dev Date: Sun, 27 Sep 2026 09:45:26 +0800 Subject: [PATCH] web: fix the interface layout and add Chinese search --- docs/design.md | 41 +++++ frontend/src/App.svelte | 2 +- frontend/src/app.css | 229 +++++++++++++++++++++++++-- frontend/src/lib/i18n.ts | 3 + frontend/src/lib/types.ts | 6 + frontend/src/pages/Api.svelte | 245 ++++++++++++++++------------- frontend/src/pages/Document.svelte | 14 +- frontend/src/pages/Search.svelte | 43 +++-- frontend/src/pages/Tools.svelte | 17 +- src/api/mod.rs | 5 + src/docs/document.rs | 238 ++++++++++++++++++++++++++-- 11 files changed, 676 insertions(+), 167 deletions(-) diff --git a/docs/design.md b/docs/design.md index 5d280e6..6df99b2 100644 --- a/docs/design.md +++ b/docs/design.md @@ -215,6 +215,47 @@ rather than as a bind failure. would require an order of application that is not defined by the command line, and the resulting behaviour would be surprising. +## Interface layout + +Three layout decisions keep the pages readable at any window width. + +A page is a **reading column**. Most pages are text, a card grid, or a form, +and all three read better in a centred column than across a whole wide +screen, so the content is capped at a measure and centred. A document is the +one page that is wider than the column, because it carries a table of +contents beside the text; the table of contents is part of the cap, and a +document with no headings drops the column and uses the reading measure on +its own. A graph breaks the rule the other way: it uses the whole width, +because a wide screen should show more of the graph rather than more margin. + +The **interface index** shows one line per crate rather than the full module +documentation. A crate summary is several paragraphs, and a grid of them is +unreadable; the index shows the first sentence and the crate page shows the +rest. A doc comment is rendered as Markdown by the backend, so a comment +that contains a list, a table, or a code block is drawn as one, and the +frontend does not have to parse it. + +The **graph** is drawn on a fixed layout space that is scaled to the +viewport. A double click opens the document or the interface item a node +names, so a graph is also a way into the documentation. + +## Search + +A query is handled in two ways. A query that yields tokens is matched +against the postings; a query that yields no token, such as a single +character, is matched as a substring, because a one character query cannot +form a token and would otherwise always return nothing. + +Chinese text needs more than word splitting. Latin text is split on +whitespace and punctuation, but Chinese is written without spaces, so +splitting a Chinese sentence on non-alphanumeric characters leaves one long +token that a two character query can never match. A run of Chinese +characters therefore yields one token per adjacent pair, which is the +smallest unit specific enough to be a word. A query of 坐标 matches a +sentence that contains 坐标, which a whole-run token would not. A single +Chinese character is left to the substring fallback rather than becoming a +token, because a token of one character would match nearly every document. + ## Testing Unit tests live beside the code they test and cover the parser, the codec, diff --git a/frontend/src/App.svelte b/frontend/src/App.svelte index ebeb556..a1ee00e 100644 --- a/frontend/src/App.svelte +++ b/frontend/src/App.svelte @@ -217,7 +217,7 @@ >{tr('docs.otherLanguage', lang)} -
+
{#if status}

{status}

{/if} diff --git a/frontend/src/app.css b/frontend/src/app.css index 2d058d5..1a43714 100644 --- a/frontend/src/app.css +++ b/frontend/src/app.css @@ -52,8 +52,8 @@ --radius-small: 5px; --mono: ui-monospace, SFMono-Regular, "SF Mono", Menlo, Consolas, monospace; --sans: system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", sans-serif; - --max-width: 1440px; --sidebar-width: 19rem; + --toc-width: 15rem; } * { @@ -282,15 +282,22 @@ hr { } .content { - padding: 1.75rem 2rem 4rem; - max-width: var(--max-width); + padding: 1.75rem 2.5rem 4rem; width: 100%; + flex: 1 1 auto; } -.content.narrow { - max-width: 58rem; +/* A document is a measure of text plus the table of contents beside it, so + the reading column keeps a comfortable line length while the table of + contents sits at its right. The width is capped so that the pair stays + centred on a very wide screen rather than drifting apart. */ +.content.doc { + max-width: calc(62rem + var(--toc-width) + 2.5rem); + margin: 0 auto; } +/* The graph and the crate grid use the whole width; a wide screen shows + more of them rather than more margin. */ .content.wide { max-width: none; } @@ -496,10 +503,52 @@ button.ghost:hover:not(:disabled) { padding: 1rem 0; } +/* Search results */ + +.result-list { + list-style: none; + padding: 0; + margin: 0; + border: 1px solid var(--border); + border-radius: var(--radius); + overflow: hidden; +} + +.result { + padding: 0.85rem 1.1rem; + border-bottom: 1px solid var(--border); + background: var(--surface); +} + +.result:last-child { + border-bottom: none; +} + +.result:hover { + background: var(--surface-raised); +} + +.result-title { + font-size: 1rem; + font-weight: 600; +} + +.result-summary { + margin: 0.35rem 0 0.55rem; + color: var(--text-muted); + font-size: 0.88rem; +} + +.result-meta { + display: flex; + gap: 0.35rem; + flex-wrap: wrap; +} + /* Documentation */ .doc-body { - max-width: 54rem; + min-width: 0; } .doc-body h1, @@ -550,21 +599,29 @@ button.ghost:hover:not(:disabled) { border-bottom: 1px dotted var(--accent-strong); } +.doc-layout { + display: grid; + grid-template-columns: minmax(0, 1fr) var(--toc-width); + gap: 2.5rem; + align-items: start; +} + +/* A document with no headings gets the full width rather than an empty + column at its right. */ +.doc-layout.no-toc { + grid-template-columns: minmax(0, 1fr); + max-width: 46rem; +} + .doc-toc { position: sticky; top: 4.5rem; - align-self: flex-start; - width: 15rem; - flex: 0 0 15rem; + align-self: start; font-size: 0.85rem; max-height: calc(100vh - 6rem); overflow-y: auto; -} - -.doc-layout { - display: flex; - gap: 2.5rem; - align-items: flex-start; + padding-left: 1rem; + border-left: 1px solid var(--border); } /* Tables */ @@ -585,11 +642,17 @@ table { th, td { text-align: left; - padding: 0.45rem 0.7rem; + padding: 0.5rem 0.8rem; border-bottom: 1px solid var(--border); vertical-align: top; } +/* A table cell that holds an identifier is kept on one line so that the + column does not break a name across two lines. */ +td:first-child { + white-space: nowrap; +} + th { background: var(--surface-raised); color: var(--text-muted); @@ -761,6 +824,140 @@ tbody tr:hover { white-space: pre-wrap; } +/* A doc comment rendered as Markdown. The block spacing is tightened + because the comment is a fragment rather than a document. */ +.api-doc { + color: var(--text-muted); + font-size: 0.85rem; + margin-top: 0.5rem; +} + +.api-doc p { + margin: 0.4rem 0; +} + +.api-doc ul, +.api-doc ol { + margin: 0.4rem 0; + padding-left: 1.2rem; +} + +.api-doc h1, +.api-doc h2, +.api-doc h3, +.api-doc h4 { + font-size: 0.9rem; + margin: 0.75rem 0 0.3rem; + color: var(--text); +} + +.api-doc figure.code { + margin: 0.5rem 0; +} + +.api-doc figure.code figcaption { + display: none; +} + +.api-doc .table-wrap { + margin: 0.5rem 0; +} + +/* The interface items a tool calls */ + +.calls { + display: flex; + align-items: center; + flex-wrap: wrap; + gap: 0.4rem; + margin-top: 0.9rem; + padding-top: 0.75rem; + border-top: 1px solid var(--border); +} + +.calls-label { + color: var(--text-faint); + font-size: 0.78rem; + text-transform: uppercase; + letter-spacing: 0.05em; + margin-right: 0.2rem; +} + +/* The interface index */ + +.crate-grid { + display: grid; + grid-template-columns: repeat(auto-fill, minmax(20rem, 1fr)); + gap: 0.75rem; + margin-top: 1rem; +} + +.crate-card { + display: block; + background: var(--surface); + border: 1px solid var(--border); + border-radius: var(--radius); + padding: 0.85rem 1rem; + color: var(--text); +} + +.crate-card:hover { + border-color: var(--accent-strong); + background: var(--surface-raised); + text-decoration: none; +} + +.crate-card header { + display: flex; + align-items: baseline; + gap: 0.5rem; + flex-wrap: wrap; +} + +.crate-card .name { + font-family: var(--mono); + font-size: 0.95rem; + color: var(--accent); + font-weight: 600; +} + +.crate-card p { + margin: 0.5rem 0 0; + color: var(--text-muted); + font-size: 0.85rem; +} + +/* A row of links that wrap, used for the item filter results. */ +.chip-row { + display: flex; + flex-wrap: wrap; + gap: 0.4rem; + margin-top: 0.85rem; +} + +.chip { + display: inline-flex; + align-items: baseline; + gap: 0.4rem; + background: var(--surface-raised); + border: 1px solid var(--border); + border-radius: 999px; + padding: 0.2rem 0.7rem; + font-family: var(--mono); + font-size: 0.78rem; + color: var(--accent); +} + +.chip:hover { + border-color: var(--accent-strong); + text-decoration: none; +} + +.chip-note { + color: var(--text-faint); + font-size: 0.72rem; +} + .kind { font-family: var(--mono); font-size: 0.7rem; diff --git a/frontend/src/lib/i18n.ts b/frontend/src/lib/i18n.ts index 52d73f8..d35c2a4 100644 --- a/frontend/src/lib/i18n.ts +++ b/frontend/src/lib/i18n.ts @@ -199,6 +199,9 @@ const STRINGS: Record = { 'api.noMatch': { zh: '没有匹配的项。', en: 'No item matches that filter.' }, 'api.associated': { zh: '关联项', en: 'associated' }, 'api.noDoc': { zh: '该项没有文档注释。', en: 'This item has no documentation comment.' }, + 'api.detail': { zh: '详情', en: 'Detail' }, + 'api.noItem': { zh: '不存在接口项', en: 'There is no interface item named' }, + 'api.crates': { zh: 'crate', en: 'Crates' }, 'api.loadingItem': { zh: '正在加载…', en: 'Loading…' }, 'tools.title': { zh: '在线工具', en: 'Online tools' }, diff --git a/frontend/src/lib/types.ts b/frontend/src/lib/types.ts index 32a2457..ed83774 100644 --- a/frontend/src/lib/types.ts +++ b/frontend/src/lib/types.ts @@ -111,6 +111,8 @@ export interface ApiCrateSummary { name: string; layer: string; summary: string; + /** The module documentation, with the Markdown markup removed. */ + summary_plain: string; items: number; } @@ -119,7 +121,10 @@ export interface ApiItem { kind: string; name: string; signature: string; + /** The documentation comment as it is written in the source. */ doc: string; + /** The documentation comment rendered as HTML by the backend. */ + doc_html: string; associated: boolean; } @@ -137,6 +142,7 @@ export interface ApiCrate { name: string; layer: string; summary: string; + summary_plain: string; items: ApiItem[]; } diff --git a/frontend/src/pages/Api.svelte b/frontend/src/pages/Api.svelte index 02edc26..2790398 100644 --- a/frontend/src/pages/Api.svelte +++ b/frontend/src/pages/Api.svelte @@ -2,13 +2,19 @@ /** * The interface documentation. * + * The page has two views. The index lists the crates with a short + * description and lets a reader filter the public items by name. A crate + * view lists the items of one crate, grouped by kind, with the signature + * and the documentation of each. An item view shows one item on its own so + * that a link from a tool or a graph node lands on a readable page. + * * The catalog is extracted from the source of the library, so what is * shown here is the interface as it is written rather than a description - * of it. Selecting a crate lists its items; searching filters the item - * names across every crate. + * of it. */ import type { ApiIndex, ApiCrate, ApiItem } from '../lib/types'; import { getJson } from '../lib/api'; + import { navigate } from '../lib/router'; import { language, tr } from '../lib/i18n'; import Signature from '../lib/Signature.svelte'; @@ -22,45 +28,41 @@ /** The current language, read so that the text re-renders on a switch. */ let lang = $derived($language); - let selectedCrate = $state(null); let crate = $state(null); let item = $state<(ApiItem & { crate: string; layer: string }) | null>(null); - let itemError = $state(null); + let notFound = $state(null); let filter = $state(''); let loading = $state(false); - /** Selects a route segment: either a crate or an item name. */ + /** The kind of a member, for the heading of a group. */ + const KIND_ORDER = ['struct', 'enum', 'trait', 'fn', 'const', 'type']; + + /** Resolves a route segment: either a crate name or an item name. */ async function resolve(segment: string | null) { - selectedCrate = null; crate = null; item = null; - itemError = null; + notFound = null; if (!segment || !index) return; - const knownCrate = index.crates.find((candidate) => candidate.name === segment); - if (knownCrate) { + if (index.crates.some((candidate) => candidate.name === segment)) { loading = true; try { crate = await getJson(`/api/api/crates/${segment}`); - selectedCrate = segment; } finally { loading = false; } return; } - const knownItem = index.items.find((candidate) => candidate.name === segment); - if (!knownItem) { - itemError = `There is no crate or interface item named ${segment}.`; + if (!index.items.some((candidate) => candidate.name === segment)) { + notFound = segment; return; } loading = true; try { - const response = await getJson( + item = await getJson( `/api/api/items/${segment}`, ); - item = response; - selectedCrate = response.crate; } finally { loading = false; } @@ -81,65 +83,136 @@ candidate.name.toLowerCase().includes(needle) || candidate.crate.toLowerCase().includes(needle), ) - .slice(0, 60); + .slice(0, 80); }); /** The items of the crate grouped by kind. */ const groups = $derived.by(() => { - if (!crate) return []; - const order = ['struct', 'enum', 'trait', 'fn', 'const', 'type']; - const result: { kind: string; items: ApiItem[] }[] = []; - for (const kind of order) { - const items = crate.items.filter((candidate) => candidate.kind === kind); - if (items.length > 0) result.push({ kind, items }); - } - return result; + const current = crate; + if (!current) return []; + return KIND_ORDER.map((kind) => ({ + kind, + items: current.items.filter((candidate) => candidate.kind === kind), + })).filter((group) => group.items.length > 0); }); + + /** Opens an item route. */ + function openItem(name: string) { + navigate(`/api/${name}`); + } + + /** + * Returns the first sentence of a crate summary. + * + * A crate summary is the module documentation, which is several + * paragraphs. The index shows one sentence per crate so that the grid + * stays readable; the full text is on the crate page. + */ + function firstSentence(summary: string): string { + const text = summary.replace(/\s+/g, ' ').trim(); + if (text === '') return ''; + const end = text.search(/[.!?](\s|$)/); + if (end === -1) return text.length > 160 ? `${text.slice(0, 157)}...` : text; + return text.slice(0, end + 1); + }
-

Interface documentation

-

- The public interface of the ApsisRS layers this service builds against, extracted from - the source doc comments. Signatures are shown as they are written, with the units and - the error cases stated in the documentation of each item. -

+

{tr('api.title', lang)}

+

{tr('api.intro', lang)}

{#if !index} -

Loading the interface catalog…

-{:else} -
- {#each index.crates as candidate} -
-
-

- {candidate.name} -

- {candidate.layer} - {candidate.items} items +

{tr('api.loading', lang)}

+{:else if crate} + +

+ {tr('nav.api', lang)} / {crate.name} + {crate.layer} +

+

{crate.name}

+

{crate.summary}

+ + {#each groups as group} +

+ {group.kind === 'fn' ? 'functions' : `${group.kind}s`} + {group.items.length} +

+ {#each group.items as candidate (candidate.name + candidate.signature)} +
+
+ {candidate.name} + {#if candidate.associated} + {tr('api.associated', lang)} + {/if} +
-

{candidate.summary}

+
+ +
+ {#if candidate.doc} + +
{@html candidate.doc_html}
+ {/if}
{/each} -
+ {/each} +{:else if item} + +
+

+ {tr('nav.api', lang)} / + {item.crate} / {item.name} +

+
+ {item.kind} + + {item.name} + +
+
+ +
+ {#if item.doc} + +
{@html item.doc_html}
+ {:else} +

{tr('api.noDoc', lang)}

+ {/if} +
+{:else} + + {#if notFound} +
+

{tr('api.noItem', lang)} {notFound}。

+
+ {/if} + {#if loading} +

{tr('api.loadingItem', lang)}

+ {/if} -
-

Find an item

+
+

{tr('api.find', lang)}

{#if filter.trim() !== ''} {#if filtered.length === 0} -

No item matches that filter.

+

{tr('api.noMatch', lang)}

{:else} -
+
{#each filtered as candidate} - + {candidate.name} - · {candidate.crate} + {candidate.crate} {/each}
@@ -147,62 +220,16 @@ {/if}
- {#if loading} -

Loading…

- {/if} - - {#if itemError} -

{itemError}

- {/if} - - {#if item} -
-

- {tr('nav.api', lang)} / - {item.crate} / {item.name} -

-
- {item.kind} - - {item.name} - -
-
- -
- {#if item.doc} -

{item.doc}

- {:else} -

This item has no documentation comment.

- {/if} -
- {/if} - - {#if crate} -
-

{tr('nav.api', lang)} / {crate.name}

-

{crate.name}

-

{crate.summary}

- - {#each groups as group} -

{group.kind === 'fn' ? 'functions' : `${group.kind}s`}

- {#each group.items as candidate} -
-
- {candidate.name} - {#if candidate.associated} - associated - {/if} -
-
- -
- {#if candidate.doc} -

{candidate.doc}

- {/if} -
- {/each} - {/each} -
- {/if} +
+ {#each index.crates as candidate} + +
+ {candidate.name} + {candidate.layer} + {candidate.items} +
+

{firstSentence(candidate.summary_plain)}

+ + {/each} +
{/if} diff --git a/frontend/src/pages/Document.svelte b/frontend/src/pages/Document.svelte index 8e7f66a..04bb29c 100644 --- a/frontend/src/pages/Document.svelte +++ b/frontend/src/pages/Document.svelte @@ -81,7 +81,7 @@

{tr('docs.back', lang)}

{:else} -
+
- + {/if}
{/if} diff --git a/frontend/src/pages/Search.svelte b/frontend/src/pages/Search.svelte index 8ce8510..f62934a 100644 --- a/frontend/src/pages/Search.svelte +++ b/frontend/src/pages/Search.svelte @@ -2,8 +2,9 @@ /** * The search results page. * - * The query is read from the route fragment. Results link to the document - * that matched. + * The query is read from the route fragment. A result links to the + * document that matched, and carries a short excerpt around the match so + * that a reader can tell why it was returned. */ import type { SearchResponse } from '../lib/types'; import { language, tr } from '../lib/i18n'; @@ -18,6 +19,22 @@ /** The current language, read so that the text re-renders on a switch. */ let lang = $derived($language); + + /** + * Returns a short excerpt of a summary around the query. + * + * When the query is not in the summary, the beginning of the summary is + * returned, because the summary is still the most useful line to show. + */ + function excerpt(summary: string, needle: string, length = 200): string { + const text = summary.replace(/\s+/g, ' ').trim(); + if (text.length <= length) return text; + const at = text.toLowerCase().indexOf(needle.toLowerCase()); + if (at <= 0) return `${text.slice(0, length - 1)}…`; + const start = Math.max(0, at - Math.floor(length / 3)); + const end = Math.min(text.length, start + length); + return `${start > 0 ? '…' : ''}${text.slice(start, end)}${end < text.length ? '…' : ''}`; + }
@@ -25,7 +42,7 @@

{#if query} {tr('search.results', lang)} {query}{#if response} — {response.count} - {tr('search.of', lang)} {response.indexed} {tr('search.documents', lang)}{/if}。 + {tr('search.of', lang)} {response.indexed} {tr('search.documents', lang)}{/if} {:else} {tr('search.enter', lang)} {/if} @@ -39,15 +56,17 @@ {:else if !response || response.hits.length === 0}

{tr('search.none', lang)}

{:else} -
+
    {#each response.hits as hit} -
    -

    {hit.title}

    -

    {hit.summary}

    - {hit.chapter} - {hit.language} - {tr('search.score', lang)} {hit.score.toFixed(2)} -
    +
  • + {hit.title} +

    {excerpt(hit.summary, query)}

    +
    + {hit.chapter} + {hit.language} + {tr('search.score', lang)} {hit.score.toFixed(2)} +
    +
    • {/each} -
+ {/if} diff --git a/frontend/src/pages/Tools.svelte b/frontend/src/pages/Tools.svelte index 493b36f..33d394d 100644 --- a/frontend/src/pages/Tools.svelte +++ b/frontend/src/pages/Tools.svelte @@ -94,12 +94,17 @@ error = null; } - /** The tools grouped by category. */ + /** The tools grouped by category, in the order the registry lists them. */ const categories = $derived.by(() => { if (!registry) return []; + // A category is matched by its English form, which the backend uses as + // the stable key of the group. Comparing the objects by identity would + // never match, because the listing carries a copy per tool. return registry.categories.map((category) => ({ category, - tools: registry.tools.filter((candidate) => candidate.category === category), + tools: registry.tools.filter( + (candidate) => candidate.category.en === category.en, + ), })); }); @@ -197,12 +202,12 @@ {#if tool.api_items.length > 0} -

- {tr('tools.calls', lang)} +

+ {tr('tools.calls', lang)} {#each tool.api_items as name} - {name} + {name} {/each} -

+
{/if}
diff --git a/src/api/mod.rs b/src/api/mod.rs index 657ca93..6925654 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -229,6 +229,7 @@ impl Router { "name" => krate.name.clone(), "layer" => krate.layer.clone(), "summary" => krate.summary.clone(), + "summary_plain" => crate::docs::markdown::to_plain_text(&krate.summary), "items" => krate.items.len(), } }) @@ -279,6 +280,8 @@ impl Router { "name" => item.name.clone(), "signature" => item.signature.clone(), "doc" => item.doc.clone(), + "doc_html" => crate::docs::markdown::render(&item.doc), + "doc_plain" => crate::docs::markdown::to_plain_text(&item.doc), "associated" => item.associated, }; Response::json(json::to_string(&body)).with_cache_control("no-cache") @@ -536,6 +539,7 @@ fn crate_json(krate: &crate::docs::ApiCrate) -> Value { "name" => item.name.clone(), "signature" => item.signature.clone(), "doc" => item.doc.clone(), + "doc_html" => crate::docs::markdown::render(&item.doc), "associated" => item.associated, } }) @@ -544,6 +548,7 @@ fn crate_json(krate: &crate::docs::ApiCrate) -> Value { "name" => krate.name.clone(), "layer" => krate.layer.clone(), "summary" => krate.summary.clone(), + "summary_plain" => crate::docs::markdown::to_plain_text(&krate.summary), "items" => items, } } diff --git a/src/docs/document.rs b/src/docs/document.rs index 7bb27f1..318bd37 100644 --- a/src/docs/document.rs +++ b/src/docs/document.rs @@ -162,23 +162,62 @@ impl DocumentVariant { /// /// Returns `None` when the `title` key is absent, because a form without /// a title cannot be listed. + /// + /// A leading top level heading is removed when it repeats the title. The + /// page already shows the title, from the metadata, in the heading of the + /// article, so a body that opens with the same heading would show it + /// twice. A document whose first heading is a different string keeps it, + /// because then the two headings say different things. pub fn parse(text: &str) -> Option { let (header, body) = split_header(text); let title = field_of(header, "title").unwrap_or_default(); if title.is_empty() { return None; } + let body = strip_leading_title(body, &title); Some(Self { title, summary: field_of(header, "summary").unwrap_or_default(), - headings: collect_headings(body), - references: collect_references(body), - markdown: body.to_string(), - html: markdown::render(body), + headings: collect_headings(&body), + references: collect_references(&body), + markdown: body.clone(), + html: markdown::render(&body), }) } } +/// Removes a leading `# heading` when it equals `title`. +/// +/// Returns the body unchanged when it does not open with such a heading. +fn strip_leading_title(body: &str, title: &str) -> String { + let mut lines = body.lines().peekable(); + // Skip leading blank lines. + while matches!(lines.peek(), Some(line) if line.trim().is_empty()) { + lines.next(); + } + let Some(first) = lines.peek() else { + return body.to_string(); + }; + let trimmed = first.trim_end(); + let hashes = trimmed + .chars() + .take_while(|character| *character == '#') + .count(); + if hashes == 1 && trimmed[hashes..].starts_with(' ') { + let heading = trimmed[hashes..].trim(); + if heading == title.trim() { + lines.next(); + // Drop one following blank line so the body does not start with + // a gap. + if matches!(lines.peek(), Some(line) if line.trim().is_empty()) { + lines.next(); + } + return lines.collect::>().join("\n"); + } + } + body.to_string() +} + /// Returns the value of a metadata key, trimmed. fn field_of(header: &str, key: &str) -> Option { for line in header.lines() { @@ -552,6 +591,9 @@ struct SearchEntry { summary: String, chapter: String, title_tokens: Vec, + /// The searchable text, lowercased, kept for the substring fallback that + /// a query of a single character uses. + haystack: String, } impl SearchIndex { @@ -582,6 +624,8 @@ impl SearchIndex { summary: entry.summary.clone(), chapter: entry.chapter.clone(), title_tokens, + haystack: format!("{} {} {}", entry.title, entry.summary, entry.text) + .to_lowercase(), }); } @@ -593,13 +637,22 @@ impl SearchIndex { /// Searches the index and returns the best matches, ordered by score. /// - /// At most `limit` results are returned. A query with no usable token - /// returns nothing. + /// At most `limit` results are returned. A query with no usable token and + /// no literal match returns nothing. + /// + /// A query is handled in two ways. A query that yields tokens is matched + /// against the postings. A query that yields no token, such as a single + /// character, is matched as a substring, because a one character query + /// cannot form a token and would otherwise always return nothing. pub fn search(&self, query: &str, limit: usize) -> Vec { - let tokens = tokenise(&query.to_lowercase()); - if tokens.is_empty() { + let needle = query.trim().to_lowercase(); + if needle.is_empty() { return Vec::new(); } + let tokens = tokenise(&needle); + if tokens.is_empty() { + return self.search_substring(&needle, limit); + } let mut scores: HashMap = HashMap::new(); for token in &tokens { @@ -645,6 +698,40 @@ impl SearchIndex { hits } + /// Returns the documents whose text contains the query literally. + /// + /// Used for a query that yields no token. A match in the title scores + /// higher than a match in the body, so the ordering stays useful. + fn search_substring(&self, needle: &str, limit: usize) -> Vec { + let mut hits: Vec = self + .entries + .iter() + .enumerate() + .filter(|(_, entry)| entry.haystack.contains(needle)) + .map(|(index, entry)| { + let title_match = entry.title.to_lowercase().contains(needle); + SearchHit { + id: entry.id.clone(), + language: entry.language, + title: entry.title.clone(), + summary: entry.summary.clone(), + chapter: entry.chapter.clone(), + score: if title_match { 2.0 } else { 1.0 } + (index as f64) * 1e-9, + } + }) + .collect(); + hits.sort_by(|left, right| { + right + .score + .partial_cmp(&left.score) + .unwrap_or(std::cmp::Ordering::Equal) + .then_with(|| left.id.cmp(&right.id)) + }); + hits.dedup_by(|left, right| left.id == right.id); + hits.truncate(limit); + hits + } + /// Returns the number of indexed documents. pub fn len(&self) -> usize { self.entries.len() @@ -675,15 +762,68 @@ pub struct SearchHit { pub score: f64, } -/// Splits text into lowercased alphanumeric tokens. +/// Splits text into lowercased tokens. /// -/// Identifiers written with underscores, hyphens, or dots are split there as -/// well, so `mean_motion` and `mean-motion` both match `mean motion`. +/// Two kinds of token are produced: +/// +/// - A run of letters and digits becomes one token per word, with identifiers +/// written with underscores, hyphens, or dots split there as well, so +/// `mean_motion` and `mean-motion` both match a query for `mean motion`. +/// A word of one character is dropped, because it would match too much. +/// - A run of Chinese characters becomes one token per adjacent pair. Chinese +/// text is not written with spaces, so a word cannot be recovered by +/// splitting on whitespace; a pair of adjacent characters is the smallest +/// unit that is specific enough to be a word. Searching 坐标 therefore +/// matches a paragraph that contains 坐标, which a whole-run token would +/// not, because the run around it is longer. fn tokenise(text: &str) -> Vec { - text.split(|character: char| !character.is_alphanumeric()) - .filter(|token| token.len() >= 2) - .map(str::to_string) - .collect() + let mut tokens = Vec::new(); + let mut word = String::new(); + let mut han: Vec = Vec::new(); + + // Flushes the pending Latin word. + fn flush_word(word: &mut String, tokens: &mut Vec) { + if word.chars().count() >= 2 { + tokens.push(word.clone()); + } + word.clear(); + } + + // Flushes the pending run of Chinese characters as adjacent pairs. A + // lone character produces no token; a query of one character is handled + // by the substring search instead, which is more precise than a token + // that would match every occurrence of that character. + fn flush_han(han: &mut Vec, tokens: &mut Vec) { + for pair in han.windows(2) { + tokens.push(pair.iter().collect()); + } + han.clear(); + } + + for character in text.chars() { + if is_han(character) { + flush_word(&mut word, &mut tokens); + han.push(character); + } else if character.is_alphanumeric() { + flush_han(&mut han, &mut tokens); + word.extend(character.to_lowercase()); + } else { + flush_word(&mut word, &mut tokens); + flush_han(&mut han, &mut tokens); + } + } + flush_word(&mut word, &mut tokens); + flush_han(&mut han, &mut tokens); + tokens +} + +/// Returns `true` when a character is a Chinese ideograph. +/// +/// The ranges cover the unified ideographs, the extension A block, and the +/// compatibility ideographs, which is everything the content uses. +fn is_han(character: char) -> bool { + matches!(character as u32, + 0x3400..=0x4dbf | 0x4e00..=0x9fff | 0xf900..=0xfaff) } #[cfg(test)] @@ -742,6 +882,33 @@ BSTAR 项。 assert!(document.english.html.contains("