diff --git a/apps/developer-hub/.gitignore b/apps/developer-hub/.gitignore index 83c6630871..f313a2e0bc 100644 --- a/apps/developer-hub/.gitignore +++ b/apps/developer-hub/.gitignore @@ -6,6 +6,9 @@ next-env.d.ts # Auto-generated by `pnpm generate:changelog` from data/changelog-diffs/. src/components/ChangeLog/generated-data.ts +# Auto-generated by `pnpm generate:price-feeds` — regenerated at each build. +src/generated/ + # Daily change-log diffs live on the `changelog-data` branch (written by # Hermes), pulled into this dir at build time by `pnpm pull:changelog`. # Never committed on main. diff --git a/apps/developer-hub/package.json b/apps/developer-hub/package.json index 6a6265a00c..a086bd780a 100644 --- a/apps/developer-hub/package.json +++ b/apps/developer-hub/package.json @@ -70,6 +70,7 @@ "fix:lint:stylelint": "stylelint --fix 'src/**/*.scss'", "generate:changelog": "bash ./scripts/pull-changelog-data.sh && tsx ./scripts/generate-changelog.ts", "generate:docs": "tsx ./scripts/generate-docs.ts", + "generate:price-feeds": "tsx ./scripts/generate-price-feeds-snapshot.ts", "pull:changelog": "bash ./scripts/pull-changelog-data.sh", "snapshot:changelog": "tsx ./scripts/snapshot-and-diff.ts", "start:dev": "next dev --port 3627", diff --git a/apps/developer-hub/scripts/generate-price-feeds-snapshot.ts b/apps/developer-hub/scripts/generate-price-feeds-snapshot.ts new file mode 100644 index 0000000000..9a4eebd15d --- /dev/null +++ b/apps/developer-hub/scripts/generate-price-feeds-snapshot.ts @@ -0,0 +1,144 @@ +/** + * Price Feeds Snapshot Generator + * + * Fetches the Hermes (Core), Hermes-beta (Core), and Lazer (Pro) feed lists at + * build time and writes them to `src/generated/price-feeds.json`. The docs + * search route (`src/app/api/search/route.ts`) imports that snapshot + * synchronously instead of fetching ~6 MB across three external APIs on every + * cold-start lambda. + * + * ## Usage + * + * Runs automatically during the build. To run it manually: + * + * ```bash + * pnpm generate:price-feeds + * ``` + * + * ## Best-effort semantics + * + * A source that fails to fetch or validate is skipped with a warning and falls + * back to the previously-generated snapshot (or an empty set on a cold build). + * This script never throws — the build must not fail because an upstream API is + * down. + */ + +import * as fs from "node:fs/promises"; +import path from "node:path"; +import type { + HermesFeed, + LazerFeed, + PriceFeedsSnapshot, +} from "../src/app/api/search/feed-schemas"; +import { hermesSchema, lazerSchema } from "../src/app/api/search/feed-schemas"; +import { SYMBOLS_API_URL } from "../src/config/pyth-pro-public"; + +const OUTPUT_PATH = "./src/generated/price-feeds.json"; + +async function readPreviousSnapshot(): Promise> { + try { + return JSON.parse( + await fs.readFile(OUTPUT_PATH, "utf8"), + ) as PriceFeedsSnapshot; + } catch { + return {}; + } +} + +async function fetchHermes(url: string): Promise { + const res = await fetch(new URL("/v2/price_feeds", url)); + const parsed = hermesSchema.safeParse(await res.json()); + if (!parsed.success) { + throw new Error(`invalid response from ${url}`); + } + return parsed.data; +} + +async function fetchLazer(): Promise { + const res = await fetch(SYMBOLS_API_URL); + const parsed = lazerSchema.safeParse(await res.json()); + if (!parsed.success) { + throw new Error(`invalid response from ${SYMBOLS_API_URL}`); + } + return parsed.data; +} + +async function fetchSource( + label: string, + fetcher: () => Promise, + previous: T[] | undefined, +): Promise { + try { + const data = await fetcher(); + // eslint-disable-next-line no-console + console.log(` ✓ ${label}: ${String(data.length)} feeds`); + return data; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + // eslint-disable-next-line no-console + console.warn( + ` ⚠ ${label}: fetch failed (${message}); using ${String(previous?.length ?? 0)} cached feeds`, + ); + return previous ?? []; + } +} + +async function writeSnapshot(snapshot: PriceFeedsSnapshot): Promise { + await fs.mkdir(path.dirname(OUTPUT_PATH), { recursive: true }); + await fs.writeFile( + OUTPUT_PATH, + JSON.stringify(snapshot, undefined, 2) + "\n", + ); +} + +// Guarantees the route's static-import target exists even if generation blew up +// before the normal write, so a cold build never fails on a missing file. +async function ensureSnapshotFile(): Promise { + try { + await fs.access(OUTPUT_PATH); + } catch { + await writeSnapshot({ hermes: [], hermesBeta: [], lazer: [] }); + } +} + +async function generatePriceFeedsSnapshot(): Promise { + // eslint-disable-next-line no-console + console.log("Generating price feeds snapshot...\n"); + + const previous = await readPreviousSnapshot(); + + const [hermes, hermesBeta, lazer] = await Promise.all([ + fetchSource( + "hermes", + () => fetchHermes("https://hermes.pyth.network"), + previous.hermes, + ), + fetchSource( + "hermes-beta", + () => fetchHermes("https://hermes-beta.pyth.network"), + previous.hermesBeta, + ), + fetchSource("lazer", fetchLazer, previous.lazer), + ]); + + await writeSnapshot({ hermes, hermesBeta, lazer }); + + // eslint-disable-next-line no-console + console.log(`\n✓ Wrote ${OUTPUT_PATH}`); +} + +try { + await generatePriceFeedsSnapshot(); +} catch (error) { + // Snapshot generation is best-effort: the previously-committed (or empty) + // snapshot is sufficient for the build. Never fail the build on this script. + const message = + error instanceof Error ? (error.stack ?? error.message) : error; + // eslint-disable-next-line no-console + console.warn( + "\n⚠ Price feeds snapshot generation failed, continuing anyway:", + ); + // eslint-disable-next-line no-console + console.warn(message); + await ensureSnapshotFile(); +} diff --git a/apps/developer-hub/src/app/api/search/feed-schemas.ts b/apps/developer-hub/src/app/api/search/feed-schemas.ts new file mode 100644 index 0000000000..e2c36c58d7 --- /dev/null +++ b/apps/developer-hub/src/app/api/search/feed-schemas.ts @@ -0,0 +1,29 @@ +import { z } from "zod"; + +// Shape of the Hermes `/v2/price_feeds` response entries we index. +export const hermesSchema = z.array( + z.object({ + attributes: z.object({ symbol: z.string() }), + id: z.string(), + }), +); + +// Shape of the Lazer (Pro) symbols API response entries we index. +export const lazerSchema = z.array( + z.object({ + description: z.string(), + name: z.string(), + pyth_lazer_id: z.number(), + symbol: z.string(), + }), +); + +export type HermesFeed = z.infer[number]; +export type LazerFeed = z.infer[number]; + +// Top-level structure of the build-time snapshot consumed by the search route. +export type PriceFeedsSnapshot = { + hermes: HermesFeed[]; + hermesBeta: HermesFeed[]; + lazer: LazerFeed[]; +}; diff --git a/apps/developer-hub/src/app/api/search/route.ts b/apps/developer-hub/src/app/api/search/route.ts index 29a8088343..82f54e859e 100644 --- a/apps/developer-hub/src/app/api/search/route.ts +++ b/apps/developer-hub/src/app/api/search/route.ts @@ -1,115 +1,268 @@ -import type { AdvancedIndex } from "fumadocs-core/search/server"; -import { createSearchAPI } from "fumadocs-core/search/server"; -import { z } from "zod"; - -import { SYMBOLS_API_URL } from "../../../config/pyth-pro-public"; +import type { AdvancedIndex, SortedResult } from "fumadocs-core/search/server"; +import { + createContentHighlighter, + initAdvancedSearch, +} from "fumadocs-core/search/server"; +import priceFeeds from "../../../generated/price-feeds.json"; import { source } from "../../../lib/source"; +import type { HermesFeed, LazerFeed, PriceFeedsSnapshot } from "./feed-schemas"; + +// Feed lists are fetched at build time by +// `scripts/generate-price-feeds-snapshot.ts` and imported statically here, so +// the search route performs zero external HTTP calls at runtime. +const snapshot = priceFeeds as PriceFeedsSnapshot; + +const CORE_FEED_PATH = "/price-feeds/core/price-feeds/price-feed-ids"; +const PRO_FEED_PATH = "/price-feeds/pro/price-feed-ids"; + +// Total results handed back to the search dialog. Without a cap, a query like +// "USD" matches most of the indexed documents and the route serialises several +// MB on every debounce tick. +const MAX_RESULTS = 24; +// Reserved slots per family, so no family can crowd out another: "USD" matches +// nearly every Core feed and used to push the first documentation hit to rank +// 624, while "solana" is dominated by docs and would otherwise surface no feeds +// at all. Whatever a family does not claim is backfilled from the rest, so a +// query with no feed matches still returns a full page of docs. +const MAX_DOC_RESULTS = 12; +const MAX_CORE_FEED_RESULTS = 6; +const MAX_PRO_FEED_RESULTS = 6; -// Define schemas for type safety -const hermesSchema = z.array( - z.object({ - id: z.string(), - attributes: z.object({ symbol: z.string() }), - }), -); - -const lazerSchema = z.array( - z.object({ - symbol: z.string(), - name: z.string(), - pyth_lazer_id: z.number(), - description: z.string(), - }), -); - -function toAdvancedIndex( - fee: z.infer[number], -): AdvancedIndex { +// fumadocs indexes `title` and `description` as their own documents, so the +// per-field `structuredData.contents` entries feeds used to carry only +// duplicated text already covered by those two fields, at 2-3x the document +// count. Symbol, name, description and feed ID all remain searchable. +const NO_STRUCTURED_DATA = { contents: [], headings: [] }; + +function hermesToAdvancedIndex(fee: HermesFeed): AdvancedIndex { return { - title: `${fee.attributes.symbol} (Core)`, description: `Price Feed ID: ${fee.id}`, - url: `/price-feeds/core/price-feeds/price-feed-ids?search=${fee.attributes.symbol}`, id: fee.id, + structuredData: NO_STRUCTURED_DATA, tag: "price-feed-core", - structuredData: { - headings: [], - contents: [ - { heading: "Symbol", content: fee.attributes.symbol }, - { heading: "ID", content: fee.id }, - ], - }, + title: `${fee.attributes.symbol} (Core)`, + url: `${CORE_FEED_PATH}?search=${fee.attributes.symbol}`, }; } -async function getHermesFeeds(): Promise { - try { - const results = await Promise.all( - ["https://hermes.pyth.network", "https://hermes-beta.pyth.network"].map( - async (url): Promise => { - const hermesResult = await fetch(new URL("/v2/price_feeds", url), { - next: { revalidate: 3600 }, - }); - const parsed = hermesSchema.safeParse(await hermesResult.json()); - return parsed.success - ? parsed.data.map((feed) => toAdvancedIndex(feed)) - : []; - }, - ), - ); - - return results.flat(); - } catch (error: unknown) { - throw new Error("Failed to fetch Hermes feeds", { cause: error }); +function lazerToAdvancedIndex(feed: LazerFeed): AdvancedIndex { + return { + description: `${feed.symbol} - ${feed.description} (ID: ${String(feed.pyth_lazer_id)})`, + id: `lazer-${String(feed.pyth_lazer_id)}`, + structuredData: NO_STRUCTURED_DATA, + tag: "price-feed-pro", + title: `${feed.name} (Pro)`, + url: `${PRO_FEED_PATH}?search=${feed.symbol}`, + }; +} + +/** + * Core feeds from mainnet and beta, deduplicated by symbol. + * + * hermes-beta republishes 99.4% of the mainnet symbol list (2,873 of 2,890), + * and both sides render to the same `SYMBOL (Core)` title pointing at the same + * `?search=SYMBOL` URL, so indexing both put literal duplicate rows in the + * search dialog and doubled the Core half of the index for nothing. Mainnet + * wins ties; the handful of beta-only symbols stay searchable. + */ +function coreFeeds(): HermesFeed[] { + const bySymbol = new Map(); + for (const feed of snapshot.hermesBeta) { + bySymbol.set(feed.attributes.symbol, feed); + } + for (const feed of snapshot.hermes) { + bySymbol.set(feed.attributes.symbol, feed); } + return [...bySymbol.values()]; } -async function getLazerFeeds(): Promise { - try { - const res = await fetch(SYMBOLS_API_URL, { next: { revalidate: 3600 } }); - const parsed = lazerSchema.safeParse(await res.json()); +type Family = "core" | "docs" | "pro"; - if (!parsed.success) { - return []; - } +// Feed symbols are dotted/slashed paths ("Crypto.BTC/USD", "Equity.US.RIVN/USD.ON"). +const SYMBOL_SEPARATORS = /[./]/; - return parsed.data.map((feed) => ({ - title: `${feed.name} (Pro)`, - description: `${feed.symbol} - ${feed.description} (ID: ${String(feed.pyth_lazer_id)})`, - url: `/price-feeds/pro/price-feed-ids?search=${feed.symbol}`, - id: `lazer-${String(feed.pyth_lazer_id)}`, - tag: "price-feed-pro", - structuredData: { - headings: [], - contents: [ - { heading: "Symbol", content: feed.symbol }, - { heading: "Name", content: feed.name }, - { heading: "Description", content: feed.description }, - { heading: "ID", content: String(feed.pyth_lazer_id) }, - ], - }, - })); - } catch (error: unknown) { - throw new Error("Failed to fetch Lazer feeds", { cause: error }); +const tokenize = (value: string): string[] => + value.toLowerCase().split(SYMBOL_SEPARATORS).filter(Boolean); + +/** + * A feed as seen by the exact-match pass. + * + * Orama ranks with BM25 over ~13k documents, which does not favour the feed a + * user actually typed: searching the exact symbol "Crypto.BTC/USD" returned 24 + * results without that feed among them, because common tokens ("crypto", + * "usd") score highly across thousands of near-identical symbols. This pass + * guarantees the obvious answer is present before Orama fills the rest. + */ +type FeedCandidate = { + family: Exclude; + index: AdvancedIndex; + name: string; + symbol: string; + tokens: Set; +}; + +function toCandidate( + family: Exclude, + symbol: string, + name: string, + index: AdvancedIndex, +): FeedCandidate { + return { + family, + index, + name: name.toLowerCase(), + symbol: symbol.toLowerCase(), + tokens: new Set([...tokenize(symbol), ...tokenize(name)]), + }; +} + +const CORE_INDEXES = coreFeeds().map((feed) => ({ + feed, + index: hermesToAdvancedIndex(feed), +})); +const PRO_INDEXES = snapshot.lazer.map((feed) => ({ + feed, + index: lazerToAdvancedIndex(feed), +})); + +const FEED_CANDIDATES: FeedCandidate[] = [ + ...CORE_INDEXES.map(({ feed, index }) => + toCandidate("core", feed.attributes.symbol, feed.attributes.symbol, index), + ), + ...PRO_INDEXES.map(({ feed, index }) => + toCandidate("pro", feed.symbol, feed.name, index), + ), +]; + +/** + * Score a feed against the query. 2 = the whole symbol or name was typed, + * 1 = every token in the query appears in the feed, 0 = no direct match. + * + * The token rule keeps "ETH/USD" on Crypto.ETH/USD without dragging in + * Crypto.ETH/USDT, and leaves vague one-character queries to Orama. + */ +function scoreCandidate( + candidate: FeedCandidate, + query: string, + queryTokens: string[], +): number { + if (candidate.symbol === query || candidate.name === query) return 2; + return queryTokens.every((token) => candidate.tokens.has(token)) ? 1 : 0; +} + +function directFeedMatches(query: string): AdvancedIndex[] { + const normalized = query.trim().toLowerCase(); + const queryTokens = tokenize(normalized); + if (queryTokens.length === 0) return []; + + const scored: { candidate: FeedCandidate; score: number }[] = []; + for (const candidate of FEED_CANDIDATES) { + const score = scoreCandidate(candidate, normalized, queryTokens); + if (score > 0) scored.push({ candidate, score }); } + + // Best match first, then the most canonical symbol: "Crypto.BTC/USD" should + // outrank "FundingRate.Binance.BTC/USDT" for the query "BTC". + scored.sort( + (a, b) => + b.score - a.score || + a.candidate.symbol.length - b.candidate.symbol.length, + ); + + const taken: AdvancedIndex[] = []; + const used = { core: 0, pro: 0 }; + const limits = { core: MAX_CORE_FEED_RESULTS, pro: MAX_PRO_FEED_RESULTS }; + for (const { candidate } of scored) { + if (used[candidate.family] >= limits[candidate.family]) continue; + used[candidate.family] += 1; + taken.push(candidate.index); + if (used.core >= limits.core && used.pro >= limits.pro) break; + } + return taken; } -export const { GET } = createSearchAPI("advanced", { - indexes: async () => { +const server = initAdvancedSearch({ + indexes: () => { const staticPages = source.getPages().map((page) => ({ - title: page.data.title, description: page.data.description, - url: page.url, id: page.url, structuredData: page.data.structuredData, + title: page.data.title, + url: page.url, })) as AdvancedIndex[]; - // Added these two functions to get the price feeds from the Hermes and Pro APIs - const [hermesFeeds, lazerFeeds] = await Promise.all([ - getHermesFeeds(), - getLazerFeeds(), - ]); - - // Combine the static pages, Hermes feeds, and Pro feeds - return [...staticPages, ...hermesFeeds, ...lazerFeeds]; + return [ + ...staticPages, + ...CORE_INDEXES.map(({ index }) => index), + ...PRO_INDEXES.map(({ index }) => index), + ]; }, }); + +const QUOTAS: Record = { + core: MAX_CORE_FEED_RESULTS, + docs: MAX_DOC_RESULTS, + pro: MAX_PRO_FEED_RESULTS, +}; + +function familyOf(result: SortedResult): Family { + if (result.url.startsWith(CORE_FEED_PATH)) return "core"; + if (result.url.startsWith(PRO_FEED_PATH)) return "pro"; + return "docs"; +} + +/** + * Trim the full match set to `MAX_RESULTS`, giving each family its reserved + * slots first and then backfilling any unused slots with the next + * highest-ranked results so a docs-only or feed-only query still fills the list. + */ +function applyQuotas(results: SortedResult[]): SortedResult[] { + const picked: SortedResult[] = []; + const overflow: SortedResult[] = []; + const used: Record = { core: 0, docs: 0, pro: 0 }; + + for (const result of results) { + if (picked.length >= MAX_RESULTS) break; + const family = familyOf(result); + if (used[family] < QUOTAS[family]) { + used[family] += 1; + picked.push(result); + } else if (overflow.length < MAX_RESULTS) { + overflow.push(result); + } + } + + return [...picked, ...overflow].slice(0, MAX_RESULTS); +} + +export async function GET(request: Request): Promise { + const params = new URL(request.url).searchParams; + const query = params.get("query"); + + if (!query) return Response.json([]); + + const tag = params.get("tag"); + const locale = params.get("locale"); + + const highlighter = createContentHighlighter(query); + const exact: SortedResult[] = directFeedMatches(query).map((index) => ({ + content: index.title, + contentWithHighlights: highlighter.highlight(index.title), + id: index.id, + type: "page", + url: index.url, + })); + + const results = await server.search(query, { + ...(tag === null ? {} : { tag: tag.split(",") }), + ...(locale === null ? {} : { locale }), + }); + + // A feed surfaced by the exact-match pass must not appear again as an Orama + // page row or as one of its own description snippets; both share the feed URL. + const seen = new Set(exact.map((result) => result.url)); + + return Response.json( + applyQuotas([...exact, ...results.filter((r) => !seen.has(r.url))]), + ); +} diff --git a/apps/developer-hub/turbo.json b/apps/developer-hub/turbo.json index 05e06fe064..84c0fdbb8d 100644 --- a/apps/developer-hub/turbo.json +++ b/apps/developer-hub/turbo.json @@ -3,7 +3,20 @@ "extends": ["//"], "tasks": { "build": { - "dependsOn": ["generate:docs", "generate:changelog"], + "dependsOn": [ + "generate:docs", + "generate:changelog", + "generate:price-feeds" + ], + "inputs": [ + "$TURBO_DEFAULT$", + "!README.md", + "!**/*.test.*", + "!jest.config.js", + "!eslint.config.js", + "!vercel.json", + "src/generated/**" + ], "env": [ "VERCEL_ENV", "COOKIE_SIGNING_SECRET", @@ -49,6 +62,11 @@ "dependsOn": ["//#install:modules", "^build"], "outputs": ["content/docs/api-reference/**"] }, + "generate:price-feeds": { + "cache": false, + "dependsOn": ["//#install:modules"], + "outputs": ["src/generated/**"] + }, "start:prod": { "dependsOn": ["//#install:modules", "build"] },