diff --git a/apps/scraper/src/fixtures/missouri-sos/calendar.html b/apps/scraper/src/fixtures/missouri-sos/calendar.html new file mode 100644 index 00000000..fb7cc28f --- /dev/null +++ b/apps/scraper/src/fixtures/missouri-sos/calendar.html @@ -0,0 +1,36 @@ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Official Election DayStyleRegistrationFilingCloseFinal Certification
April 7, 2026General Municipal Election DayJanuary 27, 2026
August 4, 2026Primary ElectionMay 26, 2026
November 3, 2026General ElectionAugust 25, 2026
+
diff --git a/apps/scraper/src/fixtures/missouri-sos/candidate-index.html b/apps/scraper/src/fixtures/missouri-sos/candidate-index.html new file mode 100644 index 00000000..fc3aa45e --- /dev/null +++ b/apps/scraper/src/fixtures/missouri-sos/candidate-index.html @@ -0,0 +1,11 @@ +
+

2026 Primary Election

+ State Senator - District 2 + State Representative - District 11 +
diff --git a/apps/scraper/src/fixtures/missouri-sos/candidates.html b/apps/scraper/src/fixtures/missouri-sos/candidates.html new file mode 100644 index 00000000..d53180d3 --- /dev/null +++ b/apps/scraper/src/fixtures/missouri-sos/candidates.html @@ -0,0 +1,49 @@ +
+

Certified Candidate List

+

2026 Primary Election

+

State Senator - District 2

+ + + + + + + + + + + + + + +
+ Republican +
NameMailing Address
Alice First101 Secret Street
Bob Second202 Private Avenue
+ + + + + + + + + + +
+ Democratic +
NameMailing Address
Carol Third303 Hidden Road
+

State Representative - District 11

+ + + + + + + + + + +
+ Libertarian +
NameMailing Address
Dana Fourth404 Confidential Lane
+
diff --git a/apps/scraper/src/fixtures/missouri-sos/historical-results-layout.html b/apps/scraper/src/fixtures/missouri-sos/historical-results-layout.html new file mode 100644 index 00000000..3f6250f1 --- /dev/null +++ b/apps/scraper/src/fixtures/missouri-sos/historical-results-layout.html @@ -0,0 +1,27 @@ + +
+

Last Updated: November 4, 2024 11:45 PM | Unofficial Results

+ + + + + + + + + + + + + + + + + + + + +
+ Governor +
CandidatePartyTotal VotesPercent
Example Candidate ARepublican1,25052.1%
Example Candidate BDemocratic1,14947.9%
+
diff --git a/apps/scraper/src/fixtures/missouri-sos/measures.html b/apps/scraper/src/fixtures/missouri-sos/measures.html new file mode 100644 index 00000000..832eecb2 --- /dev/null +++ b/apps/scraper/src/fixtures/missouri-sos/measures.html @@ -0,0 +1,56 @@ +
+

+ The following ballot measures have been certified for the + August 4, 2026 primary election. +

+

+ Official Ballot Title
Amendment 1 +

+

+ full text +

+

+ certificate +

+

Official Ballot Title:

+
+

Shall the Missouri Constitution be amended for this fixture?

+

State entities estimate no costs or savings.

+
+

Fair Ballot Language:

+
+

A “yes” vote will amend the Constitution.

+

A “no” vote will not amend it.

+
+

+ The following ballot measures have been certified for the + November 3, 2026 general election. +

+

+ Official Ballot Title
Amendment 3 +

+

+ full text +

+

+ certificate +

+

Official Ballot Title:

+
+

Shall another certified proposal be adopted?

+

The fiscal impact is unknown.

+
+

Fair Ballot Language:

+
+

A “yes” vote adopts the proposal.

+

A “no” vote rejects the proposal.

+
+
diff --git a/apps/scraper/src/fixtures/missouri-sos/showmo-unavailable.html b/apps/scraper/src/fixtures/missouri-sos/showmo-unavailable.html new file mode 100644 index 00000000..7e3e9952 --- /dev/null +++ b/apps/scraper/src/fixtures/missouri-sos/showmo-unavailable.html @@ -0,0 +1,5 @@ +
+

ShowMO Votes

+

Election resources and voter information.

+ Previous Election Results +
diff --git a/apps/scraper/src/fixtures/missouri-sos/withdrawals.html b/apps/scraper/src/fixtures/missouri-sos/withdrawals.html new file mode 100644 index 00000000..3f2e270b --- /dev/null +++ b/apps/scraper/src/fixtures/missouri-sos/withdrawals.html @@ -0,0 +1,22 @@ +
+ + + + + + + + + + + + + + + + + + + +
OfficeCandidateReasonRemoval Date
State Representative - District 11Kim Miller (Republican)
1816 Private Street		Withdrawn6/9/2026
1:02 PM
State Senator - District 2Robin Removed (Democratic)
PO Box 999		Disqualified5/19/2026
+
diff --git a/apps/scraper/src/scrapers/disabled/README.md b/apps/scraper/src/scrapers/disabled/README.md index d43ca5fb..676f5371 100644 --- a/apps/scraper/src/scrapers/disabled/README.md +++ b/apps/scraper/src/scrapers/disabled/README.md @@ -45,6 +45,16 @@ and cache the data," and its `getCached*` getters are imported nowhere. To revive: have the scraper persist to the database (a real table the API reads), add a reader in `@acme/api`, then re-register its implementation and contract. +## `missouri-sos.ts` + +Implements current-cycle Missouri candidate, ballot-measure, and election-result +snapshot ingestion. Disabled because the snapshot data is not yet connected to +a configured production database or a product surface. + +To revive it: provision the election snapshot storage and product integration, +then move the implementation/config/parser back to `scrapers/`, register +the implementation and active contract, and restore the Missouri source-limit +environment entries. ## `kansas-city-council.ts` diff --git a/apps/scraper/src/scrapers/disabled/missouri-sos-parsers.ts b/apps/scraper/src/scrapers/disabled/missouri-sos-parsers.ts new file mode 100644 index 00000000..716bfbb6 --- /dev/null +++ b/apps/scraper/src/scrapers/disabled/missouri-sos-parsers.ts @@ -0,0 +1,456 @@ +import * as cheerio from "cheerio"; + +import type { + MissouriBallotMeasure, + MissouriCandidate, + MissouriCitation, + MissouriElection, + MissouriResultContest, +} from "@acme/api/lib/missouri-election-data"; + +export const MISSOURI_STRUCTURE_VERSION = "mo-sos-current-election-v1"; +export const MISSOURI_CANDIDATES_URL = "https://s1.sos.mo.gov/candidatesonweb/"; +export const MISSOURI_MEASURES_URL = + "https://www.sos.mo.gov/petitions/2026BallotMeasures"; +export const MISSOURI_RESULTS_URL = + "https://www.sos.mo.gov/elections/showmovotes"; +export const MISSOURI_CALENDAR_URL = + "https://www.sos.mo.gov/elections/calendar/2026cal"; + +export interface MissouriOfficeReference { + office: string; + url: string; +} + +export interface MissouriCandidateDiscovery { + electionCode: string; + electionName: string; + candidatesUrl: string; + offices: MissouriOfficeReference[]; + withdrawalsUrl: string; +} + +function compact(value: string): string { + return value + .replace(/\u00a0/g, " ") + .replace(/\s+/g, " ") + .trim(); +} + +function absoluteUrl(href: string, baseUrl: string): string { + return new URL(href, baseUrl).toString(); +} + +function isoDate(value: string): string | null { + const match = + /\b(January|February|March|April|May|June|July|August|September|October|November|December)\s+(\d{1,2}),\s*(2026)\b/i.exec( + value, + ) ?? /\b(\d{1,2})\/(\d{1,2})\/(2026)\b/.exec(value); + if (!match) return null; + if (/^\d/.test(match[1] ?? "")) { + return `${match[3]}-${String(match[1]).padStart(2, "0")}-${String( + match[2], + ).padStart(2, "0")}`; + } + const month = + [ + "january", + "february", + "march", + "april", + "may", + "june", + "july", + "august", + "september", + "october", + "november", + "december", + ].indexOf((match[1] ?? "").toLowerCase()) + 1; + return `${match[3]}-${String(month).padStart(2, "0")}-${String( + match[2], + ).padStart(2, "0")}`; +} + +function citation(label: string, sourceUrl: string): MissouriCitation { + return { label, sourceUrl }; +} + +export function parseMissouriCalendar( + html: string, + asOf = new Date(), + sourceUrl = MISSOURI_CALENDAR_URL, + preferredType?: MissouriElection["type"], +): MissouriElection { + const $ = cheerio.load(html); + const elections: MissouriElection[] = []; + $("table.cal tr").each((_index, row) => { + const cells = $(row) + .find("td") + .map((_cellIndex, cell) => compact($(cell).text())) + .get(); + if (cells.length < 6) return; + const electionDate = isoDate(cells[0] ?? ""); + const finalCertificationDate = isoDate(cells[5] ?? "") ?? undefined; + const style = cells[1] ?? ""; + const type = /primary election/i.test(style) + ? "primary" + : /general election/i.test(style) && !/municipal/i.test(style) + ? "general" + : null; + if (!electionDate || !type) return; + elections.push({ + name: `2026 ${type === "primary" ? "Primary" : "General"} Election`, + type, + electionDate, + finalCertificationDate, + citation: citation("2026 Missouri election calendar", sourceUrl), + }); + }); + if (!elections.length) { + throw new Error( + "Missouri calendar contained no 2026 primary/general election", + ); + } + const preferred = preferredType + ? elections.find((election) => election.type === preferredType) + : undefined; + if (preferred) return preferred; + const today = asOf.toISOString().slice(0, 10); + return ( + elections.find((election) => election.electionDate >= today) ?? + elections.at(-1)! + ); +} + +export function parseMissouriCandidateDiscovery( + html: string, + baseUrl = MISSOURI_CANDIDATES_URL, +): MissouriCandidateDiscovery { + const $ = cheerio.load(html); + const electionName = compact($("#main h2, #right h2").first().text()); + const references = new Map(); + let electionCode = ""; + $("a[href*='DisplayCandidatesPlacement.aspx']").each((_index, anchor) => { + const href = $(anchor).attr("href"); + const office = compact($(anchor).text()); + if (!href || !office) return; + const url = new URL(href, baseUrl); + const code = url.searchParams.get("ElectionCode"); + const officeCode = url.searchParams.get("OfficeCode"); + if (!code || !officeCode) return; + electionCode ||= code; + references.set(url.toString(), { office, url: url.toString() }); + }); + if (!electionCode || !references.size || !/2026/i.test(electionName)) { + throw new Error( + "Candidate filing page did not expose a certified 2026 election", + ); + } + return { + electionCode, + electionName, + candidatesUrl: absoluteUrl( + `DisplayCandidatesPlacement.aspx?ElectionCode=${encodeURIComponent(electionCode)}`, + baseUrl, + ), + offices: [...references.values()], + withdrawalsUrl: absoluteUrl( + `CandidatesRemoved.aspx?ElectionCode=${encodeURIComponent(electionCode)}`, + baseUrl, + ), + }; +} + +function officeParts(value: string): { office: string; district?: string } { + const normalized = compact(value); + const match = /^(.*?)\s*-?\s+District\s+(\d+)$/i.exec(normalized); + return match + ? { office: compact(match[1] ?? normalized), district: match[2] } + : { office: normalized }; +} + +export function parseMissouriCandidateOffice( + html: string, + sourceUrl: string, +): MissouriCandidate[] { + const $ = cheerio.load(html); + if (!/Certified Candidate List/i.test($("#main h1").text())) { + throw new Error("Candidate office page was not a certified candidate list"); + } + const candidates: MissouriCandidate[] = []; + $("#main table").each((_tableIndex, table) => { + const party = compact($(table).find("caption").text()); + const parsedOffice = officeParts($(table).prevAll("h3").first().text()); + if (!parsedOffice.office) return; + $(table) + .find("tr") + .slice(1) + .each((rowIndex, row) => { + // Deliberately select only the public name cell. Address, filing date, + // and every other candidate contact/location field are ignored. + const name = compact($(row).find("td.NameCol").text()); + if (!name || !party) return; + candidates.push({ + ...parsedOffice, + party, + name, + ballotOrder: rowIndex + 1, + status: "certified", + citation: citation( + "Certified candidate list in ballot order", + sourceUrl, + ), + }); + }); + }); + return candidates; +} + +export function parseMissouriWithdrawals( + html: string, + sourceUrl: string, +): MissouriCandidate[] { + const $ = cheerio.load(html); + const candidates: MissouriCandidate[] = []; + $("table[title='Removed Candidates'] tr") + .slice(1) + .each((_index, row) => { + const cells = $(row).find("td"); + if (cells.length < 4) return; + const parsedOffice = officeParts($(cells[0]).text()); + // The bold span contains only name + party; the following address node is + // intentionally never read. + const identity = compact($(cells[1]).find("span").first().text()); + const match = /^(.*?)\s*\(([^()]+)\)$/.exec(identity); + const reason = compact($(cells[2]).text()); + const dateCell = $(cells[3]).clone(); + dateCell.find("br").replaceWith(" "); + const withdrawalDate = isoDate(compact(dateCell.text())) ?? undefined; + if (!match?.[1] || !match[2] || !reason) return; + candidates.push({ + ...parsedOffice, + name: compact(match[1]), + party: compact(match[2]), + ballotOrder: candidates.length + 1, + status: /withdraw/i.test(reason) ? "withdrawn" : "removed", + withdrawalReason: reason, + withdrawalDate, + citation: citation("Certified withdrawn/removed candidates", sourceUrl), + }); + }); + return candidates; +} + +interface MeasureDraft { + officialTitle: string; + electionDate: string; + fullTextUrl?: string; + certificateUrl?: string; + officialParagraphs: string[]; + fairParagraphs: string[]; +} + +export function parseMissouriBallotMeasures( + html: string, + sourceUrl = MISSOURI_MEASURES_URL, +): MissouriBallotMeasure[] { + const $ = cheerio.load(html); + let electionDate = ""; + let mode: "official" | "fair" | null = null; + let draft: MeasureDraft | null = null; + const drafts: MeasureDraft[] = []; + + $("#main") + .find("h2, p") + .each((_index, element) => { + const tag = element.tagName.toLowerCase(); + const text = compact($(element).text()); + if (tag === "h2" && /certified/i.test(text)) { + electionDate = isoDate(text) ?? electionDate; + return; + } + if (tag !== "p" || !text) return; + const title = + /^Official Ballot Title\s*(Amendment\s+\d+|Proposition\s+\w+)/i.exec( + text, + ); + if (title?.[1]) { + draft = { + officialTitle: compact(title[1]), + electionDate, + officialParagraphs: [], + fairParagraphs: [], + }; + drafts.push(draft); + mode = null; + return; + } + if (!draft) return; + const full = $(element).find("a[title*='full text' i]").attr("href"); + if (full) draft.fullTextUrl = absoluteUrl(full, sourceUrl); + const certificate = $(element) + .find("a[title*='Certificate of Official Ballot Title' i]") + .attr("href"); + if (certificate) + draft.certificateUrl = absoluteUrl(certificate, sourceUrl); + if (/^Official Ballot Title:?$/i.test(text)) { + mode = "official"; + return; + } + if (/^Fair Ballot Language:?$/i.test(text)) { + mode = "fair"; + return; + } + if (mode === "official") draft.officialParagraphs.push(text); + if (mode === "fair") draft.fairParagraphs.push(text); + }); + + return drafts.flatMap((item) => { + const fiscalStatement = item.officialParagraphs + .filter((paragraph) => + /(?:cost|saving|revenue|fiscal|tax|financial impact)/i.test(paragraph), + ) + .at(-1); + if ( + !item.electionDate || + !item.fullTextUrl || + !item.certificateUrl || + !item.officialParagraphs.length || + !item.fairParagraphs.length || + !fiscalStatement + ) { + return []; + } + return [ + { + officialTitle: item.officialTitle, + officialBallotLanguage: item.officialParagraphs.join("\n\n"), + fairBallotLanguage: item.fairParagraphs.join("\n\n"), + fiscalStatement, + electionDate: item.electionDate, + fullTextUrl: item.fullTextUrl, + certificateUrl: item.certificateUrl, + certificationStatus: "certified" as const, + citation: citation("Certified 2026 ballot measure", sourceUrl), + }, + ]; + }); +} + +export function discoverMissouriResultsUrl( + html: string, + activeElection: MissouriElection, + baseUrl = MISSOURI_RESULTS_URL, +): string | null { + const $ = cheerio.load(html); + const dateWords = new Date( + `${activeElection.electionDate}T12:00:00Z`, + ).toLocaleDateString("en-US", { + month: "long", + day: "numeric", + year: "numeric", + timeZone: "UTC", + }); + let found: string | null = null; + $("a[href]").each((_index, anchor) => { + if (found) return; + const href = $(anchor).attr("href"); + const text = compact($(anchor).text()); + const imageAlt = compact($(anchor).find("img").attr("alt") ?? ""); + const context = compact($(anchor).parent().text()); + const signal = `${text} ${imageAlt} ${context} ${href}`; + if (!href || !/result/i.test(signal)) return; + if (!/2026/.test(signal)) return; + if ( + !new RegExp(activeElection.type, "i").test(signal) && + !signal.includes(dateWords) + ) + return; + found = absoluteUrl(href, baseUrl); + }); + return found; +} + +function numeric(value: string): number { + const parsed = Number.parseInt(value.replace(/[^\d-]/g, ""), 10); + return Number.isFinite(parsed) ? parsed : 0; +} + +export function parseMissouriResults( + html: string, + sourceUrl: string, +): { contests: MissouriResultContest[]; updatedAt?: string } { + const $ = cheerio.load(html); + const contests: MissouriResultContest[] = []; + $("table").each((_tableIndex, table) => { + const headers = $(table) + .find("tr") + .first() + .find("th") + .map((_index, header) => compact($(header).text()).toLowerCase()) + .get(); + const nameIndex = headers.findIndex((header) => + /candidate|choice|option/.test(header), + ); + const votesIndex = headers.findIndex((header) => + /votes|total/.test(header), + ); + if (nameIndex < 0 || votesIndex < 0) return; + const partyIndex = headers.findIndex((header) => /party/.test(header)); + const percentIndex = headers.findIndex((header) => + /percent|%/.test(header), + ); + const contest = + compact($(table).find("caption").text()) || + compact($(table).prevAll("h2, h3, h4").first().text()); + if (!contest) return; + const choices: MissouriResultContest["choices"] = []; + $(table) + .find("tr") + .slice(1) + .each((_rowIndex, row) => { + const cells = $(row).find("td"); + const name = compact($(cells[nameIndex]).text()); + if (!name) return; + const percentText = + percentIndex >= 0 ? compact($(cells[percentIndex]).text()) : ""; + const parsedPercent = percentText + ? Number.parseFloat(percentText) + : undefined; + choices.push({ + name, + party: + partyIndex >= 0 + ? compact($(cells[partyIndex]).text()) || undefined + : undefined, + votes: numeric(compact($(cells[votesIndex]).text())), + percent: + parsedPercent !== undefined && Number.isFinite(parsedPercent) + ? parsedPercent + : undefined, + }); + }); + if (!choices.length) return; + const district = /District\s+(\d+)/i.exec(contest)?.[1]; + contests.push({ + contest, + district, + choices, + totalVotes: choices.reduce((sum, choice) => sum + choice.votes, 0), + reportingStatus: + compact($(table).attr("data-reporting-status") ?? "") || undefined, + citation: citation("Official Missouri SOS election results", sourceUrl), + }); + }); + if (!contests.length) { + throw new Error( + "Official results page contained no parseable contest tables", + ); + } + const pageText = compact($("body").text()); + const updatedAt = /(?:Last Updated|Updated)\s*:?\s*([^|]+?)(?:\||$)/i.exec( + pageText, + )?.[1]; + return { contests, updatedAt: updatedAt ? compact(updatedAt) : undefined }; +} diff --git a/apps/scraper/src/scrapers/disabled/missouri-sos.config.ts b/apps/scraper/src/scrapers/disabled/missouri-sos.config.ts new file mode 100644 index 00000000..27bff38f --- /dev/null +++ b/apps/scraper/src/scrapers/disabled/missouri-sos.config.ts @@ -0,0 +1,11 @@ +import type { ScraperEnvContract } from "@acme/env"; + +export const missouriSosConfig = { + id: "missouri-sos", + name: "Missouri SOS current election cycle", + source: "Missouri Secretary of State", + environment: { + required: ["POSTGRES_URL"], + optional: ["MISSOURI_SOS_MAX_ITEMS"], + }, +} as const satisfies ScraperEnvContract; diff --git a/apps/scraper/src/scrapers/disabled/missouri-sos.ts b/apps/scraper/src/scrapers/disabled/missouri-sos.ts new file mode 100644 index 00000000..3a59fcb9 --- /dev/null +++ b/apps/scraper/src/scrapers/disabled/missouri-sos.ts @@ -0,0 +1,250 @@ +/** Current-cycle-only Missouri SOS candidates, measures, and results. */ + +import { createHash } from "node:crypto"; +import { and, eq, ne } from "drizzle-orm"; + +import type { + MissouriResults, + MissouriSnapshotData, +} from "@acme/api/lib/missouri-election-data"; +import { + MISSOURI_CURRENT_SCOPE, + MISSOURI_CYCLE_YEAR, + MISSOURI_SOS_PROVIDER, +} from "@acme/api/lib/missouri-election-data"; +import { db } from "@acme/db/client"; +import { ElectionSourceSnapshot } from "@acme/db/schema"; + +import type { Scraper } from "../../utils/types.js"; +import { getItemLimit } from "../../utils/concurrency.js"; +import { fetchWithRetry } from "../../utils/fetch.js"; +import { createLogger } from "../../utils/log.js"; +import { + discoverMissouriResultsUrl, + MISSOURI_CALENDAR_URL, + MISSOURI_CANDIDATES_URL, + MISSOURI_MEASURES_URL, + MISSOURI_RESULTS_URL, + MISSOURI_STRUCTURE_VERSION, + parseMissouriBallotMeasures, + parseMissouriCalendar, + parseMissouriCandidateDiscovery, + parseMissouriCandidateOffice, + parseMissouriResults, + parseMissouriWithdrawals, +} from "./missouri-sos-parsers.js"; +import { missouriSosConfig } from "./missouri-sos.config.js"; + +const logger = createLogger("missouri-sos"); +const USER_AGENT = + "Mozilla/5.0 (compatible; BillionCivicBot/1.0; +https://billion.app)"; + +function sha256(value: string): string { + return createHash("sha256").update(value).digest("hex"); +} + +async function fetchOfficialHtml(url: string): Promise { + const response = await fetchWithRetry(url, { + headers: { Accept: "text/html", "User-Agent": USER_AGENT }, + timeoutMs: 45_000, + }); + return response.text(); +} + +async function upsertSnapshot( + data: MissouriSnapshotData, + diagnostics: string[], + sourceUrls: string[], + sourceVersion: string, +): Promise { + const canonicalData = JSON.stringify(data); + const contentHash = sha256(canonicalData); + const now = new Date(); + await db + .insert(ElectionSourceSnapshot) + .values({ + jurisdiction: "MO", + cycleYear: MISSOURI_CYCLE_YEAR, + provider: MISSOURI_SOS_PROVIDER, + scope: MISSOURI_CURRENT_SCOPE, + sourceVersion, + contentHash, + data: data as unknown as Record, + diagnostics, + sourceUrls, + fetchedAt: now, + }) + .onConflictDoUpdate({ + target: [ + ElectionSourceSnapshot.jurisdiction, + ElectionSourceSnapshot.cycleYear, + ElectionSourceSnapshot.provider, + ElectionSourceSnapshot.scope, + ], + set: { + sourceVersion, + contentHash, + data: data as unknown as Record, + diagnostics, + sourceUrls, + fetchedAt: now, + }, + }); + + // This invariant prevents any stale scope introduced outside this scraper + // from being mistaken for the supported current-cycle reader. + await db + .delete(ElectionSourceSnapshot) + .where( + and( + eq(ElectionSourceSnapshot.jurisdiction, "MO"), + eq(ElectionSourceSnapshot.provider, MISSOURI_SOS_PROVIDER), + eq(ElectionSourceSnapshot.scope, MISSOURI_CURRENT_SCOPE), + ne(ElectionSourceSnapshot.cycleYear, MISSOURI_CYCLE_YEAR), + ), + ); +} + +export async function scrapeMissouriSos(maxItems = 1000): Promise { + logger.info("Discovering official 2026 Missouri election sources…"); + const limit = getItemLimit(); + const [calendarHtml, candidateIndexHtml, measuresHtml, resultsEntryHtml] = + await Promise.all([ + limit(() => fetchOfficialHtml(MISSOURI_CALENDAR_URL)), + limit(() => fetchOfficialHtml(MISSOURI_CANDIDATES_URL)), + limit(() => fetchOfficialHtml(MISSOURI_MEASURES_URL)), + limit(() => fetchOfficialHtml(MISSOURI_RESULTS_URL)), + ]); + + const discovery = parseMissouriCandidateDiscovery(candidateIndexHtml); + const candidateElectionType = /general/i.test(discovery.electionName) + ? "general" + : /primary/i.test(discovery.electionName) + ? "primary" + : undefined; + const activeElection = parseMissouriCalendar( + calendarHtml, + new Date(), + MISSOURI_CALENDAR_URL, + candidateElectionType, + ); + if (!new RegExp(activeElection.type, "i").test(discovery.electionName)) { + throw new Error( + `Candidate system exposes ${discovery.electionName}, not active ${activeElection.name}`, + ); + } + + // The official cumulative endpoint contains every office and party in ballot + // order. It avoids a high-volume crawl of the individual office links. + const [candidateHtml, withdrawalHtml] = await Promise.all([ + limit(() => fetchOfficialHtml(discovery.candidatesUrl)), + limit(() => fetchOfficialHtml(discovery.withdrawalsUrl)), + ]); + const allCandidates = [ + ...parseMissouriCandidateOffice(candidateHtml, discovery.candidatesUrl), + ...parseMissouriWithdrawals(withdrawalHtml, discovery.withdrawalsUrl), + ]; + const candidates = allCandidates.slice(0, maxItems); + const ballotMeasures = parseMissouriBallotMeasures(measuresHtml); + const diagnostics: string[] = []; + if (candidates.length < allCandidates.length) { + diagnostics.push( + `MISSOURI_SOS_MAX_ITEMS limited candidates to ${candidates.length} of ${allCandidates.length}`, + ); + } + if (!ballotMeasures.length) { + throw new Error("Certified 2026 ballot-measure page contained no measures"); + } + + const resultUrl = discoverMissouriResultsUrl( + resultsEntryHtml, + activeElection, + ); + let results: MissouriResults; + if (!resultUrl) { + const diagnostic = `Official ${activeElection.name} results are not yet available from ShowMO Votes`; + diagnostics.push(diagnostic); + results = { + availability: "unavailable", + diagnostic, + citation: { + label: "ShowMO Votes results entry point", + sourceUrl: MISSOURI_RESULTS_URL, + }, + }; + logger.info(diagnostic); + } else { + try { + const resultHtml = await limit(() => fetchOfficialHtml(resultUrl)); + const parsed = parseMissouriResults(resultHtml, resultUrl); + results = { + availability: "available", + electionDate: activeElection.electionDate, + ...parsed, + citation: { + label: "Official Missouri SOS election results", + sourceUrl: resultUrl, + }, + }; + } catch (error) { + const diagnostic = `Official results link was discovered but results are not parseable yet: ${ + error instanceof Error ? error.message : String(error) + }`; + diagnostics.push(diagnostic); + results = { + availability: "unavailable", + diagnostic, + citation: { + label: "ShowMO Votes results entry point", + sourceUrl: MISSOURI_RESULTS_URL, + }, + }; + logger.warn(diagnostic); + } + } + + const sourceUrls = [ + MISSOURI_CANDIDATES_URL, + discovery.candidatesUrl, + discovery.withdrawalsUrl, + MISSOURI_MEASURES_URL, + MISSOURI_RESULTS_URL, + MISSOURI_CALENDAR_URL, + ...(resultUrl ? [resultUrl] : []), + ]; + const data: MissouriSnapshotData = { + cycleYear: MISSOURI_CYCLE_YEAR, + activeElection, + candidates, + ballotMeasures, + results, + citations: sourceUrls.map((sourceUrl) => ({ + label: "Missouri Secretary of State official source", + sourceUrl, + })), + }; + const sourceVersion = `${MISSOURI_STRUCTURE_VERSION}:${sha256( + JSON.stringify(data), + ).slice(0, 24)}`; + await upsertSnapshot( + data, + diagnostics, + [...new Set(sourceUrls)], + sourceVersion, + ); + logger.success( + `Missouri SOS: persisted ${candidates.length} candidates, ${ballotMeasures.length} measures, and ${ + results.availability === "available" + ? `${results.contests.length} result contests` + : "an unavailable-results diagnostic" + }.`, + ); +} + +export const missouriSos: Scraper = { + ...missouriSosConfig, + scrape: (options) => + scrapeMissouriSos( + (options?.maxItems ?? Number(process.env.MISSOURI_SOS_MAX_ITEMS)) || 1000, + ), +}; diff --git a/apps/scraper/src/scrapers/missouri-sos-parsers.test.ts b/apps/scraper/src/scrapers/missouri-sos-parsers.test.ts new file mode 100644 index 00000000..6db41233 --- /dev/null +++ b/apps/scraper/src/scrapers/missouri-sos-parsers.test.ts @@ -0,0 +1,157 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import test from "node:test"; + +import { + discoverMissouriResultsUrl, + parseMissouriBallotMeasures, + parseMissouriCalendar, + parseMissouriCandidateDiscovery, + parseMissouriCandidateOffice, + parseMissouriResults, + parseMissouriWithdrawals, +} from "./disabled/missouri-sos-parsers.js"; + +const fixture = (name: string) => + readFile( + new URL(`../fixtures/missouri-sos/${name}`, import.meta.url), + "utf8", + ); + +void test("discovers the active primary and dynamic certified candidate endpoints", async () => { + const active = parseMissouriCalendar( + await fixture("calendar.html"), + new Date("2026-07-22T12:00:00Z"), + ); + assert.equal(active.type, "primary"); + assert.equal(active.electionDate, "2026-08-04"); + assert.equal(active.finalCertificationDate, "2026-05-26"); + assert.equal( + parseMissouriCalendar( + await fixture("calendar.html"), + new Date("2026-08-10T12:00:00Z"), + undefined, + "primary", + ).type, + "primary", + ); + + const discovery = parseMissouriCandidateDiscovery( + await fixture("candidate-index.html"), + ); + assert.equal(discovery.electionCode, "750006905"); + assert.equal(discovery.offices.length, 2); + assert.match(discovery.candidatesUrl, /ElectionCode=750006905/); + assert.match(discovery.withdrawalsUrl, /CandidatesRemoved/); +}); + +void test("parses ballot order, parties, districts, and excludes candidate addresses", async () => { + const candidates = parseMissouriCandidateOffice( + await fixture("candidates.html"), + "https://s1.sos.mo.gov/candidatesonweb/cumulative", + ); + assert.deepEqual( + candidates.map(({ name, party, office, district, ballotOrder }) => ({ + name, + party, + office, + district, + ballotOrder, + })), + [ + { + name: "Alice First", + party: "Republican", + office: "State Senator", + district: "2", + ballotOrder: 1, + }, + { + name: "Bob Second", + party: "Republican", + office: "State Senator", + district: "2", + ballotOrder: 2, + }, + { + name: "Carol Third", + party: "Democratic", + office: "State Senator", + district: "2", + ballotOrder: 1, + }, + { + name: "Dana Fourth", + party: "Libertarian", + office: "State Representative", + district: "11", + ballotOrder: 1, + }, + ], + ); + assert.doesNotMatch( + JSON.stringify(candidates), + /Secret|Private|Hidden|Confidential/, + ); +}); + +void test("parses withdrawals without retaining their address nodes", async () => { + const candidates = parseMissouriWithdrawals( + await fixture("withdrawals.html"), + "https://s1.sos.mo.gov/candidatesonweb/withdrawals", + ); + assert.equal(candidates[0]?.status, "withdrawn"); + assert.equal(candidates[0]?.withdrawalDate, "2026-06-09"); + assert.equal(candidates[1]?.status, "removed"); + assert.doesNotMatch(JSON.stringify(candidates), /Private Street|PO Box/); +}); + +void test("parses only certified 2026 measures with language, fiscal text, and source files", async () => { + const measures = parseMissouriBallotMeasures(await fixture("measures.html")); + assert.equal(measures.length, 2); + assert.deepEqual( + measures.map((measure) => [measure.officialTitle, measure.electionDate]), + [ + ["Amendment 1", "2026-08-04"], + ["Amendment 3", "2026-11-03"], + ], + ); + assert.match(measures[0]?.officialBallotLanguage ?? "", /Constitution/); + assert.match(measures[0]?.fairBallotLanguage ?? "", /yes/); + assert.match(measures[0]?.fiscalStatement ?? "", /costs or savings/); + assert.match(measures[0]?.certificateUrl ?? "", /certificate-amendment-1/); +}); + +void test("fails soft when current results are absent and detects deterministic updates", async () => { + const active = parseMissouriCalendar( + await fixture("calendar.html"), + new Date("2026-07-22T12:00:00Z"), + ); + assert.equal( + discoverMissouriResultsUrl( + await fixture("showmo-unavailable.html"), + active, + ), + null, + ); + assert.equal( + discoverMissouriResultsUrl( + '
August 4, 2026 Primary Election Results
', + active, + ), + "https://www.sos.mo.gov/elections/2026-primary-results", + ); + + const layout = await fixture("historical-results-layout.html"); + const first = parseMissouriResults( + layout, + "https://example.test/fixture-only", + ); + const updated = parseMissouriResults( + layout.replace("1,250", "1,300").replace("11:45 PM", "11:55 PM"), + "https://example.test/fixture-only", + ); + assert.equal(first.contests[0]?.totalVotes, 2399); + assert.equal(updated.contests[0]?.totalVotes, 2449); + assert.notEqual(first.updatedAt, updated.updatedAt); +}); diff --git a/docs/missouri-sos-election-data.md b/docs/missouri-sos-election-data.md new file mode 100644 index 00000000..a13f3555 --- /dev/null +++ b/docs/missouri-sos-election-data.md @@ -0,0 +1,55 @@ +# Missouri SOS current-cycle election data + +> **Status: disabled.** The implementation is retained under +> `apps/scraper/src/scrapers/disabled`, but it is not registered or runnable +> until its database and product integration are provisioned. + +`missouri-sos` ingests only the 2026 Missouri election cycle from the four +official Secretary of State entry points named in issue #186. It does not offer +historical browsing and never requests or persists voter-level data. + +## Discovery and source boundaries + +- The [2026 election calendar](https://www.sos.mo.gov/elections/calendar/2026cal) + determines the active statewide primary or general election and its final + certification date. No archival result URL is constructed. +- The [certified candidate system](https://s1.sos.mo.gov/candidatesonweb/) + supplies the current election code. The scraper follows its cumulative + certified-list and withdrawn/removed-list links, which avoids crawling every + office page. Only office, district, party, candidate name, ballot order, and + withdrawal status are parsed. Address, filing-contact, and other location + cells are deliberately ignored. +- The [2026 certified ballot-measure page](https://www.sos.mo.gov/petitions/2026BallotMeasures) + supplies the election date, official title/language, fair ballot language, + fiscal statement, and links to the full text and signed certificate. Only + sections explicitly labeled certified on that page are accepted. +- The [ShowMO Votes entry point](https://www.sos.mo.gov/elections/showmovotes) + is searched for a link matching the active 2026 election. Before publication, + the snapshot is still stored with an explicit `availability: "unavailable"` + diagnostic. A historical-looking results fixture exists only to lock the + table parser; no historical URL appears in production code. + +All network calls use the shared retry/backoff client and shared runner +concurrency. The candidate system is read through its single cumulative page, +keeping request volume low. `MISSOURI_SOS_MAX_ITEMS` defaults to `1000` and may +be lowered for development runs; a truncation diagnostic is persisted whenever +the cap is reached. + +## Persistence and API + +Missouri reuses PR #181's provider-neutral `election_source_snapshot` table: + +- jurisdiction: `MO` +- cycle: `2026` +- provider: `mo-sos` +- scope: `current-2026` + +The normalized snapshot is SHA-256 hashed without fetch timestamps. Its natural +key is upserted, so unchanged runs remain one row while official corrections or +result updates replace the payload and checksum. Each record carries an +official-source citation, and the reader validates the JSON contract before +returning it. + +Consumers call the input-free public tRPC query +`civic.getMissouriCurrentElection`. The API has no year, election ID, or history +parameter, so it cannot expose a historical cycle accidentally. diff --git a/packages/api/package.json b/packages/api/package.json index 2da34124..0fa3748a 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -55,6 +55,10 @@ "types": "./dist/lib/texas-election-data.d.ts", "default": "./src/lib/texas-election-data.ts" }, + "./lib/missouri-election-data": { + "types": "./dist/lib/missouri-election-data.d.ts", + "default": "./src/lib/missouri-election-data.ts" + }, "./lib/measure-sources/disabled/vig-archive": { "types": "./dist/lib/measure-sources/disabled/vig-archive.d.ts", "default": "./src/lib/measure-sources/disabled/vig-archive.ts" diff --git a/packages/api/src/index.ts b/packages/api/src/index.ts index 97970162..10a27732 100644 --- a/packages/api/src/index.ts +++ b/packages/api/src/index.ts @@ -48,6 +48,18 @@ export { matchNcsbeName, } from "./lib/ncsbe-election-data"; +export type { + MissouriBallotMeasure, + MissouriCandidate, + MissouriCitation, + MissouriCurrentElectionData, + MissouriElection, + MissouriResultContest, + MissouriResults, + MissouriSnapshotData, +} from "./lib/missouri-election-data"; +export { getMissouriCurrentElectionData } from "./lib/missouri-election-data"; + // California Secretary of State live election-results feed export type { ElectionContestResult, diff --git a/packages/api/src/lib/missouri-election-data.test.ts b/packages/api/src/lib/missouri-election-data.test.ts new file mode 100644 index 00000000..2eddabcf --- /dev/null +++ b/packages/api/src/lib/missouri-election-data.test.ts @@ -0,0 +1,43 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + MISSOURI_CYCLE_YEAR, + missouriSnapshotDataSchema, +} from "./missouri-election-data"; + +const currentSnapshot = { + cycleYear: MISSOURI_CYCLE_YEAR, + activeElection: { + name: "2026 Primary Election", + type: "primary", + electionDate: "2026-08-04", + citation: { + label: "Official calendar", + sourceUrl: "https://www.sos.mo.gov/elections/calendar/2026cal", + }, + }, + candidates: [], + ballotMeasures: [], + results: { + availability: "unavailable", + diagnostic: "Official results are not yet available", + citation: { + label: "ShowMO Votes", + sourceUrl: "https://www.sos.mo.gov/elections/showmovotes", + }, + }, + citations: [], +} as const; + +void test("accepts the current-cycle unavailable-results handoff", () => { + const parsed = missouriSnapshotDataSchema.parse(currentSnapshot); + assert.equal(parsed.cycleYear, 2026); + assert.equal(parsed.results.availability, "unavailable"); +}); + +void test("rejects a historical cycle at the API boundary", () => { + assert.throws(() => + missouriSnapshotDataSchema.parse({ ...currentSnapshot, cycleYear: 2024 }), + ); +}); diff --git a/packages/api/src/lib/missouri-election-data.ts b/packages/api/src/lib/missouri-election-data.ts new file mode 100644 index 00000000..467405a3 --- /dev/null +++ b/packages/api/src/lib/missouri-election-data.ts @@ -0,0 +1,128 @@ +import { z } from "zod/v4"; + +import { and, desc, eq } from "@acme/db"; +import { db } from "@acme/db/client"; +import { ElectionSourceSnapshot } from "@acme/db/schema"; + +export const MISSOURI_CYCLE_YEAR = 2026; +export const MISSOURI_CURRENT_SCOPE = "current-2026"; +export const MISSOURI_SOS_PROVIDER = "mo-sos"; + +const citationSchema = z.object({ + label: z.string(), + sourceUrl: z.url(), +}); + +const electionSchema = z.object({ + name: z.string(), + type: z.enum(["primary", "general"]), + electionDate: z.iso.date(), + finalCertificationDate: z.iso.date().optional(), + citation: citationSchema, +}); + +const candidateSchema = z.object({ + office: z.string(), + district: z.string().optional(), + party: z.string(), + name: z.string(), + ballotOrder: z.int().positive(), + status: z.enum(["certified", "withdrawn", "removed"]), + withdrawalReason: z.string().optional(), + withdrawalDate: z.iso.date().optional(), + citation: citationSchema, +}); + +const measureSchema = z.object({ + officialTitle: z.string(), + officialBallotLanguage: z.string(), + fairBallotLanguage: z.string(), + fiscalStatement: z.string(), + electionDate: z.iso.date(), + fullTextUrl: z.url(), + certificateUrl: z.url(), + certificationStatus: z.literal("certified"), + citation: citationSchema, +}); + +const resultChoiceSchema = z.object({ + name: z.string(), + party: z.string().optional(), + votes: z.int().nonnegative(), + percent: z.number().min(0).max(100).optional(), +}); + +const resultContestSchema = z.object({ + contest: z.string(), + district: z.string().optional(), + choices: z.array(resultChoiceSchema), + totalVotes: z.int().nonnegative(), + reportingStatus: z.string().optional(), + citation: citationSchema, +}); + +const resultsSchema = z.discriminatedUnion("availability", [ + z.object({ + availability: z.literal("unavailable"), + diagnostic: z.string(), + citation: citationSchema, + }), + z.object({ + availability: z.literal("available"), + electionDate: z.iso.date(), + updatedAt: z.string().optional(), + contests: z.array(resultContestSchema), + citation: citationSchema, + }), +]); + +export const missouriSnapshotDataSchema = z.object({ + cycleYear: z.literal(MISSOURI_CYCLE_YEAR), + activeElection: electionSchema, + candidates: z.array(candidateSchema), + ballotMeasures: z.array(measureSchema), + results: resultsSchema, + citations: z.array(citationSchema), +}); + +export type MissouriCitation = z.infer; +export type MissouriElection = z.infer; +export type MissouriCandidate = z.infer; +export type MissouriBallotMeasure = z.infer; +export type MissouriResultContest = z.infer; +export type MissouriResults = z.infer; +export type MissouriSnapshotData = z.infer; + +export interface MissouriCurrentElectionData extends MissouriSnapshotData { + diagnostics: string[]; + sourceVersion: string; + contentHash: string; + fetchedAt: string; +} + +/** Read the sole public 2026 Missouri snapshot. No historical selector exists. */ +export async function getMissouriCurrentElectionData(): Promise { + const [row] = await db + .select() + .from(ElectionSourceSnapshot) + .where( + and( + eq(ElectionSourceSnapshot.jurisdiction, "MO"), + eq(ElectionSourceSnapshot.cycleYear, MISSOURI_CYCLE_YEAR), + eq(ElectionSourceSnapshot.provider, MISSOURI_SOS_PROVIDER), + eq(ElectionSourceSnapshot.scope, MISSOURI_CURRENT_SCOPE), + ), + ) + .orderBy(desc(ElectionSourceSnapshot.fetchedAt)) + .limit(1); + if (!row) return null; + + const parsed = missouriSnapshotDataSchema.parse(row.data); + return { + ...parsed, + diagnostics: row.diagnostics, + sourceVersion: row.sourceVersion, + contentHash: row.contentHash, + fetchedAt: row.fetchedAt.toISOString(), + }; +} diff --git a/packages/api/src/router/civic.ts b/packages/api/src/router/civic.ts index 39ffa34f..9f13c641 100644 --- a/packages/api/src/router/civic.ts +++ b/packages/api/src/router/civic.ts @@ -10,6 +10,7 @@ import { getVoterInfo, } from "../lib/civic"; import { getElectedOfficials } from "../lib/elected-officials"; +import { getMissouriCurrentElectionData } from "../lib/missouri-election-data"; import { getCurrentNcElectionData } from "../lib/ncsbe-election-data"; import { getTexasCurrentElectionData } from "../lib/texas-election-data"; import { publicProcedure } from "../trpc"; @@ -63,6 +64,22 @@ export const civicRouter = { } }), + /** Official, current-cycle-only Missouri SOS election data. */ + getMissouriCurrentElection: publicProcedure.query(async () => { + try { + return await getMissouriCurrentElectionData(); + } catch (error) { + throw new TRPCError({ + code: "INTERNAL_SERVER_ERROR", + message: + error instanceof Error + ? error.message + : "Failed to read current Missouri election data", + cause: error, + }); + } + }), + /** * Get live California statewide election results (Secretary of State feed). * Defaults to the marquee races (governor + secretary of state) when no