diff --git a/apps/expo/src/components/UpcomingMeetingsSection.tsx b/apps/expo/src/components/UpcomingMeetingsSection.tsx index 93bbbf27..ef122c32 100644 --- a/apps/expo/src/components/UpcomingMeetingsSection.tsx +++ b/apps/expo/src/components/UpcomingMeetingsSection.tsx @@ -7,16 +7,16 @@ import { import { FontAwesome } from "@expo/vector-icons"; import { useQuery } from "@tanstack/react-query"; -import type { LegistarMeeting } from "@acme/api/integrations/legistar"; +import type { RouterOutputs } from "@acme/api"; import { Text, View } from "~/components/Themed"; import { fontBody, fontEditorial, fontSize, rd, sp, useTheme } from "~/styles"; import { trpc } from "~/utils/api"; +type Meeting = RouterOutputs["localGovernment"]["listMeetings"][number]; + interface UpcomingMeetingsSectionProps { - onMeetingPress?: ( - meeting: LegistarMeeting & { jurisdiction: string }, - ) => void; + onMeetingPress?: (meeting: Meeting) => void; } function formatDate(iso: string): string { @@ -34,7 +34,7 @@ export function UpcomingMeetingsSection({ const { theme } = useTheme(); const meetingsQuery = useQuery( - trpc.legistar.getMeetings.queryOptions({ daysAhead: 30 }), + trpc.localGovernment.listMeetings.queryOptions({ daysAhead: 90 }), ); return ( @@ -47,30 +47,43 @@ export function UpcomingMeetingsSection({ {meetingsQuery.data?.slice(0, 8).map((meeting, index) => ( onMeetingPress?.(meeting)} + onPress={() => + onMeetingPress + ? onMeetingPress(meeting) + : void Linking.openURL(meeting.canonicalUrl) + } activeOpacity={0.8} > {meeting.jurisdiction} - {formatDate(meeting.EventDate)} + + {formatDate(meeting.startsAt.toString())} + - {meeting.EventBodyName} + {meeting.isCancelled ? "Cancelled: " : ""} + {meeting.title} - {meeting.EventLocation && ( + {meeting.location && ( - {meeting.EventLocation} + {meeting.location} )} - {meeting.EventAgendaFile && ( + {meeting.documents.find( + (document) => document.type === "agenda", + ) && ( - void Linking.openURL(meeting.EventAgendaFile ?? "") + void Linking.openURL( + meeting.documents.find( + (document) => document.type === "agenda", + )?.url ?? "", + ) } hitSlop={8} > @@ -81,11 +94,9 @@ export function UpcomingMeetingsSection({ /> )} - {meeting.EventVideoPath && ( + {meeting.videoUrl && ( - void Linking.openURL(meeting.EventVideoPath ?? "") - } + onPress={() => void Linking.openURL(meeting.videoUrl ?? "")} hitSlop={8} > )} - {meeting.EventMinutesFile && ( + {meeting.documents.find( + (document) => document.type === "minutes", + ) && ( - void Linking.openURL(meeting.EventMinutesFile ?? "") + void Linking.openURL( + meeting.documents.find( + (document) => document.type === "minutes", + )?.url ?? "", + ) } hitSlop={8} > diff --git a/apps/scraper/README.md b/apps/scraper/README.md index 0d8828bf..9626d7d0 100644 --- a/apps/scraper/README.md +++ b/apps/scraper/README.md @@ -4,15 +4,20 @@ Pulls in government content like bills, court cases, and White House content and ## Active data sources -Only these five are registered and run by `all`: - -| CLI name | Source and data fetched | Stored/used as | -| ------------------- | ------------------------------------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------- | -| `federalregister` | Federal Register API presidential documents, then each document's body HTML | `government_content`; AI article/summary and feed-image enrichment | -| `congress` | Congress.gov API bill list, detail, CRS summaries, formatted text, and legislative actions | `bill`; powers federal bill content and AI/feed enrichment | -| `scotus` | CourtListener opinion clusters, dockets, and sub-opinion text for the Supreme Court | `court_case`; powers court content and AI/feed enrichment | -| `scc-cvig` | Hand-configured Santa Clara County voter-guide PDFs | Candidate statements in `CivicApiCache`; the API matches statements to candidates | -| `ca-sos-statements` | California SOS statewide-office candidate-statement pages | Candidate statements in `CivicApiCache`; the API reads the cache and can fall back to the live source | +These data sources are registered and run by `all`: + +| CLI name | Source and data fetched | Stored/used as | +| ------------------------ | ------------------------------------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------- | +| `federalregister` | Federal Register API presidential documents, then each document's body HTML | `government_content`; AI article/summary and feed-image enrichment | +| `durham-bocc` | Durham County's official Legistar API (current election cycle only) | Provider-neutral meetings, agenda items, actions, votes, and official document links; no AI required | +| `congress` | Congress.gov API bill list, detail, CRS summaries, formatted text, and legislative actions | `bill`; powers federal bill content and AI/feed enrichment | +| `scotus` | CourtListener opinion clusters, dockets, and sub-opinion text for the Supreme Court | `court_case`; powers court content and AI/feed enrichment | +| `scc-cvig` | Hand-configured Santa Clara County voter-guide PDFs | Candidate statements in `CivicApiCache`; the API matches statements to candidates | +| `ca-sos-statements` | California SOS statewide-office candidate-statement pages | Candidate statements in `CivicApiCache`; the API reads the cache and can fall back to the live source | +| `ncsbe` | Current-cycle NCSBE candidate CSV, referendum PDFs, and result ZIPs | Provider-neutral election tables; powers `civic.getNcElectionData` with exact file provenance | +| `texas-legislature` | Texas Legislative Council anonymous FTP: current-session history XML and bulk documents | State-aware `bill` rows; read through `content.texasBills` and `content.getById` | +| `texas-current-election` | Texas SOS structured election feed and TLC amendment analyses | Current-cycle snapshots; powers `civic.getTexasCurrentElection` and measure enrichment | +| `cedar-park-council` | Cedar Park's CivicEngage City Council page and its official Municode Meetings embed | Provider-neutral local meetings, documents, agenda items, motions, outcomes, and votes | `vote411`, `ca-lao-fiscal`, and `ca-vig-archive` remain under `src/scrapers/disabled/` and do not run. Their caches had no application @@ -57,6 +62,7 @@ work: | `COURTLISTENER_API_KEY` | Optional | Higher CourtListener limits for `scotus`. | | `GOOGLE_API_KEY` / `GOOGLE_SEARCH_ENGINE_ID` | Optional pair | Google Custom Search article thumbnails. | | `GOOGLE_GENERATIVE_AI_API_KEY` | Optional | Gemini vision fallback for `scc-cvig` PDF extraction. | +| `OPEN_STATES_API_KEY` | Optional | Adds an exact Open States bill ID when Texas jurisdiction/session/identifier match. | See [the launch environment guide](../../docs/launch.md) for the complete per-scraper matrix, provider setup links, defaults, and production guidance. @@ -128,6 +134,11 @@ CONGRESS_MAX_ITEMS=10 pnpm --filter @acme/scraper run start congress | `SCOTUS_MAX_ITEMS` | 50 | CourtListener opinion clusters | | `SCC_CVIG_MAX_ITEMS` | 10 | Voter-guide PDF documents | | `CA_SOS_MAX_ITEMS` | 9 | Statewide-office candidate-statement pages | +| `NCSBE_MAX_ITEMS` | 4 | Current-cycle candidate/referendum/result files | +| `TEXAS_LEGISLATURE_MAX_ITEMS` | 100 | Bills from the latest Texas bulk session | +| `TX_SOS_MAX_ITEMS` | 12 | Current-cycle Texas SOS election payloads | +| `CEDAR_PARK_COUNCIL_MAX_ITEMS` | 100 | Council meetings (after the 12-month cutoff) | +| `DURHAM_BOCC_MAX_ITEMS` | 100 | Current-cycle Durham County BOCC meetings | | `SCRAPER_MAX_NEW_ITEMS_PER_RUN` | 10 | New records receiving expensive AI/image enrichment | These are per-run limits, not durable calendar-day quotas. Schedule one run per @@ -137,6 +148,33 @@ invocation gets a fresh allowance. Source limits cap API/page work; bills that require a generated description are deferred before insertion; other content may still be stored raw for later backfill. +The NCSBE integration is intentionally current-cycle only and excludes voter +history plus candidate contact/address fields. See +[`docs/ncsbe-election-data.md`](../../docs/ncsbe-election-data.md) for discovery, +idempotency, provenance, API, and deterministic Civic-matching details. + +## Cedar Park City Council (`civicengage.ts`) + +Cedar Park's public site is CivicEngage, but the City Council records page now +embeds a keyless Municode Meetings publish page. The adapter follows that +official embed, keeps a 12-month Council-only window, downloads at most two +documents concurrently, and deterministically parses PDF text. AI/vision is not +used. Agendas, packets, minutes, and later document URLs are versioned beneath +one provider-neutral meeting record; unchanged reruns produce the same natural +keys and checksums. + +```bash +pnpm --filter @acme/scraper run start cedar-park-council --max-items 1 +``` + +To add a second CivicEngage jurisdiction using the same kind of official +Municode embed, add a `CivicEngageJurisdictionConfig` beside +`cedarParkCouncilSource` with its CivicEngage host/path, IANA timezone, +Municode `cid`/`ppid`, and governing-body matcher, then instantiate the same +discovery/parser pipeline. If its records page uses Agenda Center, Legistar, or +another provider, implement that provider behind the same local-government +persistence contract instead. + --- ## Congress bills (`congress.ts`) @@ -166,6 +204,34 @@ for why the cursor is source-based and why the order matters. --- +## Texas Legislature Online (`texas-legislature.ts`) + +Uses only the Texas Legislative Council's anonymous FTP service at +`ftp.legis.state.tx.us`; it does not crawl interactive TLO bill pages. The job: + +- discovers the newest session directory under `/bills` and rejects a stale + `TEXAS_LEGISLATURE_SESSION` assertion (official codes look like `89R` or + `892`); +- parses bill-history XML for identity, caption, sponsors, subjects, actions, + structured votes when present, and document metadata; +- downloads bill text, analyses, and fiscal notes from the matching FTP bulk + HTML paths and stores their extracted text alongside official HTML/PDF links; +- optionally stores an exact Open States ID without using Open States as the + legislative data source. + +Run a small current-session import: + +```bash +pnpm --filter @acme/scraper run start texas-legislature --max-items 10 +``` + +Apply `packages/db/migrations/add_state_legislation_fields.sql` before the first +run. The public `content.texasBills` procedure lists only the newest persisted +Texas session; `content.getById` returns its documents, actions, and votes. This +work deliberately does not provide historical-session browsing or backfills. + +--- + ## Court cases (`scotus.ts`) Uses the [CourtListener API](https://www.courtlistener.com/api/) — free, works without a key. Fetches recent opinions and pulls in the plain-text opinion content for AI article generation. @@ -193,6 +259,9 @@ All scrapers call into `src/utils/db/operations.ts`. Each time a bill or case is - If the **content changed** → regenerates the article - If **nothing changed** → backfills any missing AI summary/article/thumbnail fields, otherwise skips AI generation +The Texas importer passes `skipEnrichment` so official source data is persisted +without AI summaries, generated imagery, or videos. + Set `SCRAPER_FORCE_AI_REGEN=1` to force a full AI refresh even when the record already has AI content. For a new bill whose description must be generated, the summary is now created diff --git a/apps/scraper/package.json b/apps/scraper/package.json index 4792e76b..3e5d1f46 100644 --- a/apps/scraper/package.json +++ b/apps/scraper/package.json @@ -14,6 +14,7 @@ "@ai-sdk/openai-compatible": "2.0.59", "@openrouter/ai-sdk-provider": "^2.10.0", "ai": "^6.0.141", + "basic-ftp": "^6.0.1", "cheerio": "^1.2.0", "consola": "^3.4.2", "drizzle-orm": "^0.45.2", diff --git a/apps/scraper/src/fixtures/missouri-sos/calendar.html b/apps/scraper/src/fixtures/missouri-sos/calendar.html new file mode 100644 index 00000000..fb7cc28f --- /dev/null +++ b/apps/scraper/src/fixtures/missouri-sos/calendar.html @@ -0,0 +1,36 @@ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Official Election DayStyleRegistrationFilingCloseFinal Certification
April 7, 2026General Municipal Election DayJanuary 27, 2026
August 4, 2026Primary ElectionMay 26, 2026
November 3, 2026General ElectionAugust 25, 2026
+
diff --git a/apps/scraper/src/fixtures/missouri-sos/candidate-index.html b/apps/scraper/src/fixtures/missouri-sos/candidate-index.html new file mode 100644 index 00000000..fc3aa45e --- /dev/null +++ b/apps/scraper/src/fixtures/missouri-sos/candidate-index.html @@ -0,0 +1,11 @@ + diff --git a/apps/scraper/src/fixtures/missouri-sos/candidates.html b/apps/scraper/src/fixtures/missouri-sos/candidates.html new file mode 100644 index 00000000..d53180d3 --- /dev/null +++ b/apps/scraper/src/fixtures/missouri-sos/candidates.html @@ -0,0 +1,49 @@ +
+

Certified Candidate List

+

2026 Primary Election

+

State Senator - District 2

+ + + + + + + + + + + + + + +
+ Republican +
NameMailing Address
Alice First101 Secret Street
Bob Second202 Private Avenue
+ + + + + + + + + + +
+ Democratic +
NameMailing Address
Carol Third303 Hidden Road
+

State Representative - District 11

+ + + + + + + + + + +
+ Libertarian +
NameMailing Address
Dana Fourth404 Confidential Lane
+
diff --git a/apps/scraper/src/fixtures/missouri-sos/historical-results-layout.html b/apps/scraper/src/fixtures/missouri-sos/historical-results-layout.html new file mode 100644 index 00000000..3f6250f1 --- /dev/null +++ b/apps/scraper/src/fixtures/missouri-sos/historical-results-layout.html @@ -0,0 +1,27 @@ + +
+

Last Updated: November 4, 2024 11:45 PM | Unofficial Results

+ + + + + + + + + + + + + + + + + + + + +
+ Governor +
CandidatePartyTotal VotesPercent
Example Candidate ARepublican1,25052.1%
Example Candidate BDemocratic1,14947.9%
+
diff --git a/apps/scraper/src/fixtures/missouri-sos/measures.html b/apps/scraper/src/fixtures/missouri-sos/measures.html new file mode 100644 index 00000000..832eecb2 --- /dev/null +++ b/apps/scraper/src/fixtures/missouri-sos/measures.html @@ -0,0 +1,56 @@ +
+

+ The following ballot measures have been certified for the + August 4, 2026 primary election. +

+

+ Official Ballot Title
Amendment 1 +

+

+ full text +

+

+ certificate +

+

Official Ballot Title:

+
+

Shall the Missouri Constitution be amended for this fixture?

+

State entities estimate no costs or savings.

+
+

Fair Ballot Language:

+
+

A “yes” vote will amend the Constitution.

+

A “no” vote will not amend it.

+
+

+ The following ballot measures have been certified for the + November 3, 2026 general election. +

+

+ Official Ballot Title
Amendment 3 +

+

+ full text +

+

+ certificate +

+

Official Ballot Title:

+
+

Shall another certified proposal be adopted?

+

The fiscal impact is unknown.

+
+

Fair Ballot Language:

+
+

A “yes” vote adopts the proposal.

+

A “no” vote rejects the proposal.

+
+
diff --git a/apps/scraper/src/fixtures/missouri-sos/showmo-unavailable.html b/apps/scraper/src/fixtures/missouri-sos/showmo-unavailable.html new file mode 100644 index 00000000..7e3e9952 --- /dev/null +++ b/apps/scraper/src/fixtures/missouri-sos/showmo-unavailable.html @@ -0,0 +1,5 @@ +
+

ShowMO Votes

+

Election resources and voter information.

+ Previous Election Results +
diff --git a/apps/scraper/src/fixtures/missouri-sos/withdrawals.html b/apps/scraper/src/fixtures/missouri-sos/withdrawals.html new file mode 100644 index 00000000..3f2e270b --- /dev/null +++ b/apps/scraper/src/fixtures/missouri-sos/withdrawals.html @@ -0,0 +1,22 @@ +
+ + + + + + + + + + + + + + + + + + + +
OfficeCandidateReasonRemoval Date
State Representative - District 11Kim Miller (Republican)
1816 Private Street		Withdrawn6/9/2026
1:02 PM
State Senator - District 2Robin Removed (Democratic)
PO Box 999		Disqualified5/19/2026
+
diff --git a/apps/scraper/src/fixtures/ncsbe/candidates-2026.csv b/apps/scraper/src/fixtures/ncsbe/candidates-2026.csv new file mode 100644 index 00000000..cae1dfc3 --- /dev/null +++ b/apps/scraper/src/fixtures/ncsbe/candidates-2026.csv @@ -0,0 +1,3 @@ +"election_dt","county_name","contest_name","name_on_ballot","party_candidate","has_primary","is_partisan","vote_for","term" +"03/03/2026","DURHAM","US HOUSE OF REPRESENTATIVES DISTRICT 04","Nida Allam","DEM","TRUE","TRUE","1","2" +"03/03/2026","WAKE","WAKE COUNTY BOARD OF COMMISSIONERS AT-LARGE","Marguerite Creel","DEM","TRUE","TRUE","2","4" diff --git a/apps/scraper/src/fixtures/ncsbe/referendums-2026.txt b/apps/scraper/src/fixtures/ncsbe/referendums-2026.txt new file mode 100644 index 00000000..f3a816ea --- /dev/null +++ b/apps/scraper/src/fixtures/ncsbe/referendums-2026.txt @@ -0,0 +1,12 @@ +REFERENDUM CHOICES LIST GROUPED BY REFERENDUM +GATES BOARD OF ELECTIONS +CRITERIA: Election: 03/03/2026, County: ALL COUNTIES +GATES +GATES COUNTY LOCAL SALES AND USE TAX REFERENDUM +For Local sales and use tax at the rate of one-quarter percent (0.25%) in addition to all other State and local sales and use taxes. +Against Local sales and use tax at the rate of one-quarter percent (0.25%) in addition to all other State and local sales and use taxes. +GRANVILLE BOARD OF ELECTIONS +GRANVILLE +GRANVILLE COUNTY LOCAL SALES AND USE TAX REFERENDUM +For Local sales and use tax at the rate of one-quarter percent (0.25%) in addition to all other State and local sales and use taxes. +Against Local sales and use tax at the rate of one-quarter percent (0.25%) in addition to all other State and local sales and use taxes. diff --git a/apps/scraper/src/fixtures/ncsbe/results-2024.tsv b/apps/scraper/src/fixtures/ncsbe/results-2024.tsv new file mode 100644 index 00000000..3eb56e2c --- /dev/null +++ b/apps/scraper/src/fixtures/ncsbe/results-2024.tsv @@ -0,0 +1,3 @@ +county election_date precinct contest_group_id contest_type contest_name candidate party vote_for election_day early_voting absentee_by_mail provisional total_votes real_precinct +DURHAM 11/05/2024 01 1373 S US PRESIDENT Kamala D. Harris DEM 1 100 200 30 2 332 Y +WAKE 11/05/2024 01-01 1373 S US PRESIDENT Kamala D. Harris DEM 1 150 250 40 3 443 Y diff --git a/apps/scraper/src/fixtures/ncsbe/results-2026.tsv b/apps/scraper/src/fixtures/ncsbe/results-2026.tsv new file mode 100644 index 00000000..a08290a8 --- /dev/null +++ b/apps/scraper/src/fixtures/ncsbe/results-2026.tsv @@ -0,0 +1,3 @@ +County Election Date Precinct Contest Group ID Contest Type Contest Name Choice Choice Party Vote For Election Day Early Voting Absentee by Mail Provisional Total Votes Real Precinct +DURHAM 03/03/2026 04 2114 S US HOUSE OF REPRESENTATIVES DISTRICT 04 (DEM) Nida Allam DEM 1 371 0 0 0 371 Y +WAKE 03/03/2026 06-05 1 C WAKE COUNTY BOARD OF COMMISSIONERS AT-LARGE (DEM) Marguerite Creel DEM 2 12 0 0 0 12 Y diff --git a/apps/scraper/src/scraper-contracts.ts b/apps/scraper/src/scraper-contracts.ts index bfcfd04f..5e78aa8e 100644 --- a/apps/scraper/src/scraper-contracts.ts +++ b/apps/scraper/src/scraper-contracts.ts @@ -1,15 +1,27 @@ import type { ScraperEnvContract } from "@acme/env"; import { caSosStatementsConfig } from "./scrapers/ca-sos-statements.config.js"; +import { cedarParkCouncilConfig } from "./scrapers/civicengage.config.js"; import { congressConfig } from "./scrapers/congress.config.js"; +import { durhamBoccConfig } from "./scrapers/durham-bocc.config.js"; +import { durhamOnBaseConfig } from "./scrapers/durham-onbase.config.js"; import { federalregisterConfig } from "./scrapers/federalregister.config.js"; +import { ncsbeConfig } from "./scrapers/ncsbe.config.js"; import { sccCvigConfig } from "./scrapers/scc-cvig.config.js"; import { scotusConfig } from "./scrapers/scotus.config.js"; +import { texasCurrentElectionConfig } from "./scrapers/texas-current-election.config.js"; +import { texasLegislatureConfig } from "./scrapers/texas-legislature.config.js"; export const scraperContracts: readonly ScraperEnvContract[] = [ federalregisterConfig, + durhamBoccConfig, congressConfig, scotusConfig, sccCvigConfig, caSosStatementsConfig, + ncsbeConfig, + texasCurrentElectionConfig, + texasLegislatureConfig, + cedarParkCouncilConfig, + durhamOnBaseConfig, ]; diff --git a/apps/scraper/src/scrapers.ts b/apps/scraper/src/scrapers.ts index fb006216..0ef73be5 100644 --- a/apps/scraper/src/scrapers.ts +++ b/apps/scraper/src/scrapers.ts @@ -1,12 +1,24 @@ import type { Scraper } from "./utils/types.js"; import { caSosStatements } from "./scrapers/ca-sos-statements.js"; +import { cedarParkCouncil } from "./scrapers/civicengage.js"; import { congress } from "./scrapers/congress.js"; +import { durhamBocc } from "./scrapers/durham-bocc.js"; +import { durhamOnBase } from "./scrapers/durham-onbase.js"; import { federalregister } from "./scrapers/federalregister.js"; +import { ncsbe } from "./scrapers/ncsbe.js"; import { sccCvig } from "./scrapers/scc-cvig.js"; +import { texasCurrentElection } from "./scrapers/texas-current-election.js"; +import { texasLegislature } from "./scrapers/texas-legislature.js"; export const scrapers: readonly Scraper[] = [ federalregister, + durhamBocc, congress, sccCvig, caSosStatements, + ncsbe, + texasCurrentElection, + texasLegislature, + cedarParkCouncil, + durhamOnBase, ]; diff --git a/apps/scraper/src/scrapers/__fixtures__/durham-onbase-agenda.html b/apps/scraper/src/scrapers/__fixtures__/durham-onbase-agenda.html new file mode 100644 index 00000000..55a984b8 --- /dev/null +++ b/apps/scraper/src/scrapers/__fixtures__/durham-onbase-agenda.html @@ -0,0 +1,6 @@ + +
Consent Agenda
+
1. Approval of City Council Minutes
To approve the listed minutes. [Approved by Vote: 7/0]
+
General Business Agenda
+
21. Consolidated Annexation – 4802 Cheek Road
Motion 1: To adopt the ordinance. [FAILED by Vote: 0/7] Ayes: None. Nays: Mayor Williams, Council Member Baker. Motion 2: Consider rezoning. No vote taken.
+ diff --git a/apps/scraper/src/scrapers/__fixtures__/durham-onbase-index.html b/apps/scraper/src/scrapers/__fixtures__/durham-onbase-index.html new file mode 100644 index 00000000..ed7cac0d --- /dev/null +++ b/apps/scraper/src/scrapers/__fixtures__/durham-onbase-index.html @@ -0,0 +1,3 @@ + diff --git a/apps/scraper/src/scrapers/__fixtures__/durham-onbase-item.html b/apps/scraper/src/scrapers/__fixtures__/durham-onbase-item.html new file mode 100644 index 00000000..af308ff9 --- /dev/null +++ b/apps/scraper/src/scrapers/__fixtures__/durham-onbase-item.html @@ -0,0 +1,4 @@ +

Supporting Documents

+Final-Published Attachment - Approval Memo +March 16 City Council Minutes + diff --git a/apps/scraper/src/scrapers/civicengage-parser.test.ts b/apps/scraper/src/scrapers/civicengage-parser.test.ts new file mode 100644 index 00000000..3b87f7a6 --- /dev/null +++ b/apps/scraper/src/scrapers/civicengage-parser.test.ts @@ -0,0 +1,115 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import test from "node:test"; + +import { + parseAgendaItems, + parseCentralMeetingDate, + parseMunicodePublishPage, +} from "./civicengage-parser.js"; +import { cedarParkCouncilSource } from "./civicengage.config.js"; + +const fixture = (name: string) => + readFile(new URL(`./fixtures/civicengage/${name}`, import.meta.url), "utf8"); + +test("parses regular, special, cancelled, and amended meeting records", async () => { + const meetings = parseMunicodePublishPage( + await fixture("meetings.html"), + cedarParkCouncilSource, + { now: new Date("2026-03-01T00:00:00Z") }, + ); + assert.equal(meetings.length, 4); + assert.deepEqual( + meetings.map(({ title, meetingType, status }) => ({ + title, + meetingType, + status, + })), + [ + { + title: "City Council Mtg. - CANCELLED", + meetingType: "regular", + status: "cancelled", + }, + { + title: "City Council - Special Called", + meetingType: "special", + status: "held", + }, + { + title: "City Council Mtg.", + meetingType: "regular", + status: "completed", + }, + { + title: "City Council Mtg. - Amended", + meetingType: "regular", + status: "amended", + }, + ], + ); + assert.match(meetings[2]?.externalId ?? "", /^[a-f0-9]{32}$/); + assert.equal( + meetings[2]?.documents[0]?.url, + "https://mccmeetings.blob.core.usgovcloudapi.net/cptx-pubu/MEET-Agenda-2c1128fc619f48d6bf29cb94897a2d7f.pdf", + ); + assert.equal( + meetings[2]?.canonicalUrl, + "https://www.cedarparktexas.gov/596/City-Council-Agendas", + ); + assert.deepEqual( + meetings[3]?.documents.map(({ isCurrent }) => isCurrent), + [true, false], + ); +}); + +test("applies a twelve-month cutoff and parses Central daylight time", async () => { + const meetings = parseMunicodePublishPage( + await fixture("meetings.html"), + cedarParkCouncilSource, + { + now: new Date("2026-07-21T12:00:00Z"), + cutoff: new Date("2025-07-21T12:00:00Z"), + }, + ); + assert.equal(meetings.length, 3); + assert.equal( + parseCentralMeetingDate("2/12/2026", "6:00 PM").toISOString(), + "2026-02-13T00:00:00.000Z", + ); + assert.equal( + parseCentralMeetingDate("7/9/2026", "7:00 PM").toISOString(), + "2026-07-10T00:00:00.000Z", + ); +}); + +test("deterministically parses items, motions, outcomes, tallies, and roll calls", async () => { + const agenda = await fixture("agenda.txt"); + const minutes = await fixture("minutes.txt"); + const sourceUrl = "https://official.example/agenda.pdf"; + const first = parseAgendaItems(agenda, minutes, sourceUrl); + const second = parseAgendaItems(agenda, minutes, sourceUrl); + assert.deepEqual(first, second); + assert.equal(first.length, 6); + + const ordinance = first.find((item) => item.itemNumber === "E.1"); + assert.equal(ordinance?.itemType, "ordinance"); + assert.equal(ordinance?.consent, true); + assert.equal(ordinance?.outcome, "approved"); + + const resolution = first.find((item) => item.itemNumber === "F.1"); + assert.equal( + resolution?.motion, + "Motion to approve Agenda Item F.1 as presented.", + ); + assert.match(resolution?.voteSummary ?? "", /^6-0/); + assert.equal(resolution?.outcome, "approved"); + + const rollCall = first.find((item) => item.itemNumber === "H.1"); + assert.equal(rollCall?.votes.length, 6); + assert.deepEqual(rollCall?.votes.at(-1), { voterName: "Darby", value: "no" }); + assert.equal(rollCall?.sourceUrl, sourceUrl); + + const noAction = first.find((item) => item.itemNumber === "H.2"); + assert.equal(noAction?.outcome, "no_action"); +}); diff --git a/apps/scraper/src/scrapers/civicengage-parser.ts b/apps/scraper/src/scrapers/civicengage-parser.ts new file mode 100644 index 00000000..4f5760ed --- /dev/null +++ b/apps/scraper/src/scrapers/civicengage-parser.ts @@ -0,0 +1,414 @@ +import { createHash } from "node:crypto"; +import * as cheerio from "cheerio"; + +import type { CivicEngageJurisdictionConfig } from "./civicengage.config.js"; + +export type MeetingType = + | "regular" + | "special" + | "work_session" + | "retreat" + | "notice" + | "other"; +export type MeetingStatus = + | "scheduled" + | "held" + | "completed" + | "cancelled" + | "amended"; +export type DocumentType = "agenda" | "packet" | "minutes" | "html"; + +export interface DiscoveredDocument { + type: DocumentType; + title: string; + url: string; + mediaType: string; + isCurrent: boolean; +} + +export interface DiscoveredMeeting { + externalId: string; + title: string; + meetingType: MeetingType; + status: MeetingStatus; + startsAt: Date; + location?: string; + canonicalUrl: string; + documents: DiscoveredDocument[]; +} + +export interface ParsedVote { + voterName: string; + value: string; +} + +export interface ParsedAgendaItem { + externalId: string; + sequence: number; + itemNumber: string; + section?: string; + itemType: string; + title: string; + description?: string; + consent: boolean; + motion?: string; + outcome?: string; + voteSummary?: string; + sourceUrl: string; + votes: ParsedVote[]; +} + +const SOURCE_VERSION = "civicengage-municode-v1"; +export { SOURCE_VERSION }; + +function clean(value: string): string { + return value.replace(/\s+/g, " ").trim(); +} + +function directDocumentUrl(href: string, baseUrl: string): string { + const url = new URL(href, baseUrl); + if (url.hostname === "meetings.municode.com" && url.pathname === "/d/f") { + const target = url.searchParams.get("u"); + if (target) return new URL(target).toString(); + } + return url.toString(); +} + +function meetingType(title: string): MeetingType { + if (/notice\s+of\s+(?:a\s+)?possible\s+quorum/i.test(title)) return "notice"; + if (/retreat/i.test(title)) return "retreat"; + if (/work\s*shop|work\s+session/i.test(title)) return "work_session"; + if (/special\s+(?:called\s+)?(?:meeting|mtg)|special\s+called/i.test(title)) { + return "special"; + } + if (/council\s+(?:meeting|mtg)|regular/i.test(title)) return "regular"; + return "other"; +} + +function meetingStatus( + title: string, + startsAt: Date, + hasMinutes: boolean, + now: Date, +): MeetingStatus { + if (/cancel(?:led|ed|lation)/i.test(title)) return "cancelled"; + if (/amend(?:ed|ment)/i.test(title)) return "amended"; + if (hasMinutes) return "completed"; + return startsAt.getTime() > now.getTime() ? "scheduled" : "held"; +} + +export function parseMeetingDate( + dateText: string, + timeText: string, + timezone: string, +): Date { + const match = dateText.trim().match(/^(\d{1,2})\/(\d{1,2})\/(\d{4})$/); + const time = timeText.trim().match(/^(\d{1,2}):(\d{2})\s*([AP]M)$/i); + if (!match || !time) + throw new Error(`Unrecognized meeting date: ${dateText} ${timeText}`); + const month = Number(match[1]); + const day = Number(match[2]); + const year = Number(match[3]); + let hour = Number(time[1]) % 12; + if (time[3]?.toUpperCase() === "PM") hour += 12; + const minute = Number(time[2]); + const desiredUtc = Date.UTC(year, month - 1, day, hour, minute); + // Intl gives the local wall-clock parts for a UTC guess. The delta converts + // the desired local clock time to its real instant, including DST. + let instant = desiredUtc; + for (let pass = 0; pass < 2; pass++) { + const parts = new Intl.DateTimeFormat("en-US", { + timeZone: timezone, + year: "numeric", + month: "2-digit", + day: "2-digit", + hour: "2-digit", + minute: "2-digit", + hourCycle: "h23", + }).formatToParts(new Date(instant)); + const part = (type: Intl.DateTimeFormatPartTypes) => + Number(parts.find((value) => value.type === type)?.value); + const representedUtc = Date.UTC( + part("year"), + part("month") - 1, + part("day"), + part("hour"), + part("minute"), + ); + instant += desiredUtc - representedUtc; + } + return new Date(instant); +} + +export function parseCentralMeetingDate( + dateText: string, + timeText: string, +): Date { + return parseMeetingDate(dateText, timeText, "America/Chicago"); +} + +function stableMeetingId(title: string, startsAt: Date): string { + const normalizedTitle = title + .replace(/\b(?:amended|cancelled|canceled)\b/gi, "") + .replace(/[^a-z0-9]+/gi, " ") + .trim() + .toLowerCase(); + return createHash("sha256") + .update(`${startsAt.toISOString()}|${normalizedTitle}`) + .digest("hex") + .slice(0, 32); +} + +export function parseMunicodePublishPage( + html: string, + config: CivicEngageJurisdictionConfig, + options: { now?: Date; cutoff?: Date } = {}, +): DiscoveredMeeting[] { + const $ = cheerio.load(html); + const now = options.now ?? new Date(); + const canonicalUrl = new URL( + config.recordsPagePath, + config.civicEngageBaseUrl, + ).toString(); + const meetings: DiscoveredMeeting[] = []; + + $("table tr").each((_index, row) => { + const $row = $(row); + const title = clean($row.find("td.meeting").text()); + const dateText = clean($row.find("td.date").text()); + const timeText = clean($row.find("td.time").text()); + if ( + !title || + !dateText || + !timeText || + !config.provider.meetingNamePattern.test(title) + ) + return; + + let startsAt: Date; + try { + startsAt = parseMeetingDate(dateText, timeText, config.timezone); + } catch { + return; + } + if (options.cutoff && startsAt < options.cutoff) return; + + const documents: DiscoveredDocument[] = []; + for (const type of ["agenda", "packet", "minutes"] as const) { + const link = $row.find(`td.${type} a`).first(); + const href = link.attr("href"); + if (!href) continue; + documents.push({ + type, + title: clean(link.find("img").attr("alt") ?? `${type} for ${title}`), + url: directDocumentUrl(href, "https://meetings.municode.com"), + mediaType: "application/pdf", + isCurrent: true, + }); + } + + meetings.push({ + externalId: stableMeetingId(title, startsAt), + title, + meetingType: meetingType(title), + status: meetingStatus( + title, + startsAt, + documents.some((document) => document.type === "minutes"), + now, + ), + startsAt, + location: clean($row.find("td.venue").text()) || undefined, + canonicalUrl, + documents, + }); + }); + + const grouped = new Map(); + for (const candidate of meetings) { + const existing = grouped.get(candidate.externalId); + if (!existing) { + grouped.set(candidate.externalId, candidate); + continue; + } + const candidatePreferred = + ["amended", "cancelled"].includes(candidate.status) && + !["amended", "cancelled"].includes(existing.status); + const preferred = candidatePreferred ? candidate : existing; + const secondary = candidatePreferred ? existing : candidate; + const preferredTypes = new Set( + preferred.documents.map((document) => document.type), + ); + const documents = [ + ...preferred.documents.map((document) => ({ + ...document, + isCurrent: true, + })), + ...secondary.documents.map((document) => ({ + ...document, + isCurrent: !preferredTypes.has(document.type), + })), + ].filter( + (document, index, all) => + all.findIndex( + (candidateDocument) => candidateDocument.url === document.url, + ) === index, + ); + grouped.set(candidate.externalId, { ...preferred, documents }); + } + + return [...grouped.values()].sort( + (a, b) => b.startsAt.getTime() - a.startsAt.getTime(), + ); +} + +const SECTION_NAMES = [ + "Consent Agenda", + "Public Hearings", + "Regular Agenda (Non-Consent)", + "Regular Agenda", + "Executive Session", + "Open Meeting", +]; + +function normalizedLines(text: string): string[] { + return text + .replace(/\r/g, "") + .split("\n") + .map((line) => clean(line)) + .filter(Boolean) + .filter((line) => !/^City Council (?:Agenda|Minutes)$/i.test(line)) + .filter( + (line) => + !/^(?:January|February|March|April|May|June|July|August|September|October|November|December) \d{1,2}, \d{4}$/i.test( + line, + ), + ); +} + +interface TextItemBlock { + number: string; + text: string; + section?: string; +} + +function itemBlocks(text: string): TextItemBlock[] { + const lines = normalizedLines(text); + const blocks: TextItemBlock[] = []; + let section: string | undefined; + let current: TextItemBlock | undefined; + + for (const line of lines) { + const sectionName = SECTION_NAMES.find( + (candidate) => candidate.toLowerCase() === line.toLowerCase(), + ); + if (sectionName) { + section = sectionName; + continue; + } + const start = line.match(/^([A-Z]{1,2}\.\d+)\s+(.+)$/); + if (start) { + if (current) blocks.push(current); + current = { number: start[1]!, text: start[2]!, section }; + } else if (current && !/^(?:Page \d+|-{3,})$/i.test(line)) { + current.text += `\n${line}`; + } + } + if (current) blocks.push(current); + return blocks; +} + +function itemType(text: string, section?: string): string { + if (/public hearing/i.test(text) || /public hearings?/i.test(section ?? "")) + return "public_hearing"; + if (/\bordinance\b/i.test(text)) return "ordinance"; + if (/\bresolution\b/i.test(text)) return "resolution"; + if (/\bminutes\b/i.test(text)) return "minutes"; + if (/presentation|proclamation/i.test(text)) return "presentation"; + if (/executive session/i.test(section ?? "")) return "executive_session"; + return "agenda_item"; +} + +function firstMatch(text: string, pattern: RegExp): string | undefined { + return clean(text.match(pattern)?.[1] ?? "") || undefined; +} + +function parseRollCall(text: string): ParsedVote[] { + const votes: ParsedVote[] = []; + const patterns: [RegExp, string][] = [ + [/(?:Voting|Voted)\s+(?:Aye|Yes):\s*([^\n]+)/gi, "yes"], + [/(?:Voting|Voted)\s+(?:Nay|No):\s*([^\n]+)/gi, "no"], + [/Abstain(?:ing)?:\s*([^\n]+)/gi, "abstain"], + [/Absent:\s*([^\n]+)/gi, "absent"], + ]; + for (const [pattern, value] of patterns) { + let match: RegExpExecArray | null; + while ((match = pattern.exec(text)) !== null) { + for (const rawName of (match[1] ?? "").split(/,|\band\b/i)) { + const voterName = clean(rawName.replace(/Council(?:member)?/gi, "")); + if (voterName) votes.push({ voterName, value }); + } + } + } + return votes; +} + +function outcome(text: string, voteSummary?: string): string | undefined { + if (/no action taken/i.test(text)) return "no_action"; + if (/approved under the consent agenda|motion (?:carried|passed)/i.test(text)) + return "approved"; + const tally = voteSummary?.match(/(\d+)\s*[-–]\s*(\d+)/); + if (tally) return Number(tally[1]) > Number(tally[2]) ? "approved" : "failed"; + return undefined; +} + +export function parseAgendaItems( + agendaText: string, + minutesText: string | undefined, + sourceUrl: string, +): ParsedAgendaItem[] { + const agenda = itemBlocks(agendaText); + const minutes = new Map( + itemBlocks(minutesText ?? "").map((block) => [block.number, block]), + ); + + return agenda.map((block, index) => { + const minutesBlock = minutes.get(block.number); + const combined = minutesBlock?.text ?? block.text; + const parsedMotion = firstMatch( + combined, + /(Motion to[\s\S]*?)(?=\n(?:Movant|Second|Vote):|$)/i, + ); + const motion = + parsedMotion && + /Agenda Items?/i.test(parsedMotion) && + !new RegExp(`\\b${block.number.replace(".", "\\.")}\\b`, "i").test( + parsedMotion, + ) && + !/consent agenda/i.test(block.section ?? "") + ? undefined + : parsedMotion; + const voteSummary = firstMatch(combined, /Vote:\s*([^\n]+)/i); + const title = clean(block.text.split("\n")[0] ?? block.text); + const description = clean( + block.text.replace(block.text.split("\n")[0] ?? "", ""), + ); + return { + externalId: block.number.toLowerCase(), + sequence: index + 1, + itemNumber: block.number, + section: block.section, + itemType: itemType(block.text, block.section), + title, + description: description || undefined, + consent: + /consent agenda/i.test(block.section ?? "") || + /approved under the consent agenda/i.test(combined), + motion, + outcome: outcome(combined, voteSummary), + voteSummary, + sourceUrl, + votes: parseRollCall(combined), + }; + }); +} diff --git a/apps/scraper/src/scrapers/civicengage.config.ts b/apps/scraper/src/scrapers/civicengage.config.ts new file mode 100644 index 00000000..0bb5befc --- /dev/null +++ b/apps/scraper/src/scrapers/civicengage.config.ts @@ -0,0 +1,57 @@ +import type { ScraperEnvContract } from "@acme/env"; + +export interface CivicEngageJurisdictionConfig { + id: string; + name: string; + governingBody: string; + civicEngageBaseUrl: string; + recordsPagePath: string; + timezone: string; + provider: { + kind: "municode-publish-page"; + clientId: string; + publishPageId: string; + meetingNamePattern: RegExp; + }; +} + +/** + * Cedar Park's public website is CivicEngage, but its City Council records page + * now embeds Municode Meetings. Keeping both halves in configuration makes the + * adapter reusable when another CivicEngage jurisdiction uses the same embed, + * without pretending all CivicEngage Agenda Centers share this provider. + */ +export const cedarParkCouncilSource: CivicEngageJurisdictionConfig = { + id: "cedar-park-tx", + name: "City of Cedar Park, Texas", + governingBody: "City Council", + civicEngageBaseUrl: "https://www.cedarparktexas.gov", + recordsPagePath: "/596/City-Council-Agendas", + timezone: "America/Chicago", + provider: { + kind: "municode-publish-page", + clientId: "CPTX", + publishPageId: "d5927f56-2e55-4a02-a095-4ed5e6109cfd", + meetingNamePattern: /\b(?:city\s+)?council\b/i, + }, +}; + +export function municodePublishPageUrl( + config: CivicEngageJurisdictionConfig, +): string { + const url = new URL("https://meetings.municode.com/PublishPage/index"); + url.searchParams.set("cid", config.provider.clientId); + url.searchParams.set("ppid", config.provider.publishPageId); + url.searchParams.set("p", "-1"); + return url.toString(); +} + +export const cedarParkCouncilConfig = { + id: "cedar-park-council", + name: "Cedar Park City Council", + source: "Cedar Park CivicEngage / official Municode Meetings embed", + environment: { + required: ["POSTGRES_URL"], + optional: ["CEDAR_PARK_COUNCIL_MAX_ITEMS"], + }, +} as const satisfies ScraperEnvContract; diff --git a/apps/scraper/src/scrapers/civicengage.ts b/apps/scraper/src/scrapers/civicengage.ts new file mode 100644 index 00000000..dec3f12a --- /dev/null +++ b/apps/scraper/src/scrapers/civicengage.ts @@ -0,0 +1,353 @@ +import { createHash } from "node:crypto"; +import pLimit from "p-limit"; +import { getDocumentProxy } from "unpdf"; + +import { and, eq, notInArray } from "@acme/db"; +import { db } from "@acme/db/client"; +import { + LocalGovernmentAgendaItem, + LocalGovernmentDocument, + LocalGovernmentMeeting, + LocalGovernmentVote, +} from "@acme/db/schema"; + +import type { Scraper } from "../utils/types.js"; +import type { + DiscoveredDocument, + DiscoveredMeeting, + ParsedAgendaItem, +} from "./civicengage-parser.js"; +import { setExpectedTotal } from "../utils/db/metrics.js"; +import { fetchWithRetry } from "../utils/fetch.js"; +import { createLogger } from "../utils/log.js"; +import { + parseAgendaItems, + parseMunicodePublishPage, + SOURCE_VERSION, +} from "./civicengage-parser.js"; +import { + cedarParkCouncilConfig, + cedarParkCouncilSource, + municodePublishPageUrl, +} from "./civicengage.config.js"; + +const logger = createLogger("cedar-park-council"); +const documentLimit = pLimit(2); +const USER_AGENT = + "Billion civic data scraper (+https://github.com/billion-app/billion)"; + +interface FetchedDocument extends DiscoveredDocument { + checksum?: string; + extractedText?: string; + fetchedAt?: Date; +} + +function checksum(bytes: Uint8Array): string { + return createHash("sha256").update(bytes).digest("hex"); +} + +export async function extractPdfText(bytes: Uint8Array): Promise { + const proxy = await getDocumentProxy(bytes); + const pages: string[] = []; + for (let pageNumber = 1; pageNumber <= proxy.numPages; pageNumber++) { + const page = await proxy.getPage(pageNumber); + const content = await page.getTextContent(); + const items = content.items + .map((raw) => { + const item = raw as { str?: string; transform?: number[] }; + return item.str && item.transform + ? { + text: item.str, + x: item.transform[4] ?? 0, + y: item.transform[5] ?? 0, + } + : null; + }) + .filter( + (item): item is { text: string; x: number; y: number } => item !== null, + ) + .sort((a, b) => (Math.abs(a.y - b.y) > 2 ? b.y - a.y : a.x - b.x)); + + const lines: { y: number; parts: string[] }[] = []; + for (const item of items) { + const line = lines.find( + (candidate) => Math.abs(candidate.y - item.y) <= 2, + ); + if (line) line.parts.push(item.text); + else lines.push({ y: item.y, parts: [item.text] }); + } + pages.push(lines.map((line) => line.parts.join(" ")).join("\n")); + } + return pages.join("\n\n"); +} + +async function fetchDocument( + document: DiscoveredDocument, +): Promise { + const [cached] = await db + .select({ + checksum: LocalGovernmentDocument.checksum, + extractedText: LocalGovernmentDocument.extractedText, + fetchedAt: LocalGovernmentDocument.fetchedAt, + }) + .from(LocalGovernmentDocument) + .where(eq(LocalGovernmentDocument.url, document.url)) + .limit(1); + if (cached?.checksum) { + return { + ...document, + checksum: cached.checksum, + extractedText: cached.extractedText ?? undefined, + fetchedAt: cached.fetchedAt ?? undefined, + }; + } + + try { + const response = await fetchWithRetry(document.url, { + headers: { Accept: document.mediaType, "User-Agent": USER_AGENT }, + timeoutMs: 60_000, + }); + const bytes = new Uint8Array(await response.arrayBuffer()); + const shouldExtract = + document.type === "agenda" || document.type === "minutes"; + return { + ...document, + checksum: checksum(bytes), + extractedText: shouldExtract ? await extractPdfText(bytes) : undefined, + fetchedAt: new Date(), + }; + } catch (error) { + logger.warn(`Could not fetch ${document.type} ${document.url}`, error); + return document; + } +} + +function meetingHash( + meeting: DiscoveredMeeting, + documents: FetchedDocument[], + items: ParsedAgendaItem[], +): string { + return createHash("sha256") + .update( + JSON.stringify({ + title: meeting.title, + meetingType: meeting.meetingType, + status: meeting.status, + startsAt: meeting.startsAt.toISOString(), + location: meeting.location, + documents: documents.map(({ type, url, checksum, isCurrent }) => ({ + type, + url, + checksum, + isCurrent, + })), + items, + }), + ) + .digest("hex"); +} + +async function persistMeeting( + meeting: DiscoveredMeeting, + documents: FetchedDocument[], + items: ParsedAgendaItem[], +): Promise { + const fetchedAt = new Date(); + const contentHash = meetingHash(meeting, documents, items); + + await db.transaction(async (tx) => { + const [row] = await tx + .insert(LocalGovernmentMeeting) + .values({ + source: "civicengage", + sourceVersion: SOURCE_VERSION, + jurisdiction: cedarParkCouncilSource.id, + governingBody: cedarParkCouncilSource.governingBody, + externalId: meeting.externalId, + title: meeting.title, + meetingType: meeting.meetingType, + status: meeting.status, + startsAt: meeting.startsAt, + location: meeting.location, + canonicalUrl: meeting.canonicalUrl, + contentHash, + fetchedAt, + }) + .onConflictDoUpdate({ + target: [ + LocalGovernmentMeeting.source, + LocalGovernmentMeeting.jurisdiction, + LocalGovernmentMeeting.externalId, + ], + set: { + sourceVersion: SOURCE_VERSION, + title: meeting.title, + meetingType: meeting.meetingType, + status: meeting.status, + startsAt: meeting.startsAt, + location: meeting.location, + canonicalUrl: meeting.canonicalUrl, + contentHash, + fetchedAt, + updatedAt: fetchedAt, + }, + }) + .returning({ id: LocalGovernmentMeeting.id }); + if (!row) + throw new Error(`Failed to persist meeting ${meeting.externalId}`); + + await tx + .update(LocalGovernmentDocument) + .set({ isCurrent: false, updatedAt: fetchedAt }) + .where(eq(LocalGovernmentDocument.meetingId, row.id)); + + for (const document of documents) { + await tx + .insert(LocalGovernmentDocument) + .values({ + meetingId: row.id, + type: document.type, + title: document.title, + url: document.url, + mediaType: document.mediaType, + checksum: document.checksum, + extractedText: document.extractedText, + isCurrent: document.isCurrent, + fetchedAt: document.fetchedAt, + }) + .onConflictDoUpdate({ + target: [ + LocalGovernmentDocument.meetingId, + LocalGovernmentDocument.type, + LocalGovernmentDocument.url, + ], + set: { + title: document.title, + mediaType: document.mediaType, + checksum: document.checksum, + extractedText: document.extractedText, + isCurrent: document.isCurrent, + fetchedAt: document.fetchedAt, + updatedAt: fetchedAt, + }, + }); + } + + for (const item of items) { + const [itemRow] = await tx + .insert(LocalGovernmentAgendaItem) + .values({ + meetingId: row.id, + externalId: item.externalId, + sequence: item.sequence, + itemNumber: item.itemNumber, + section: item.section, + itemType: item.itemType, + title: item.title, + description: item.description, + consent: item.consent, + motion: item.motion, + outcome: item.outcome, + voteSummary: item.voteSummary, + sourceUrl: item.sourceUrl, + }) + .onConflictDoUpdate({ + target: [ + LocalGovernmentAgendaItem.meetingId, + LocalGovernmentAgendaItem.externalId, + ], + set: { + sequence: item.sequence, + itemNumber: item.itemNumber, + section: item.section, + itemType: item.itemType, + title: item.title, + description: item.description, + consent: item.consent, + motion: item.motion, + outcome: item.outcome, + voteSummary: item.voteSummary, + sourceUrl: item.sourceUrl, + updatedAt: fetchedAt, + }, + }) + .returning({ id: LocalGovernmentAgendaItem.id }); + if (!itemRow) continue; + await tx + .delete(LocalGovernmentVote) + .where(eq(LocalGovernmentVote.agendaItemId, itemRow.id)); + if (item.votes.length === 0) continue; + await tx.insert(LocalGovernmentVote).values( + item.votes.map((vote) => ({ + agendaItemId: itemRow.id, + voterName: vote.voterName, + value: vote.value, + })), + ); + } + const staleItemFilter = + items.length === 0 + ? eq(LocalGovernmentAgendaItem.meetingId, row.id) + : and( + eq(LocalGovernmentAgendaItem.meetingId, row.id), + notInArray( + LocalGovernmentAgendaItem.externalId, + items.map((item) => item.externalId), + ), + ); + await tx.delete(LocalGovernmentAgendaItem).where(staleItemFilter); + }); +} + +async function processMeeting(meeting: DiscoveredMeeting): Promise { + const documents = await Promise.all( + meeting.documents.map((document) => + documentLimit(() => fetchDocument(document)), + ), + ); + const agenda = documents.find((document) => document.type === "agenda"); + const minutes = documents.find((document) => document.type === "minutes"); + const items = agenda?.extractedText + ? parseAgendaItems(agenda.extractedText, minutes?.extractedText, agenda.url) + : []; + await persistMeeting(meeting, documents, items); + logger.success( + `${meeting.startsAt.toISOString().slice(0, 10)} ${meeting.title}: ${items.length} items`, + ); +} + +async function scrape(maxItems = 100): Promise { + const now = new Date(); + const cutoff = new Date(now); + cutoff.setUTCFullYear(cutoff.getUTCFullYear() - 1); + const listingUrl = municodePublishPageUrl(cedarParkCouncilSource); + logger.info( + `Discovering Cedar Park Council meetings since ${cutoff.toISOString()}`, + ); + const response = await fetchWithRetry(listingUrl, { + headers: { Accept: "text/html", "User-Agent": USER_AGENT }, + }); + const meetings = parseMunicodePublishPage( + await response.text(), + cedarParkCouncilSource, + { + now, + cutoff, + }, + ).slice(0, maxItems); + setExpectedTotal(meetings.length); + + // Meetings are intentionally sequential; only document requests within one + // meeting run concurrently, capped at two against the public records host. + for (const meeting of meetings) await processMeeting(meeting); + logger.success(`Completed ${meetings.length} Cedar Park Council meetings`); +} + +export const cedarParkCouncil: Scraper = { + ...cedarParkCouncilConfig, + scrape: (options) => + scrape( + (options?.maxItems ?? Number(process.env.CEDAR_PARK_COUNCIL_MAX_ITEMS)) || + 100, + ), +}; diff --git a/apps/scraper/src/scrapers/disabled/README.md b/apps/scraper/src/scrapers/disabled/README.md index 98456133..eb620282 100644 --- a/apps/scraper/src/scrapers/disabled/README.md +++ b/apps/scraper/src/scrapers/disabled/README.md @@ -45,3 +45,46 @@ and cache the data," and its `getCached*` getters are imported nowhere. To revive: have the scraper persist to the database (a real table the API reads), add a reader in `@acme/api`, then re-register its implementation and contract. +## `st-louis-aldermen.ts` + +Implements active-session St. Louis Board of Aldermen meeting, legislation, and +document ingestion. Disabled because this data is not yet connected to a +configured production database or an application consumer. + +To revive it: provision and migrate the local-government tables, connect the +product to their API reader, then move the implementation/config/parser +back to `scrapers/`, register the implementation and active contract, and +restore the St. Louis source-limit environment entry. + +## `missouri-legislature.ts` + +Implements current Missouri General Assembly bill ingestion from official XML. +Disabled because the state-legislation data is not yet connected to a configured +production database or a product surface. + +To revive it: provision the bill storage and product integration, then move the +implementation/config/parser/source back to `scrapers/`, register the +implementation and active contract, and restore the Missouri source-limit +environment entries. + +## `missouri-sos.ts` + +Implements current-cycle Missouri candidate, ballot-measure, and election-result +snapshot ingestion. Disabled because the snapshot data is not yet connected to +a configured production database or a product surface. + +To revive it: provision the election snapshot storage and product integration, +then move the implementation/config/parser back to `scrapers/`, register +the implementation and active contract, and restore the Missouri source-limit +environment entries. + +## `kansas-city-council.ts` + +Implements current-term Kansas City Council meeting, document, agenda-item, and +vote ingestion from Legistar. Disabled because this data is not yet connected +to a configured production database or an application consumer. + +To revive it: provision and migrate the local-government tables, connect the +product to their API reader, then move the implementation/config back to +`scrapers/`, register the implementation in `scrapers.ts`, restore its active +contract, and add `KANSAS_CITY_COUNCIL_MAX_ITEMS` to the environment registry. diff --git a/apps/scraper/src/scrapers/disabled/kansas-city-council.config.ts b/apps/scraper/src/scrapers/disabled/kansas-city-council.config.ts new file mode 100644 index 00000000..8eb91283 --- /dev/null +++ b/apps/scraper/src/scrapers/disabled/kansas-city-council.config.ts @@ -0,0 +1,11 @@ +import type { ScraperEnvContract } from "@acme/env"; + +export const kansasCityCouncilConfig = { + id: "kansas-city-council", + name: "Kansas City Council", + source: "Kansas City Legistar Web API — Council meetings and roll-call votes", + environment: { + required: ["POSTGRES_URL"], + optional: ["KANSAS_CITY_COUNCIL_MAX_ITEMS"], + }, +} as const satisfies ScraperEnvContract; diff --git a/apps/scraper/src/scrapers/disabled/kansas-city-council.ts b/apps/scraper/src/scrapers/disabled/kansas-city-council.ts new file mode 100644 index 00000000..f0546245 --- /dev/null +++ b/apps/scraper/src/scrapers/disabled/kansas-city-council.ts @@ -0,0 +1,654 @@ +import { createHash } from "node:crypto"; +import pLimit from "p-limit"; +import { z } from "zod/v4"; + +import { and, eq, notInArray } from "@acme/db"; +import { db } from "@acme/db/client"; +import { + LocalGovernmentAgendaItem, + LocalGovernmentDocument, + LocalGovernmentMeeting, + LocalGovernmentVote, +} from "@acme/db/schema"; + +import type { Scraper } from "../../utils/types.js"; +import { setExpectedTotal } from "../../utils/db/metrics.js"; +import { fetchWithRetry } from "../../utils/fetch.js"; +import { createLogger } from "../../utils/log.js"; +import { kansasCityCouncilConfig } from "./kansas-city-council.config.js"; + +const API_BASE = "https://webapi.legistar.com/v1/kansascity"; +const SITE_BASE = "https://kansascity.legistar.com"; +const SITE_GROUP = "D2E89A09-8736-4EFB-B4AE-572E0903BD5A"; +const SITE_GROUP_ID = 821; +const PROVIDER = "legistar"; +const JURISDICTION = "kansas-city-mo"; +const COUNCIL_BODY_ID = 138; +const SOURCE_VERSION = "kansas-city-legistar-v1"; +const TIMEZONE = "America/Chicago"; +const DEFAULT_MAX_ITEMS = 250; +const USER_AGENT = + "Billion civic data scraper (+https://github.com/billion-app/billion)"; +const logger = createLogger(kansasCityCouncilConfig.name); + +const attachmentSchema = z + .object({ + MatterAttachmentId: z.number(), + MatterAttachmentLastModifiedUtc: z.string(), + MatterAttachmentRowVersion: z.string(), + MatterAttachmentName: z.string(), + MatterAttachmentHyperlink: z.string().nullable(), + MatterAttachmentShowOnInternetPage: z.boolean().optional(), + }) + .passthrough(); + +export const kansasCityEventSchema = z + .object({ + EventId: z.number(), + EventGuid: z.string(), + EventLastModifiedUtc: z.string(), + EventRowVersion: z.string(), + EventBodyId: z.number(), + EventBodyName: z.string(), + EventDate: z.string(), + EventTime: z.string().nullable(), + EventAgendaStatusName: z.string().nullable(), + EventMinutesStatusName: z.string().nullable(), + EventLocation: z.string().nullable(), + EventAgendaFile: z.string().nullable(), + EventMinutesFile: z.string().nullable(), + EventAgendaLastPublishedUTC: z.string().nullable().optional(), + EventMinutesLastPublishedUTC: z.string().nullable().optional(), + EventComment: z.string().nullable(), + EventVideoPath: z.string().nullable(), + EventMedia: z.union([z.string(), z.number()]).nullable().optional(), + EventInSiteURL: z.string().nullable(), + }) + .passthrough(); + +export const kansasCityItemSchema = z + .object({ + EventItemId: z.number(), + EventItemLastModifiedUtc: z.string(), + EventItemRowVersion: z.string(), + EventItemEventId: z.number(), + EventItemAgendaSequence: z.number(), + EventItemAgendaNumber: z.string().nullable(), + EventItemVersion: z.string().nullable().optional(), + EventItemAgendaNote: z.string().nullable(), + EventItemMinutesNote: z.string().nullable(), + EventItemActionId: z.number().nullable(), + EventItemActionName: z.string().nullable(), + EventItemActionText: z.string().nullable(), + EventItemPassedFlagName: z.string().nullable(), + EventItemRollCallFlag: z.number().nullable(), + EventItemTitle: z.string().nullable(), + EventItemTally: z.string().nullable(), + EventItemConsent: z.number(), + EventItemMover: z.string().nullable(), + EventItemSeconder: z.string().nullable(), + EventItemMatterId: z.number().nullable(), + EventItemMatterGuid: z.string().nullable(), + EventItemMatterFile: z.string().nullable(), + EventItemMatterName: z.string().nullable(), + EventItemMatterType: z.string().nullable(), + EventItemMatterStatus: z.string().nullable(), + EventItemMatterAttachments: z.array(attachmentSchema).nullable(), + }) + .passthrough(); + +export const kansasCityVoteSchema = z + .object({ + VoteId: z.number(), + VoteLastModifiedUtc: z.string(), + VotePersonId: z.number(), + VotePersonName: z.string(), + VoteValueName: z.string(), + VoteSort: z.number(), + VoteEventItemId: z.number(), + }) + .passthrough(); + +type KansasCityEvent = z.infer; +type KansasCityItem = z.infer; +type KansasCityVote = z.infer; +type DocumentType = "agenda" | "packet" | "minutes" | "attachment"; + +interface KansasCityDocument { + type: DocumentType; + title: string; + url: string; + mediaType: string | null; + checksum: string; +} + +interface EnrichedItem { + item: ReturnType; + votes: ReturnType[] | undefined; +} + +function hash(value: unknown): string { + return createHash("sha256").update(JSON.stringify(value)).digest("hex"); +} + +function datePart(value: string): string { + const match = /^(\d{4}-\d{2}-\d{2})/.exec(value); + if (!match) throw new Error(`Invalid Legistar date: ${value}`); + return match[1]!; +} + +function nthSunday(year: number, monthIndex: number, nth: number): number { + const first = new Date(Date.UTC(year, monthIndex, 1)).getUTCDay(); + return 1 + ((7 - first) % 7) + (nth - 1) * 7; +} + +function centralUtcOffset(date: string): "-05:00" | "-06:00" { + const [year, month, day] = date.split("-").map(Number) as [ + number, + number, + number, + ]; + const dstStart = nthSunday(year, 2, 2); + const dstEnd = nthSunday(year, 10, 1); + const isDst = + (month > 3 && month < 11) || + (month === 3 && day >= dstStart) || + (month === 11 && day < dstEnd); + return isDst ? "-05:00" : "-06:00"; +} + +export function parseKansasCityStart( + eventDate: string, + eventTime: string | null, +): Date { + const date = datePart(eventDate); + const match = eventTime?.trim().match(/^(\d{1,2}):(\d{2})\s*([AP]M)$/i); + let hour = 0; + let minute = 0; + if (match) { + hour = Number(match[1]) % 12; + minute = Number(match[2]); + if (match[3]!.toUpperCase() === "PM") hour += 12; + } + return new Date( + `${date}T${String(hour).padStart(2, "0")}:${String(minute).padStart(2, "0")}:00${centralUtcOffset(date)}`, + ); +} + +/** Kansas City Council terms start August 1 every four years (2023 anchor). */ +export function currentKansasCityCouncilCycleStart(now = new Date()): Date { + const anchorYear = 2023; + const year = now.getUTCFullYear(); + let startYear = anchorYear + Math.floor((year - anchorYear) / 4) * 4; + const candidate = new Date(Date.UTC(startYear, 7, 1)); + if (candidate > now) startYear -= 4; + return new Date(Date.UTC(startYear, 7, 1)); +} + +function mediaType(url: string): string | null { + const extension = new URL(url).pathname.split(".").pop()?.toLowerCase(); + if (extension === "pdf") return "application/pdf"; + if (extension === "doc") return "application/msword"; + if (extension === "docx") + return "application/vnd.openxmlformats-officedocument.wordprocessingml.document"; + if (extension === "pptx") + return "application/vnd.openxmlformats-officedocument.presentationml.presentation"; + return null; +} + +function meetingDocuments( + event: KansasCityEvent, + isAmended: boolean, +): KansasCityDocument[] { + const documents: KansasCityDocument[] = []; + if (event.EventAgendaFile) { + documents.push({ + type: "agenda", + title: `${isAmended ? "Revised " : ""}Agenda`, + url: event.EventAgendaFile, + mediaType: mediaType(event.EventAgendaFile), + checksum: hash({ + url: event.EventAgendaFile, + publishedAt: event.EventAgendaLastPublishedUTC, + rowVersion: event.EventRowVersion, + }), + }); + } + if (event.EventMinutesFile) { + documents.push({ + type: "minutes", + title: /revis/i.test(event.EventMinutesStatusName ?? "") + ? "Revised Minutes" + : "Minutes", + url: event.EventMinutesFile, + mediaType: mediaType(event.EventMinutesFile), + checksum: hash({ + url: event.EventMinutesFile, + publishedAt: event.EventMinutesLastPublishedUTC, + rowVersion: event.EventRowVersion, + }), + }); + } + return documents; +} + +function videoUrl(event: KansasCityEvent): string | null { + if (event.EventVideoPath) { + return new URL(event.EventVideoPath, SITE_BASE).toString(); + } + if (!event.EventMedia) return null; + return `${SITE_BASE}/Video.aspx?Mode=Granicus&ID1=${event.EventMedia}&G=${SITE_GROUP}&Mode2=Video`; +} + +function markerText(event: KansasCityEvent): string { + return [ + event.EventComment, + event.EventAgendaStatusName, + event.EventMinutesStatusName, + event.EventAgendaFile, + event.EventMinutesFile, + ] + .filter(Boolean) + .join(" "); +} + +export function isDiscoverableKansasCityEvent(input: unknown): boolean { + const event = kansasCityEventSchema.parse(input); + return ( + event.EventBodyId === COUNCIL_BODY_ID && + !/hidden/i.test(event.EventAgendaStatusName ?? "") && + !/^test meeting$/i.test(event.EventComment?.trim() ?? "") + ); +} + +export function adaptKansasCityMeeting(input: unknown) { + const event = kansasCityEventSchema.parse(input); + const markers = markerText(event); + const isCancelled = /\bcancel(?:led|ed|lation)?\b/i.test(markers); + const isAmended = /\b(amend(?:ed|ment)?|revis(?:ed|ion)|corrected)\b/i.test( + markers, + ); + const isSpecial = /\bspecial\b/i.test(markers); + const meetingType = isSpecial ? "Special Meeting" : "Regular Meeting"; + const sourceUrl = + event.EventInSiteURL ?? + `${SITE_BASE}/MeetingDetail.aspx?LEGID=${event.EventId}&GID=${SITE_GROUP_ID}&G=${SITE_GROUP}`; + const mapped = { + source: PROVIDER, + jurisdiction: JURISDICTION, + externalId: String(event.EventId), + governingBody: event.EventBodyName, + title: `${event.EventBodyName} ${meetingType}`, + meetingType, + startsAt: parseKansasCityStart(event.EventDate, event.EventTime), + timezone: TIMEZONE, + location: event.EventLocation, + status: isCancelled + ? "cancelled" + : (event.EventMinutesStatusName ?? + event.EventAgendaStatusName ?? + "scheduled"), + isCancelled, + isAmended, + canonicalUrl: sourceUrl, + videoUrl: videoUrl(event), + documents: meetingDocuments(event, isAmended), + sourceVersion: `${SOURCE_VERSION}:${event.EventRowVersion}`, + sourceUpdatedAt: new Date(event.EventLastModifiedUtc), + }; + return { ...mapped, contentHash: hash(mapped) }; +} + +function attachmentDocuments(item: KansasCityItem): KansasCityDocument[] { + return (item.EventItemMatterAttachments ?? []) + .filter( + (attachment) => + attachment.MatterAttachmentHyperlink && + attachment.MatterAttachmentShowOnInternetPage !== false, + ) + .map((attachment) => { + const url = attachment.MatterAttachmentHyperlink!; + const isPacket = /\bagenda\s+packet\b/i.test( + attachment.MatterAttachmentName, + ); + return { + type: isPacket ? ("packet" as const) : ("attachment" as const), + title: attachment.MatterAttachmentName, + url, + mediaType: mediaType(url), + checksum: hash({ + id: attachment.MatterAttachmentId, + rowVersion: attachment.MatterAttachmentRowVersion, + updatedAt: attachment.MatterAttachmentLastModifiedUtc, + url, + }), + }; + }); +} + +export function adaptKansasCityItem(input: unknown) { + const item = kansasCityItemSchema.parse(input); + const title = item.EventItemTitle?.trim() || "Untitled agenda item"; + const isSection = + !item.EventItemMatterId && !item.EventItemActionId && /:\s*$/.test(title); + const mapped = { + externalId: String(item.EventItemId), + meetingExternalId: String(item.EventItemEventId), + sequence: item.EventItemAgendaSequence, + itemNumber: item.EventItemMatterFile ?? item.EventItemAgendaNumber, + section: isSection ? title.replace(/:\s*$/, "") : null, + itemType: + item.EventItemMatterType ?? (isSection ? "section" : "agenda-item"), + title, + description: item.EventItemAgendaNote, + minutesNote: item.EventItemMinutesNote, + consent: item.EventItemConsent === 1, + action: item.EventItemActionName, + motion: item.EventItemActionText, + outcome: item.EventItemPassedFlagName ?? item.EventItemMatterStatus, + voteSummary: item.EventItemTally, + mover: item.EventItemMover, + seconder: item.EventItemSeconder, + matterId: item.EventItemMatterId, + matterGuid: item.EventItemMatterGuid, + documents: attachmentDocuments(item), + shouldFetchVotes: + item.EventItemRollCallFlag === 1 || item.EventItemActionId !== null, + sourceVersion: `${SOURCE_VERSION}:${item.EventItemRowVersion}`, + sourceUpdatedAt: new Date(item.EventItemLastModifiedUtc), + sourceUrl: `${SITE_BASE}/MeetingDetail.aspx?LEGID=${item.EventItemEventId}&GID=${SITE_GROUP_ID}&G=${SITE_GROUP}`, + }; + return { ...mapped, contentHash: hash(mapped) }; +} + +export function adaptKansasCityVote(input: unknown) { + const vote = kansasCityVoteSchema.parse(input); + return { + externalId: String(vote.VoteId), + itemExternalId: String(vote.VoteEventItemId), + voterExternalId: String(vote.VotePersonId), + voterName: vote.VotePersonName, + value: vote.VoteValueName, + sort: vote.VoteSort, + sourceUpdatedAt: new Date(vote.VoteLastModifiedUtc), + }; +} + +function summarizeVotes( + votes: ReturnType[], +): string | null { + if (votes.length === 0) return null; + const counts = new Map(); + for (const vote of votes) { + counts.set(vote.value, (counts.get(vote.value) ?? 0) + 1); + } + return [...counts.entries()] + .map(([value, count]) => `${count} ${value}`) + .join(", "); +} + +async function fetchJson(url: URL): Promise { + const response = await fetchWithRetry(url.toString(), { + headers: { Accept: "application/json", "User-Agent": USER_AGENT }, + timeoutMs: 30_000, + }); + return response.json() as Promise; +} + +async function fetchVotes( + item: ReturnType, +): Promise[] | undefined> { + if (!item.shouldFetchVotes) return undefined; + try { + const url = new URL(`${API_BASE}/EventItems/${item.externalId}/Votes`); + return z + .array(z.unknown()) + .parse(await fetchJson(url)) + .flatMap((raw) => { + const vote = kansasCityVoteSchema.safeParse(raw); + if (!vote.success) { + logger.warn(`Skipping invalid vote for item ${item.externalId}`); + return []; + } + return [adaptKansasCityVote(vote.data)]; + }); + } catch (error) { + logger.warn(`Votes unavailable for item ${item.externalId}`, error); + return undefined; + } +} + +function dedupeDocuments(documents: KansasCityDocument[]) { + return [ + ...new Map(documents.map((document) => [document.url, document])).values(), + ]; +} + +async function persistMeeting( + meeting: ReturnType, + enrichedItems: EnrichedItem[], +): Promise { + const fetchedAt = new Date(); + const documents = dedupeDocuments([ + ...meeting.documents, + ...enrichedItems.flatMap(({ item }) => item.documents), + ]); + const contentHash = hash({ + meeting: meeting.contentHash, + documents: documents.map(({ type, url, checksum }) => ({ + type, + url, + checksum, + })), + items: enrichedItems.map(({ item, votes }) => ({ + hash: item.contentHash, + votes, + })), + }); + const { + documents: _meetingDocuments, + contentHash: _baseHash, + ...meetingRow + } = meeting; + + await db.transaction(async (tx) => { + const [storedMeeting] = await tx + .insert(LocalGovernmentMeeting) + .values({ ...meetingRow, contentHash, fetchedAt }) + .onConflictDoUpdate({ + target: [ + LocalGovernmentMeeting.source, + LocalGovernmentMeeting.jurisdiction, + LocalGovernmentMeeting.externalId, + ], + set: { ...meetingRow, contentHash, fetchedAt, updatedAt: fetchedAt }, + }) + .returning({ id: LocalGovernmentMeeting.id }); + if (!storedMeeting) + throw new Error(`Failed to persist meeting ${meeting.externalId}`); + + await tx + .update(LocalGovernmentDocument) + .set({ isCurrent: false, updatedAt: fetchedAt }) + .where(eq(LocalGovernmentDocument.meetingId, storedMeeting.id)); + + for (const document of documents) { + await tx + .insert(LocalGovernmentDocument) + .values({ + meetingId: storedMeeting.id, + type: document.type, + title: document.title, + url: document.url, + mediaType: document.mediaType, + checksum: document.checksum, + isCurrent: true, + fetchedAt, + }) + .onConflictDoUpdate({ + target: [ + LocalGovernmentDocument.meetingId, + LocalGovernmentDocument.type, + LocalGovernmentDocument.url, + ], + set: { + title: document.title, + mediaType: document.mediaType, + checksum: document.checksum, + isCurrent: true, + fetchedAt, + updatedAt: fetchedAt, + }, + }); + } + + for (const enriched of enrichedItems) { + const { item, votes } = enriched; + const voteSummary = item.voteSummary ?? summarizeVotes(votes ?? []); + const itemRow = { + externalId: item.externalId, + sequence: item.sequence, + itemNumber: item.itemNumber, + section: item.section, + itemType: item.itemType, + title: item.title, + description: item.description, + minutesNote: item.minutesNote, + consent: item.consent, + action: item.action, + motion: item.motion, + outcome: item.outcome, + voteSummary, + mover: item.mover, + seconder: item.seconder, + sourceVersion: item.sourceVersion, + contentHash: item.contentHash, + sourceUpdatedAt: item.sourceUpdatedAt, + sourceUrl: item.sourceUrl, + }; + const [storedItem] = await tx + .insert(LocalGovernmentAgendaItem) + .values({ ...itemRow, meetingId: storedMeeting.id }) + .onConflictDoUpdate({ + target: [ + LocalGovernmentAgendaItem.meetingId, + LocalGovernmentAgendaItem.externalId, + ], + set: { ...itemRow, updatedAt: fetchedAt }, + }) + .returning({ id: LocalGovernmentAgendaItem.id }); + if (!storedItem) + throw new Error(`Failed to persist item ${item.externalId}`); + + // An unavailable vote endpoint preserves the last known roll call. + if (votes === undefined) continue; + await tx + .delete(LocalGovernmentVote) + .where(eq(LocalGovernmentVote.agendaItemId, storedItem.id)); + if (votes.length > 0) { + await tx.insert(LocalGovernmentVote).values( + votes.map((vote) => ({ + agendaItemId: storedItem.id, + externalId: vote.externalId, + voterExternalId: vote.voterExternalId, + voterName: vote.voterName, + value: vote.value, + sort: vote.sort, + sourceUpdatedAt: vote.sourceUpdatedAt, + fetchedAt, + })), + ); + } + } + + const staleItemFilter = + enrichedItems.length === 0 + ? eq(LocalGovernmentAgendaItem.meetingId, storedMeeting.id) + : and( + eq(LocalGovernmentAgendaItem.meetingId, storedMeeting.id), + notInArray( + LocalGovernmentAgendaItem.externalId, + enrichedItems.map(({ item }) => item.externalId), + ), + ); + await tx.delete(LocalGovernmentAgendaItem).where(staleItemFilter); + }); +} + +async function scrapeMeeting(rawEvent: unknown): Promise { + const meeting = adaptKansasCityMeeting(rawEvent); + const itemsUrl = new URL( + `${API_BASE}/Events/${meeting.externalId}/EventItems`, + ); + itemsUrl.searchParams.set("AgendaNote", "1"); + itemsUrl.searchParams.set("MinutesNote", "1"); + itemsUrl.searchParams.set("Attachments", "1"); + const rawItems = z.array(z.unknown()).parse(await fetchJson(itemsUrl)); + const items = rawItems.flatMap((rawItem) => { + const parsed = kansasCityItemSchema.safeParse(rawItem); + if (!parsed.success) { + logger.warn(`Skipping invalid item for meeting ${meeting.externalId}`); + return []; + } + return [adaptKansasCityItem(parsed.data)]; + }); + + // Kansas City stores votes on actioned items even when RollCallFlag is zero. + // Two concurrent vote requests keeps pressure on the public host conservative. + const voteLimit = pLimit(2); + const enrichedItems = await Promise.all( + items.map(async (item) => ({ + item, + votes: await voteLimit(() => fetchVotes(item)), + })), + ); + await persistMeeting(meeting, enrichedItems); + logger.success(`Synced ${meeting.title} (${meeting.externalId})`); +} + +async function scrape(maxItems = DEFAULT_MAX_ITEMS): Promise { + const start = currentKansasCityCouncilCycleStart(); + const end = new Date(Date.UTC(start.getUTCFullYear() + 4, 7, 1)); + const eventsUrl = new URL(`${API_BASE}/Events`); + eventsUrl.searchParams.set( + "$filter", + `EventDate ge datetime'${start.toISOString().slice(0, 10)}' and EventDate lt datetime'${end.toISOString().slice(0, 10)}' and EventBodyId eq ${COUNCIL_BODY_ID}`, + ); + eventsUrl.searchParams.set("$orderby", "EventDate asc"); + eventsUrl.searchParams.set("$top", String(maxItems)); + + logger.info( + `Syncing Council term from ${start.toISOString().slice(0, 10)} (structured source; no archive backfill)`, + ); + const rawEvents = z + .array(z.unknown()) + .parse(await fetchJson(eventsUrl)) + .filter(isDiscoverableKansasCityEvent) + .slice(0, maxItems); + setExpectedTotal(rawEvents.length); + + // Meetings are intentionally sequential. Only vote lookups within one + // meeting run concurrently, capped above at two requests. + for (const rawEvent of rawEvents) { + try { + await scrapeMeeting(rawEvent); + } catch (error) { + const parsed = kansasCityEventSchema.safeParse(rawEvent); + logger.error( + `Meeting ${parsed.success ? parsed.data.EventId : "unknown"} failed without aborting run`, + error, + ); + } + } + logger.success(`Completed ${rawEvents.length} Kansas City Council meetings`); +} + +export const kansasCityCouncil: Scraper = { + ...kansasCityCouncilConfig, + scrape: (options) => + scrape( + (options?.maxItems ?? + Number(process.env.KANSAS_CITY_COUNCIL_MAX_ITEMS)) || + DEFAULT_MAX_ITEMS, + ), +}; diff --git a/apps/scraper/src/scrapers/disabled/missouri-legislature-parser.ts b/apps/scraper/src/scrapers/disabled/missouri-legislature-parser.ts new file mode 100644 index 00000000..c6e5aad0 --- /dev/null +++ b/apps/scraper/src/scrapers/disabled/missouri-legislature-parser.ts @@ -0,0 +1,413 @@ +import { load } from "cheerio"; + +export const MISSOURI_JURISDICTION = + "ocd-jurisdiction/country:us/state:mo/government"; + +export interface MissouriSession { + code: string; + baseUrl: string; + kind: "regular" | "special"; +} + +export interface MissouriBillListEntry { + billNumber: string; + url: string; + sourceVersion: string; + sourceUpdatedAt?: Date; +} + +export type MissouriDocument = { + type: "bill_text" | "analysis" | "fiscal_note"; + description: string; + pdfUrl?: string; +}; + +function clean(value: string | undefined): string | undefined { + const normalized = value?.replace(/\s+/g, " ").trim(); + return normalized || undefined; +} + +function variable(script: string, name: string): string | undefined { + return new RegExp(`\\bvar\\s+${name}\\s*=\\s*['\"]([^'\"]+)['\"]`, "i").exec( + script, + )?.[1]; +} + +export function parseMissouriSessions(script: string): MissouriSession[] { + const code = variable(script, "sessionyearcode"); + const baseUrl = variable(script, "baseURL"); + if (!code || !baseUrl) { + throw new Error("SessionSet.js is missing the active regular session"); + } + const sessions: MissouriSession[] = [{ code, baseUrl, kind: "regular" }]; + if (variable(script, "showSpec")?.toLowerCase() !== "true") return sessions; + for (const [codeName, urlName] of [ + ["specsessionyearcode", "specbaseURL"], + ["specsession2yearcode", "specbaseURL2"], + ] as const) { + const specialCode = variable(script, codeName); + const specialUrl = variable(script, urlName); + if (specialCode && specialUrl && specialCode !== code) { + sessions.push({ + code: specialCode, + baseUrl: specialUrl, + kind: "special", + }); + } + } + return sessions; +} + +export function normalizeMissouriBillNumber(value: string): string { + const match = /^(HCR|HJR|HR|HB|SCR|SJR|SR|SB)\s*0*(\d+)$/i.exec( + value.replace(/\s+/g, ""), + ); + if (!match) + throw new Error(`Unrecognized Missouri bill identifier: ${value}`); + return `${match[1]!.toUpperCase()} ${Number(match[2])}`; +} + +export function parseMissouriTimestamp(value: string): Date | undefined { + const match = + /^(\d{2})-(\d{2})-(\d{4}) (\d{2}):(\d{2}):(\d{2})(?:\.(\d{1,3}))?$/.exec( + value.trim(), + ); + if (!match) return undefined; + const [, month, day, year, hour, minute, second, millis = "0"] = match; + const wallClockUtc = Date.UTC( + Number(year), + Number(month) - 1, + Number(day), + Number(hour), + Number(minute), + Number(second), + Number(millis.padEnd(3, "0")), + ); + const guess = new Date(wallClockUtc); + const parts = new Intl.DateTimeFormat("en-US", { + timeZone: "America/Chicago", + year: "numeric", + month: "2-digit", + day: "2-digit", + hour: "2-digit", + minute: "2-digit", + second: "2-digit", + hourCycle: "h23", + }).formatToParts(guess); + const part = (type: Intl.DateTimeFormatPartTypes) => + Number(parts.find((item) => item.type === type)?.value); + const chicagoAsUtc = Date.UTC( + part("year"), + part("month") - 1, + part("day"), + part("hour"), + part("minute"), + part("second"), + Number(millis.padEnd(3, "0")), + ); + return new Date(wallClockUtc - (chicagoAsUtc - guess.valueOf())); +} + +export function parseMissouriBillList(xml: string): MissouriBillListEntry[] { + const $ = load(xml, { xml: true }); + return $("BillXML") + .map((_, element) => { + const row = $(element); + const type = clean(row.find("BillType").first().text()); + const number = clean(row.find("BillNumber").first().text()); + const url = clean(row.find("BillXMLLink").first().text()); + const sourceVersion = clean(row.find("LastTimeRun").first().text()); + if (!type || !number || !url || !sourceVersion) return undefined; + return { + billNumber: normalizeMissouriBillNumber(`${type}${number}`), + url, + sourceVersion, + ...(parseMissouriTimestamp(sourceVersion) && { + sourceUpdatedAt: parseMissouriTimestamp(sourceVersion), + }), + }; + }) + .get() + .filter((entry): entry is MissouriBillListEntry => Boolean(entry)) + .sort((left, right) => left.billNumber.localeCompare(right.billNumber)); +} + +function isoDate(value: string | undefined): string | undefined { + const text = clean(value); + if (!text) return undefined; + const match = /^(\d{1,2})\/(\d{1,2})\/(\d{4})/.exec(text); + if (match) { + return `${match[3]}-${match[1]!.padStart(2, "0")}-${match[2]!.padStart(2, "0")}`; + } + const parsed = new Date(text); + return Number.isNaN(parsed.valueOf()) + ? undefined + : parsed.toISOString().slice(0, 10); +} + +function documents($: ReturnType): MissouriDocument[] { + const result: MissouriDocument[] = []; + const add = ( + type: MissouriDocument["type"], + description: string, + url?: string, + ) => { + const pdfUrl = clean(url); + if (pdfUrl) + result.push({ type, description: clean(description) ?? type, pdfUrl }); + }; + $("BillText").each((_, element) => { + const row = $(element); + add( + "bill_text", + row.find("DocumentName").first().text(), + row.find("BillTextLink").first().text(), + ); + }); + $("BillSummary").each((_, element) => { + const row = $(element); + add( + "analysis", + `Bill summary — ${row.find("DocumentName").first().text()}`, + row.find("SummaryTextLink").first().text(), + ); + }); + $("FiscalNote").each((index, element) => + add( + "fiscal_note", + `Fiscal note ${index + 1}`, + $(element).find("FiscalNoteLink").first().text(), + ), + ); + $("Amendment").each((_, element) => { + const row = $(element); + add( + "analysis", + `Amendment — ${row.find("AmendmentDescription").first().text()}`, + row.find("AmendmentText").first().text(), + ); + }); + add("analysis", "Summary sheet", $("SummarySheetLink").first().text()); + add( + "analysis", + "Governor veto letter", + $("GovernorsVetoLetter").first().text(), + ); + $("Witness").each((index, element) => + add( + "analysis", + `Witness forms — ${ + clean($(element).find("WitnessFormsLinkDescription").first().text()) ?? + index + 1 + }`, + $(element).find("WitnessFormsLink").first().text(), + ), + ); + return result; +} + +export function parseMissouriBill( + xml: string, + options: { + session: string; + sourceVersion: string; + sourceUpdatedAt?: Date; + coverage: "complete_house_export" | "senate_with_house_actions_only"; + }, +) { + const $ = load(xml, { xml: true }); + const root = $("BillInformation").first(); + const billNumber = normalizeMissouriBillNumber( + root.find("BillNumber").first().text(), + ); + const chamber = billNumber.startsWith("H") + ? ("House" as const) + : ("Senate" as const); + const title = + clean(root.find("Title > LongTitle").first().text()) ?? + clean(root.children("LongTitle").first().text()) ?? + clean(root.find("Title > ShortTitle").first().text()) ?? + clean(root.children("ShortTitle").first().text()); + if (!title) throw new Error(`${billNumber} is missing a title`); + const shortTitle = + clean(root.find("Title > ShortTitle").first().text()) ?? + clean(root.children("ShortTitle").first().text()); + + const sponsorships: { + name: string; + classification: "primary" | "cosponsor"; + chamber: "House" | "Senate"; + }[] = []; + root.children("Sponsor").each((_, element) => { + const row = $(element); + const name = clean(row.find("FullName").first().text()); + if (!name) return; + sponsorships.push({ + name, + classification: /co/i.test(row.find("SponsorType").first().text()) + ? "cosponsor" + : "primary", + chamber, + }); + }); + const senateSponsor = clean(root.children("SponsorName").first().text()); + if (senateSponsor) + sponsorships.push({ + name: senateSponsor, + classification: "primary", + chamber: "Senate", + }); + + const actions: { date: string; text: string; type?: string }[] = []; + const votes: { + identifier: string; + date?: string; + chamber?: "House" | "Senate"; + motion?: string; + counts: { option: string; value: number }[]; + votes: never[]; + }[] = []; + root.children("Action").each((_, element) => { + const row = $(element); + const date = isoDate(row.find("PubDate").first().text()); + const text = clean(row.find("Description").first().text()); + if (!date || !text) return; + const guid = clean(row.find("Guid").first().text()); + actions.push({ date, text, ...(guid && { type: `guid:${guid}` }) }); + const rollCall = row.find("RollCall").first(); + if (rollCall.length) { + const count = (selector: string) => + Number(clean(rollCall.find(selector).first().text())); + const counts = [ + ["Yes", count("TotalYes")], + ["No", count("TotalNo")], + ["Present", count("TotalPresent")], + ] as const; + votes.push({ + identifier: guid ?? `${date}:${text}`, + date, + chamber: /\(S\)/.test(text) ? "Senate" : "House", + motion: text, + counts: counts + .filter((entry) => Number.isInteger(entry[1])) + .map(([option, value]) => ({ option, value })), + votes: [], + }); + } + }); + const effectiveDateText = isoDate( + root.find("ProposedEffectiveDate").first().text(), + ); + const sponsor = sponsorships + .filter((item) => item.classification === "primary") + .map((item) => item.name) + .join(" | ") + .slice(0, 256); + root.find("Hearings").each((_, element) => { + const hearing = $(element); + const name = clean(hearing.find("CommitteeName, CommName").first().text()); + const date = + isoDate(hearing.find("NoticeDate").first().text()) ?? actions[0]?.date; + if (name && date) { + actions.push({ + date, + text: `Committee hearing: ${name}`, + type: "committee", + }); + } + }); + if (effectiveDateText) { + actions.push({ + date: effectiveDateText, + text: "Proposed effective date", + type: "effective_date", + }); + } + actions.sort((left, right) => left.date.localeCompare(right.date)); + const url = + clean(root.children("Action").first().find("Link").first().text()) ?? + `https://house.mo.gov/bill.aspx?bill=${billNumber.replace(/\s+/g, "")}&year=20${options.session.slice(0, 2)}&code=${options.session.slice(2) === "1" ? "R" : `S${options.session.slice(2)}`}`; + return { + billNumber, + title, + ...(shortTitle && { description: shortTitle }), + ...(sponsor && { sponsor }), + ...(clean(root.find("LastAction").first().text()) && { + status: clean(root.find("LastAction").first().text())!.slice(0, 100), + }), + ...(actions[0]?.date && { + introducedDate: new Date(`${actions[0].date}T12:00:00Z`), + }), + chamber, + jurisdiction: MISSOURI_JURISDICTION, + legislativeSession: options.session, + subjects: root + .find("SubjectIndex > SubjectName") + .map((_, element) => clean($(element).text())) + .get() + .filter(Boolean) as string[], + sponsorships, + documents: documents($), + votes, + actions, + versions: [ + { + hash: options.sourceVersion, + updatedAt: + options.sourceUpdatedAt?.toISOString() ?? + `${actions.at(-1)?.date ?? "1970-01-01"}T00:00:00.000Z`, + changes: options.coverage, + }, + ], + url, + }; +} + +export function missouriCommitteesFromActions( + actions: readonly { text: string; type?: string }[], +): string[] { + return [ + ...new Set( + actions.flatMap((action) => { + if (action.type === "committee") { + const name = clean( + action.text.replace(/^Committee hearing:\s*/i, ""), + ); + return name ? [name] : []; + } + const match = + /(?:Re-referred to Committee|Referred):\s*(.+?)(?:\([HS]\))?$/i.exec( + action.text, + ); + const name = clean(match?.[1]); + return name ? [name] : []; + }), + ), + ]; +} + +export function missouriEffectiveDateFromActions( + actions: readonly { date: string; type?: string }[], +): string | undefined { + return actions.find((action) => action.type === "effective_date")?.date; +} + +export function parseMissouriSenateActionList( + xml: string, + session: string, + sourceVersion: string, + sourceUpdatedAt?: Date, +) { + const $ = load(xml, { xml: true }); + return $("SenateBillsWithHouseActions > BillInformation") + .map((_, element) => + parseMissouriBill($.xml(element), { + session, + sourceVersion, + ...(sourceUpdatedAt && { sourceUpdatedAt }), + coverage: "senate_with_house_actions_only", + }), + ) + .get() + .filter((bill) => bill.chamber === "Senate"); +} diff --git a/apps/scraper/src/scrapers/disabled/missouri-legislature-source.ts b/apps/scraper/src/scrapers/disabled/missouri-legislature-source.ts new file mode 100644 index 00000000..498161db --- /dev/null +++ b/apps/scraper/src/scrapers/disabled/missouri-legislature-source.ts @@ -0,0 +1,6 @@ +export function missouriRefreshExpiresAt( + now: Date, + intervalMs = 30 * 60 * 1000, +): Date { + return new Date(now.valueOf() + intervalMs); +} diff --git a/apps/scraper/src/scrapers/disabled/missouri-legislature.config.ts b/apps/scraper/src/scrapers/disabled/missouri-legislature.config.ts new file mode 100644 index 00000000..c79ff53a --- /dev/null +++ b/apps/scraper/src/scrapers/disabled/missouri-legislature.config.ts @@ -0,0 +1,12 @@ +import type { ScraperEnvContract } from "@acme/env"; + +export const missouriLegislatureConfig = { + id: "missouri-legislature", + name: "Missouri General Assembly", + source: + "Missouri House official current-session BillList.XML, bill XML, and partial SenateActList.XML", + environment: { + required: ["POSTGRES_URL"], + optional: ["OPEN_STATES_API_KEY", "MISSOURI_LEGISLATURE_MAX_ITEMS"], + }, +} as const satisfies ScraperEnvContract; diff --git a/apps/scraper/src/scrapers/disabled/missouri-legislature.ts b/apps/scraper/src/scrapers/disabled/missouri-legislature.ts new file mode 100644 index 00000000..94672880 --- /dev/null +++ b/apps/scraper/src/scrapers/disabled/missouri-legislature.ts @@ -0,0 +1,280 @@ +import pLimit from "p-limit"; + +import { and, eq, inArray, lte, notInArray } from "@acme/db"; +import { db } from "@acme/db/client"; +import { Bill, CivicApiCache } from "@acme/db/schema"; + +import type { BillData, Scraper } from "../../utils/types.js"; +import { setExpectedTotal } from "../../utils/db/metrics.js"; +import { upsertContent } from "../../utils/db/operations.js"; +import { fetchWithRetry } from "../../utils/fetch.js"; +import { createContentHash } from "../../utils/hash.js"; +import { createLogger } from "../../utils/log.js"; +import { + MISSOURI_JURISDICTION, + parseMissouriBill, + parseMissouriBillList, + parseMissouriSenateActionList, + parseMissouriSessions, +} from "./missouri-legislature-parser.js"; +import { missouriRefreshExpiresAt } from "./missouri-legislature-source.js"; +import { missouriLegislatureConfig } from "./missouri-legislature.config.js"; + +const logger = createLogger("missouri-legislature"); +const SESSION_DESCRIPTOR = "https://documents.house.mo.gov/SessionSet.js"; +const SOURCE_WEBSITE = "documents.house.mo.gov"; +const REFRESH_SOURCE = "missouri-house-xml"; +const REFRESH_ADDRESS_HASH = createContentHash(REFRESH_SOURCE); +const REFRESH_PARAMS = "{}"; +const REFRESH_INTERVAL_MS = 30 * 60 * 1000; +const FETCH_CONCURRENCY = 2; + +async function claimRefresh(now = new Date()): Promise { + const expiresAt = missouriRefreshExpiresAt(now, REFRESH_INTERVAL_MS); + const rows = await db + .insert(CivicApiCache) + .values({ + addressHash: REFRESH_ADDRESS_HASH, + endpoint: REFRESH_SOURCE, + params: REFRESH_PARAMS, + responseData: { source: REFRESH_SOURCE }, + fetchedAt: now, + expiresAt, + }) + .onConflictDoUpdate({ + target: [ + CivicApiCache.addressHash, + CivicApiCache.endpoint, + CivicApiCache.params, + ], + set: { fetchedAt: now, expiresAt }, + setWhere: lte(CivicApiCache.expiresAt, now), + }) + .returning({ id: CivicApiCache.id }); + return rows.length > 0; +} + +async function text(url: string): Promise { + return (await fetchWithRetry(url, { timeoutMs: 30_000 })).text(); +} + +function openStatesSession(session: string): string { + const year = `20${session.slice(0, 2)}`; + const suffix = Number(session.slice(2)); + return suffix <= 1 ? year : `${year}S${suffix - 2}`; +} + +interface OpenStatesSearchResponse { + results?: { id: string; identifier: string; session: string }[]; +} + +export async function matchMissouriOpenStatesBillId( + session: string, + billNumber: string, + apiKey = process.env.OPEN_STATES_API_KEY, +): Promise { + if (!apiKey) return undefined; + const url = new URL("https://v3.openstates.org/bills"); + url.searchParams.set("jurisdiction", MISSOURI_JURISDICTION); + url.searchParams.set("session", openStatesSession(session)); + url.searchParams.set("q", billNumber); + url.searchParams.set("per_page", "5"); + const response = await fetch(url, { + headers: { Accept: "application/json", "X-API-KEY": apiKey }, + }); + if (!response.ok) return undefined; + const payload = (await response.json()) as OpenStatesSearchResponse; + return payload.results?.find( + (bill) => + bill.session === openStatesSession(session) && + bill.identifier.replace(/\s+/g, "").toUpperCase() === + billNumber.replace(/\s+/g, "").toUpperCase(), + )?.id; +} + +async function persistBill(data: BillData): Promise { + if (!data.legislativeSession) { + throw new Error(`${data.billNumber} is missing its Missouri session`); + } + let openStatesId: string | undefined; + try { + openStatesId = await matchMissouriOpenStatesBillId( + data.legislativeSession, + data.billNumber, + ); + } catch (error) { + logger.debug( + `Optional Open States match skipped for ${data.billNumber}: ${error instanceof Error ? error.message : error}`, + ); + } + await upsertContent( + { + type: "bill", + data: { + ...data, + ...(openStatesId && { openStatesId }), + sourceWebsite: SOURCE_WEBSITE, + }, + }, + { skipEnrichment: true }, + ); +} + +function withVersionHistory( + bill: ReturnType, + histories: Map>, +): BillData { + const key = `${bill.legislativeSession}:${bill.billNumber}`; + const previous = histories.get(key) ?? []; + const current = bill.versions[0]!; + const versions = + previous.at(-1)?.hash === current.hash + ? previous + : [...previous, current].slice(-50); + histories.set(key, versions); + return { ...bill, versions, sourceWebsite: SOURCE_WEBSITE }; +} + +export async function scrapeMissouriLegislature(options: { + maxItems: number; + now?: Date; +}): Promise { + const refreshNow = options.now ?? new Date(); + if (!(await claimRefresh(refreshNow))) { + logger.info( + "Skipping Missouri XML refresh: the official 30-minute polling interval has not elapsed.", + ); + return; + } + + const descriptor = await text(SESSION_DESCRIPTOR); + const sessions = parseMissouriSessions(descriptor); + const activeSessionCodes = sessions.map((session) => session.code); + await db + .delete(Bill) + .where( + and( + eq(Bill.jurisdiction, MISSOURI_JURISDICTION), + eq(Bill.sourceWebsite, SOURCE_WEBSITE), + notInArray(Bill.legislativeSession, activeSessionCodes), + ), + ); + const existing = await db + .select({ + billNumber: Bill.billNumber, + session: Bill.legislativeSession, + versions: Bill.versions, + }) + .from(Bill) + .where( + and( + eq(Bill.jurisdiction, MISSOURI_JURISDICTION), + eq(Bill.sourceWebsite, SOURCE_WEBSITE), + inArray(Bill.legislativeSession, activeSessionCodes), + ), + ); + const versionHistories = new Map( + existing.map((row) => [ + `${row.session}:${row.billNumber}`, + row.versions ?? [], + ]), + ); + const currentVersion = (key: string) => + versionHistories.get(key)?.at(-1)?.hash; + const limit = pLimit(FETCH_CONCURRENCY); + const pending: BillData[] = []; + + for (const [sessionIndex, session] of sessions.entries()) { + const sessionBudget = + Math.floor(options.maxItems / sessions.length) + + (sessionIndex < options.maxItems % sessions.length ? 1 : 0); + const sessionEnd = pending.length + sessionBudget; + const billListXml = await text(`${session.baseUrl}BillList.XML`); + const changed = parseMissouriBillList(billListXml) + .filter( + (entry) => + currentVersion(`${session.code}:${entry.billNumber}`) !== + entry.sourceVersion, + ) + .sort((left, right) => { + const leftMissing = !versionHistories.has( + `${session.code}:${left.billNumber}`, + ); + const rightMissing = !versionHistories.has( + `${session.code}:${right.billNumber}`, + ); + return Number(rightMissing) - Number(leftMissing); + }); + const senateReserve = Math.min( + sessionBudget, + Math.max(1, Math.floor(sessionBudget * 0.2)), + ); + const remaining = Math.max(0, sessionEnd - pending.length - senateReserve); + const houseBills = await Promise.all( + changed.slice(0, remaining).map((entry) => + limit(async () => + parseMissouriBill(await text(entry.url), { + session: session.code, + sourceVersion: entry.sourceVersion, + ...(entry.sourceUpdatedAt && { + sourceUpdatedAt: entry.sourceUpdatedAt, + }), + coverage: "complete_house_export", + }), + ), + ), + ); + pending.push( + ...houseBills.map((bill) => withVersionHistory(bill, versionHistories)), + ); + + if (pending.length < sessionEnd) { + const senateXml = await text(`${session.baseUrl}SenateActList.XML`); + const senateVersion = `sha256:${createContentHash(senateXml)}`; + const senateBills = parseMissouriSenateActionList( + senateXml, + session.code, + senateVersion, + refreshNow, + ) + .filter( + (bill) => + currentVersion(`${session.code}:${bill.billNumber}`) !== + senateVersion, + ) + .sort((left, right) => { + const leftMissing = !versionHistories.has( + `${session.code}:${left.billNumber}`, + ); + const rightMissing = !versionHistories.has( + `${session.code}:${right.billNumber}`, + ); + return Number(rightMissing) - Number(leftMissing); + }); + pending.push( + ...senateBills + .slice(0, sessionEnd - pending.length) + .map((bill) => withVersionHistory(bill, versionHistories)), + ); + } + } + + setExpectedTotal(pending.length); + for (const bill of pending) await persistBill(bill); + logger.success( + `Persisted ${pending.length} changed Missouri bills across active session(s) ${activeSessionCodes.join(", ")}. Senate rows include House actions only.`, + ); +} + +async function scrape(options?: { maxItems?: number }): Promise { + await scrapeMissouriLegislature({ + maxItems: + options?.maxItems ?? + (Number(process.env.MISSOURI_LEGISLATURE_MAX_ITEMS) || 100), + }); +} + +export const missouriLegislature: Scraper = { + ...missouriLegislatureConfig, + scrape, +}; diff --git a/apps/scraper/src/scrapers/disabled/missouri-sos-parsers.ts b/apps/scraper/src/scrapers/disabled/missouri-sos-parsers.ts new file mode 100644 index 00000000..716bfbb6 --- /dev/null +++ b/apps/scraper/src/scrapers/disabled/missouri-sos-parsers.ts @@ -0,0 +1,456 @@ +import * as cheerio from "cheerio"; + +import type { + MissouriBallotMeasure, + MissouriCandidate, + MissouriCitation, + MissouriElection, + MissouriResultContest, +} from "@acme/api/lib/missouri-election-data"; + +export const MISSOURI_STRUCTURE_VERSION = "mo-sos-current-election-v1"; +export const MISSOURI_CANDIDATES_URL = "https://s1.sos.mo.gov/candidatesonweb/"; +export const MISSOURI_MEASURES_URL = + "https://www.sos.mo.gov/petitions/2026BallotMeasures"; +export const MISSOURI_RESULTS_URL = + "https://www.sos.mo.gov/elections/showmovotes"; +export const MISSOURI_CALENDAR_URL = + "https://www.sos.mo.gov/elections/calendar/2026cal"; + +export interface MissouriOfficeReference { + office: string; + url: string; +} + +export interface MissouriCandidateDiscovery { + electionCode: string; + electionName: string; + candidatesUrl: string; + offices: MissouriOfficeReference[]; + withdrawalsUrl: string; +} + +function compact(value: string): string { + return value + .replace(/\u00a0/g, " ") + .replace(/\s+/g, " ") + .trim(); +} + +function absoluteUrl(href: string, baseUrl: string): string { + return new URL(href, baseUrl).toString(); +} + +function isoDate(value: string): string | null { + const match = + /\b(January|February|March|April|May|June|July|August|September|October|November|December)\s+(\d{1,2}),\s*(2026)\b/i.exec( + value, + ) ?? /\b(\d{1,2})\/(\d{1,2})\/(2026)\b/.exec(value); + if (!match) return null; + if (/^\d/.test(match[1] ?? "")) { + return `${match[3]}-${String(match[1]).padStart(2, "0")}-${String( + match[2], + ).padStart(2, "0")}`; + } + const month = + [ + "january", + "february", + "march", + "april", + "may", + "june", + "july", + "august", + "september", + "october", + "november", + "december", + ].indexOf((match[1] ?? "").toLowerCase()) + 1; + return `${match[3]}-${String(month).padStart(2, "0")}-${String( + match[2], + ).padStart(2, "0")}`; +} + +function citation(label: string, sourceUrl: string): MissouriCitation { + return { label, sourceUrl }; +} + +export function parseMissouriCalendar( + html: string, + asOf = new Date(), + sourceUrl = MISSOURI_CALENDAR_URL, + preferredType?: MissouriElection["type"], +): MissouriElection { + const $ = cheerio.load(html); + const elections: MissouriElection[] = []; + $("table.cal tr").each((_index, row) => { + const cells = $(row) + .find("td") + .map((_cellIndex, cell) => compact($(cell).text())) + .get(); + if (cells.length < 6) return; + const electionDate = isoDate(cells[0] ?? ""); + const finalCertificationDate = isoDate(cells[5] ?? "") ?? undefined; + const style = cells[1] ?? ""; + const type = /primary election/i.test(style) + ? "primary" + : /general election/i.test(style) && !/municipal/i.test(style) + ? "general" + : null; + if (!electionDate || !type) return; + elections.push({ + name: `2026 ${type === "primary" ? "Primary" : "General"} Election`, + type, + electionDate, + finalCertificationDate, + citation: citation("2026 Missouri election calendar", sourceUrl), + }); + }); + if (!elections.length) { + throw new Error( + "Missouri calendar contained no 2026 primary/general election", + ); + } + const preferred = preferredType + ? elections.find((election) => election.type === preferredType) + : undefined; + if (preferred) return preferred; + const today = asOf.toISOString().slice(0, 10); + return ( + elections.find((election) => election.electionDate >= today) ?? + elections.at(-1)! + ); +} + +export function parseMissouriCandidateDiscovery( + html: string, + baseUrl = MISSOURI_CANDIDATES_URL, +): MissouriCandidateDiscovery { + const $ = cheerio.load(html); + const electionName = compact($("#main h2, #right h2").first().text()); + const references = new Map(); + let electionCode = ""; + $("a[href*='DisplayCandidatesPlacement.aspx']").each((_index, anchor) => { + const href = $(anchor).attr("href"); + const office = compact($(anchor).text()); + if (!href || !office) return; + const url = new URL(href, baseUrl); + const code = url.searchParams.get("ElectionCode"); + const officeCode = url.searchParams.get("OfficeCode"); + if (!code || !officeCode) return; + electionCode ||= code; + references.set(url.toString(), { office, url: url.toString() }); + }); + if (!electionCode || !references.size || !/2026/i.test(electionName)) { + throw new Error( + "Candidate filing page did not expose a certified 2026 election", + ); + } + return { + electionCode, + electionName, + candidatesUrl: absoluteUrl( + `DisplayCandidatesPlacement.aspx?ElectionCode=${encodeURIComponent(electionCode)}`, + baseUrl, + ), + offices: [...references.values()], + withdrawalsUrl: absoluteUrl( + `CandidatesRemoved.aspx?ElectionCode=${encodeURIComponent(electionCode)}`, + baseUrl, + ), + }; +} + +function officeParts(value: string): { office: string; district?: string } { + const normalized = compact(value); + const match = /^(.*?)\s*-?\s+District\s+(\d+)$/i.exec(normalized); + return match + ? { office: compact(match[1] ?? normalized), district: match[2] } + : { office: normalized }; +} + +export function parseMissouriCandidateOffice( + html: string, + sourceUrl: string, +): MissouriCandidate[] { + const $ = cheerio.load(html); + if (!/Certified Candidate List/i.test($("#main h1").text())) { + throw new Error("Candidate office page was not a certified candidate list"); + } + const candidates: MissouriCandidate[] = []; + $("#main table").each((_tableIndex, table) => { + const party = compact($(table).find("caption").text()); + const parsedOffice = officeParts($(table).prevAll("h3").first().text()); + if (!parsedOffice.office) return; + $(table) + .find("tr") + .slice(1) + .each((rowIndex, row) => { + // Deliberately select only the public name cell. Address, filing date, + // and every other candidate contact/location field are ignored. + const name = compact($(row).find("td.NameCol").text()); + if (!name || !party) return; + candidates.push({ + ...parsedOffice, + party, + name, + ballotOrder: rowIndex + 1, + status: "certified", + citation: citation( + "Certified candidate list in ballot order", + sourceUrl, + ), + }); + }); + }); + return candidates; +} + +export function parseMissouriWithdrawals( + html: string, + sourceUrl: string, +): MissouriCandidate[] { + const $ = cheerio.load(html); + const candidates: MissouriCandidate[] = []; + $("table[title='Removed Candidates'] tr") + .slice(1) + .each((_index, row) => { + const cells = $(row).find("td"); + if (cells.length < 4) return; + const parsedOffice = officeParts($(cells[0]).text()); + // The bold span contains only name + party; the following address node is + // intentionally never read. + const identity = compact($(cells[1]).find("span").first().text()); + const match = /^(.*?)\s*\(([^()]+)\)$/.exec(identity); + const reason = compact($(cells[2]).text()); + const dateCell = $(cells[3]).clone(); + dateCell.find("br").replaceWith(" "); + const withdrawalDate = isoDate(compact(dateCell.text())) ?? undefined; + if (!match?.[1] || !match[2] || !reason) return; + candidates.push({ + ...parsedOffice, + name: compact(match[1]), + party: compact(match[2]), + ballotOrder: candidates.length + 1, + status: /withdraw/i.test(reason) ? "withdrawn" : "removed", + withdrawalReason: reason, + withdrawalDate, + citation: citation("Certified withdrawn/removed candidates", sourceUrl), + }); + }); + return candidates; +} + +interface MeasureDraft { + officialTitle: string; + electionDate: string; + fullTextUrl?: string; + certificateUrl?: string; + officialParagraphs: string[]; + fairParagraphs: string[]; +} + +export function parseMissouriBallotMeasures( + html: string, + sourceUrl = MISSOURI_MEASURES_URL, +): MissouriBallotMeasure[] { + const $ = cheerio.load(html); + let electionDate = ""; + let mode: "official" | "fair" | null = null; + let draft: MeasureDraft | null = null; + const drafts: MeasureDraft[] = []; + + $("#main") + .find("h2, p") + .each((_index, element) => { + const tag = element.tagName.toLowerCase(); + const text = compact($(element).text()); + if (tag === "h2" && /certified/i.test(text)) { + electionDate = isoDate(text) ?? electionDate; + return; + } + if (tag !== "p" || !text) return; + const title = + /^Official Ballot Title\s*(Amendment\s+\d+|Proposition\s+\w+)/i.exec( + text, + ); + if (title?.[1]) { + draft = { + officialTitle: compact(title[1]), + electionDate, + officialParagraphs: [], + fairParagraphs: [], + }; + drafts.push(draft); + mode = null; + return; + } + if (!draft) return; + const full = $(element).find("a[title*='full text' i]").attr("href"); + if (full) draft.fullTextUrl = absoluteUrl(full, sourceUrl); + const certificate = $(element) + .find("a[title*='Certificate of Official Ballot Title' i]") + .attr("href"); + if (certificate) + draft.certificateUrl = absoluteUrl(certificate, sourceUrl); + if (/^Official Ballot Title:?$/i.test(text)) { + mode = "official"; + return; + } + if (/^Fair Ballot Language:?$/i.test(text)) { + mode = "fair"; + return; + } + if (mode === "official") draft.officialParagraphs.push(text); + if (mode === "fair") draft.fairParagraphs.push(text); + }); + + return drafts.flatMap((item) => { + const fiscalStatement = item.officialParagraphs + .filter((paragraph) => + /(?:cost|saving|revenue|fiscal|tax|financial impact)/i.test(paragraph), + ) + .at(-1); + if ( + !item.electionDate || + !item.fullTextUrl || + !item.certificateUrl || + !item.officialParagraphs.length || + !item.fairParagraphs.length || + !fiscalStatement + ) { + return []; + } + return [ + { + officialTitle: item.officialTitle, + officialBallotLanguage: item.officialParagraphs.join("\n\n"), + fairBallotLanguage: item.fairParagraphs.join("\n\n"), + fiscalStatement, + electionDate: item.electionDate, + fullTextUrl: item.fullTextUrl, + certificateUrl: item.certificateUrl, + certificationStatus: "certified" as const, + citation: citation("Certified 2026 ballot measure", sourceUrl), + }, + ]; + }); +} + +export function discoverMissouriResultsUrl( + html: string, + activeElection: MissouriElection, + baseUrl = MISSOURI_RESULTS_URL, +): string | null { + const $ = cheerio.load(html); + const dateWords = new Date( + `${activeElection.electionDate}T12:00:00Z`, + ).toLocaleDateString("en-US", { + month: "long", + day: "numeric", + year: "numeric", + timeZone: "UTC", + }); + let found: string | null = null; + $("a[href]").each((_index, anchor) => { + if (found) return; + const href = $(anchor).attr("href"); + const text = compact($(anchor).text()); + const imageAlt = compact($(anchor).find("img").attr("alt") ?? ""); + const context = compact($(anchor).parent().text()); + const signal = `${text} ${imageAlt} ${context} ${href}`; + if (!href || !/result/i.test(signal)) return; + if (!/2026/.test(signal)) return; + if ( + !new RegExp(activeElection.type, "i").test(signal) && + !signal.includes(dateWords) + ) + return; + found = absoluteUrl(href, baseUrl); + }); + return found; +} + +function numeric(value: string): number { + const parsed = Number.parseInt(value.replace(/[^\d-]/g, ""), 10); + return Number.isFinite(parsed) ? parsed : 0; +} + +export function parseMissouriResults( + html: string, + sourceUrl: string, +): { contests: MissouriResultContest[]; updatedAt?: string } { + const $ = cheerio.load(html); + const contests: MissouriResultContest[] = []; + $("table").each((_tableIndex, table) => { + const headers = $(table) + .find("tr") + .first() + .find("th") + .map((_index, header) => compact($(header).text()).toLowerCase()) + .get(); + const nameIndex = headers.findIndex((header) => + /candidate|choice|option/.test(header), + ); + const votesIndex = headers.findIndex((header) => + /votes|total/.test(header), + ); + if (nameIndex < 0 || votesIndex < 0) return; + const partyIndex = headers.findIndex((header) => /party/.test(header)); + const percentIndex = headers.findIndex((header) => + /percent|%/.test(header), + ); + const contest = + compact($(table).find("caption").text()) || + compact($(table).prevAll("h2, h3, h4").first().text()); + if (!contest) return; + const choices: MissouriResultContest["choices"] = []; + $(table) + .find("tr") + .slice(1) + .each((_rowIndex, row) => { + const cells = $(row).find("td"); + const name = compact($(cells[nameIndex]).text()); + if (!name) return; + const percentText = + percentIndex >= 0 ? compact($(cells[percentIndex]).text()) : ""; + const parsedPercent = percentText + ? Number.parseFloat(percentText) + : undefined; + choices.push({ + name, + party: + partyIndex >= 0 + ? compact($(cells[partyIndex]).text()) || undefined + : undefined, + votes: numeric(compact($(cells[votesIndex]).text())), + percent: + parsedPercent !== undefined && Number.isFinite(parsedPercent) + ? parsedPercent + : undefined, + }); + }); + if (!choices.length) return; + const district = /District\s+(\d+)/i.exec(contest)?.[1]; + contests.push({ + contest, + district, + choices, + totalVotes: choices.reduce((sum, choice) => sum + choice.votes, 0), + reportingStatus: + compact($(table).attr("data-reporting-status") ?? "") || undefined, + citation: citation("Official Missouri SOS election results", sourceUrl), + }); + }); + if (!contests.length) { + throw new Error( + "Official results page contained no parseable contest tables", + ); + } + const pageText = compact($("body").text()); + const updatedAt = /(?:Last Updated|Updated)\s*:?\s*([^|]+?)(?:\||$)/i.exec( + pageText, + )?.[1]; + return { contests, updatedAt: updatedAt ? compact(updatedAt) : undefined }; +} diff --git a/apps/scraper/src/scrapers/disabled/missouri-sos.config.ts b/apps/scraper/src/scrapers/disabled/missouri-sos.config.ts new file mode 100644 index 00000000..27bff38f --- /dev/null +++ b/apps/scraper/src/scrapers/disabled/missouri-sos.config.ts @@ -0,0 +1,11 @@ +import type { ScraperEnvContract } from "@acme/env"; + +export const missouriSosConfig = { + id: "missouri-sos", + name: "Missouri SOS current election cycle", + source: "Missouri Secretary of State", + environment: { + required: ["POSTGRES_URL"], + optional: ["MISSOURI_SOS_MAX_ITEMS"], + }, +} as const satisfies ScraperEnvContract; diff --git a/apps/scraper/src/scrapers/disabled/missouri-sos.ts b/apps/scraper/src/scrapers/disabled/missouri-sos.ts new file mode 100644 index 00000000..3a59fcb9 --- /dev/null +++ b/apps/scraper/src/scrapers/disabled/missouri-sos.ts @@ -0,0 +1,250 @@ +/** Current-cycle-only Missouri SOS candidates, measures, and results. */ + +import { createHash } from "node:crypto"; +import { and, eq, ne } from "drizzle-orm"; + +import type { + MissouriResults, + MissouriSnapshotData, +} from "@acme/api/lib/missouri-election-data"; +import { + MISSOURI_CURRENT_SCOPE, + MISSOURI_CYCLE_YEAR, + MISSOURI_SOS_PROVIDER, +} from "@acme/api/lib/missouri-election-data"; +import { db } from "@acme/db/client"; +import { ElectionSourceSnapshot } from "@acme/db/schema"; + +import type { Scraper } from "../../utils/types.js"; +import { getItemLimit } from "../../utils/concurrency.js"; +import { fetchWithRetry } from "../../utils/fetch.js"; +import { createLogger } from "../../utils/log.js"; +import { + discoverMissouriResultsUrl, + MISSOURI_CALENDAR_URL, + MISSOURI_CANDIDATES_URL, + MISSOURI_MEASURES_URL, + MISSOURI_RESULTS_URL, + MISSOURI_STRUCTURE_VERSION, + parseMissouriBallotMeasures, + parseMissouriCalendar, + parseMissouriCandidateDiscovery, + parseMissouriCandidateOffice, + parseMissouriResults, + parseMissouriWithdrawals, +} from "./missouri-sos-parsers.js"; +import { missouriSosConfig } from "./missouri-sos.config.js"; + +const logger = createLogger("missouri-sos"); +const USER_AGENT = + "Mozilla/5.0 (compatible; BillionCivicBot/1.0; +https://billion.app)"; + +function sha256(value: string): string { + return createHash("sha256").update(value).digest("hex"); +} + +async function fetchOfficialHtml(url: string): Promise { + const response = await fetchWithRetry(url, { + headers: { Accept: "text/html", "User-Agent": USER_AGENT }, + timeoutMs: 45_000, + }); + return response.text(); +} + +async function upsertSnapshot( + data: MissouriSnapshotData, + diagnostics: string[], + sourceUrls: string[], + sourceVersion: string, +): Promise { + const canonicalData = JSON.stringify(data); + const contentHash = sha256(canonicalData); + const now = new Date(); + await db + .insert(ElectionSourceSnapshot) + .values({ + jurisdiction: "MO", + cycleYear: MISSOURI_CYCLE_YEAR, + provider: MISSOURI_SOS_PROVIDER, + scope: MISSOURI_CURRENT_SCOPE, + sourceVersion, + contentHash, + data: data as unknown as Record, + diagnostics, + sourceUrls, + fetchedAt: now, + }) + .onConflictDoUpdate({ + target: [ + ElectionSourceSnapshot.jurisdiction, + ElectionSourceSnapshot.cycleYear, + ElectionSourceSnapshot.provider, + ElectionSourceSnapshot.scope, + ], + set: { + sourceVersion, + contentHash, + data: data as unknown as Record, + diagnostics, + sourceUrls, + fetchedAt: now, + }, + }); + + // This invariant prevents any stale scope introduced outside this scraper + // from being mistaken for the supported current-cycle reader. + await db + .delete(ElectionSourceSnapshot) + .where( + and( + eq(ElectionSourceSnapshot.jurisdiction, "MO"), + eq(ElectionSourceSnapshot.provider, MISSOURI_SOS_PROVIDER), + eq(ElectionSourceSnapshot.scope, MISSOURI_CURRENT_SCOPE), + ne(ElectionSourceSnapshot.cycleYear, MISSOURI_CYCLE_YEAR), + ), + ); +} + +export async function scrapeMissouriSos(maxItems = 1000): Promise { + logger.info("Discovering official 2026 Missouri election sources…"); + const limit = getItemLimit(); + const [calendarHtml, candidateIndexHtml, measuresHtml, resultsEntryHtml] = + await Promise.all([ + limit(() => fetchOfficialHtml(MISSOURI_CALENDAR_URL)), + limit(() => fetchOfficialHtml(MISSOURI_CANDIDATES_URL)), + limit(() => fetchOfficialHtml(MISSOURI_MEASURES_URL)), + limit(() => fetchOfficialHtml(MISSOURI_RESULTS_URL)), + ]); + + const discovery = parseMissouriCandidateDiscovery(candidateIndexHtml); + const candidateElectionType = /general/i.test(discovery.electionName) + ? "general" + : /primary/i.test(discovery.electionName) + ? "primary" + : undefined; + const activeElection = parseMissouriCalendar( + calendarHtml, + new Date(), + MISSOURI_CALENDAR_URL, + candidateElectionType, + ); + if (!new RegExp(activeElection.type, "i").test(discovery.electionName)) { + throw new Error( + `Candidate system exposes ${discovery.electionName}, not active ${activeElection.name}`, + ); + } + + // The official cumulative endpoint contains every office and party in ballot + // order. It avoids a high-volume crawl of the individual office links. + const [candidateHtml, withdrawalHtml] = await Promise.all([ + limit(() => fetchOfficialHtml(discovery.candidatesUrl)), + limit(() => fetchOfficialHtml(discovery.withdrawalsUrl)), + ]); + const allCandidates = [ + ...parseMissouriCandidateOffice(candidateHtml, discovery.candidatesUrl), + ...parseMissouriWithdrawals(withdrawalHtml, discovery.withdrawalsUrl), + ]; + const candidates = allCandidates.slice(0, maxItems); + const ballotMeasures = parseMissouriBallotMeasures(measuresHtml); + const diagnostics: string[] = []; + if (candidates.length < allCandidates.length) { + diagnostics.push( + `MISSOURI_SOS_MAX_ITEMS limited candidates to ${candidates.length} of ${allCandidates.length}`, + ); + } + if (!ballotMeasures.length) { + throw new Error("Certified 2026 ballot-measure page contained no measures"); + } + + const resultUrl = discoverMissouriResultsUrl( + resultsEntryHtml, + activeElection, + ); + let results: MissouriResults; + if (!resultUrl) { + const diagnostic = `Official ${activeElection.name} results are not yet available from ShowMO Votes`; + diagnostics.push(diagnostic); + results = { + availability: "unavailable", + diagnostic, + citation: { + label: "ShowMO Votes results entry point", + sourceUrl: MISSOURI_RESULTS_URL, + }, + }; + logger.info(diagnostic); + } else { + try { + const resultHtml = await limit(() => fetchOfficialHtml(resultUrl)); + const parsed = parseMissouriResults(resultHtml, resultUrl); + results = { + availability: "available", + electionDate: activeElection.electionDate, + ...parsed, + citation: { + label: "Official Missouri SOS election results", + sourceUrl: resultUrl, + }, + }; + } catch (error) { + const diagnostic = `Official results link was discovered but results are not parseable yet: ${ + error instanceof Error ? error.message : String(error) + }`; + diagnostics.push(diagnostic); + results = { + availability: "unavailable", + diagnostic, + citation: { + label: "ShowMO Votes results entry point", + sourceUrl: MISSOURI_RESULTS_URL, + }, + }; + logger.warn(diagnostic); + } + } + + const sourceUrls = [ + MISSOURI_CANDIDATES_URL, + discovery.candidatesUrl, + discovery.withdrawalsUrl, + MISSOURI_MEASURES_URL, + MISSOURI_RESULTS_URL, + MISSOURI_CALENDAR_URL, + ...(resultUrl ? [resultUrl] : []), + ]; + const data: MissouriSnapshotData = { + cycleYear: MISSOURI_CYCLE_YEAR, + activeElection, + candidates, + ballotMeasures, + results, + citations: sourceUrls.map((sourceUrl) => ({ + label: "Missouri Secretary of State official source", + sourceUrl, + })), + }; + const sourceVersion = `${MISSOURI_STRUCTURE_VERSION}:${sha256( + JSON.stringify(data), + ).slice(0, 24)}`; + await upsertSnapshot( + data, + diagnostics, + [...new Set(sourceUrls)], + sourceVersion, + ); + logger.success( + `Missouri SOS: persisted ${candidates.length} candidates, ${ballotMeasures.length} measures, and ${ + results.availability === "available" + ? `${results.contests.length} result contests` + : "an unavailable-results diagnostic" + }.`, + ); +} + +export const missouriSos: Scraper = { + ...missouriSosConfig, + scrape: (options) => + scrapeMissouriSos( + (options?.maxItems ?? Number(process.env.MISSOURI_SOS_MAX_ITEMS)) || 1000, + ), +}; diff --git a/apps/scraper/src/scrapers/disabled/st-louis-aldermen-parser.ts b/apps/scraper/src/scrapers/disabled/st-louis-aldermen-parser.ts new file mode 100644 index 00000000..a51170c4 --- /dev/null +++ b/apps/scraper/src/scrapers/disabled/st-louis-aldermen-parser.ts @@ -0,0 +1,670 @@ +import { createHash } from "node:crypto"; +import * as cheerio from "cheerio"; +import { z } from "zod/v4"; + +const CITY_BASE = "https://www.stlouis-mo.gov"; +const AGENDA_INDEX = `${CITY_BASE}/government/departments/aldermen/aldermanic-legislative-session.cfm`; + +export interface StLouisSession { + id: string; + label: string; +} + +export interface StLouisCalendarMeeting { + eventId: string; + civicClerkId: string | null; + title: string; + startsAt: Date; + canonicalUrl: string; +} + +export interface StLouisDocument { + externalId: string; + type: "agenda" | "minutes" | "packet" | "attachment" | "bill-text"; + title: string; + url: string; + sourceVersion: string; + checksum: string; +} + +export interface StLouisLegislationRef { + kind: "board-bill" | "resolution"; + externalId: string; + number: string; + title: string; + sponsors: string[]; + sourceUrl: string; + action?: string; + introducedAt?: Date; + documents: StLouisDocument[]; +} + +export interface StLouisAgendaWeek { + agendaViewId: string; + sessionId: string; + week: number; + weekOf: string; + detailUrl: string; + documents: StLouisDocument[]; +} + +export interface StLouisAgendaDetail extends StLouisAgendaWeek { + eventId: string | null; + legislation: StLouisLegislationRef[]; +} + +export interface StLouisEventDetail { + eventId: string; + civicClerkId: string | null; + title: string; + meetingType: string; + location: string | null; + isCancelled: boolean; + videoUrl: string | null; + legislation: StLouisLegislationRef[]; +} + +const civicFileSchema = z + .object({ + fileId: z.number(), + type: z.string(), + publishedOn: z.string(), + name: z.string(), + url: z.url(), + }) + .passthrough(); + +const civicAttachmentSchema = z + .object({ + id: z.number(), + name: z.string(), + publicUrl: z.url(), + }) + .passthrough(); + +const civicFieldSchema = z + .object({ name: z.string(), value: z.string() }) + .passthrough(); + +export const stLouisCivicItemSchema: z.ZodType = z.lazy(() => + z + .object({ + id: z.number(), + idNumber: z.string().default(""), + itemName: z.string(), + agendaObjectItemOutlineNumber: z.string().nullable().default(""), + agendaObjItemCategoryTypeDesc: z.string().nullable().default(null), + agendaObjectItemDescription: z.string().nullable().default(null), + isSection: z.boolean().default(false), + customTextField8: civicFieldSchema.optional(), + recommendedActions: z.array(z.unknown()).default([]), + attachmentsList: z.array(civicAttachmentSchema).default([]), + childItems: z.array(stLouisCivicItemSchema).nullable().default(null), + }) + .passthrough(), +); + +export interface StLouisCivicItem { + id: number; + idNumber: string; + itemName: string; + agendaObjectItemOutlineNumber: string | null; + agendaObjItemCategoryTypeDesc: string | null; + agendaObjectItemDescription: string | null; + isSection: boolean; + customTextField8?: { name: string; value: string }; + recommendedActions: unknown[]; + attachmentsList: z.infer[]; + childItems: StLouisCivicItem[] | null; + [key: string]: unknown; +} + +export const stLouisCivicMeetingSchema = z + .object({ + id: z.number(), + name: z.string(), + meetingDate: z.string(), + files: z.array(civicFileSchema).default([]), + items: z.array(stLouisCivicItemSchema).default([]), + }) + .passthrough(); + +export type StLouisCivicMeeting = z.infer; + +export interface AdaptedStLouisItem { + externalId: string; + sequence: number; + itemNumber: string | null; + section: string | null; + itemType: string; + title: string; + description: string | null; + minutesNote: string | null; + action: string | null; + outcome: string | null; + legislativeId: string | null; + sponsors: string[]; + sourceVersion: string; + contentHash: string; + sourceUrl: string; + documents: StLouisDocument[]; +} + +export function hash(value: unknown): string { + return createHash("sha256").update(JSON.stringify(value)).digest("hex"); +} + +function cleanText(value: string): string { + return value + .replace(/\u00a0/g, " ") + .replace(/\s+/g, " ") + .trim(); +} + +function htmlText(value: string): string { + const spaced = value.replace(//gi, " "); + return cleanText(cheerio.load(`
${spaced}
`)("div").text()); +} + +function absoluteUrl(value: string, base = CITY_BASE): string { + return new URL(value, base).toString(); +} + +function stableUrlKey(value: string): string { + const url = new URL(value, CITY_BASE); + return `${url.origin}${url.pathname}`; +} + +function documentFromLink( + href: string, + type: StLouisDocument["type"], + title: string, + version = title, +): StLouisDocument { + const url = absoluteUrl(href); + const stable = stableUrlKey(url); + return { + externalId: `city:${hash(`${type}:${stable}`).slice(0, 24)}`, + type, + title: cleanText(title), + url, + sourceVersion: version, + checksum: hash({ stable, version }), + }; +} + +function selectedSession(html: string, selector: string): StLouisSession { + const $ = cheerio.load(html); + const option = $(`${selector} option[selected]`).first(); + const fallback = $(`${selector} option`).first(); + const selected = option.length > 0 ? option : fallback; + const id = selected.attr("value")?.trim(); + const label = cleanText(selected.text()); + if (!id || !/^\d{4}-\d{4}$/.test(label)) { + throw new Error("Unable to discover the active St. Louis session metadata"); + } + return { id, label }; +} + +export function parseActiveAgendaSession(html: string): StLouisSession { + return selectedSession(html, "#sessionYear"); +} + +export function parseActiveCalendarSession(html: string): StLouisSession { + return selectedSession(html, "#session"); +} + +function parseUsDate(value: string): string { + const match = value.match(/(\d{2})\/(\d{2})\/(\d{4})/); + if (!match) throw new Error(`Invalid St. Louis date: ${value}`); + return `${match[3]}-${match[1]}-${match[2]}`; +} + +function centralOffset(date: string): "-05:00" | "-06:00" { + const [year, month, day] = date.split("-").map(Number) as [ + number, + number, + number, + ]; + const nthSunday = (monthIndex: number, nth: number) => { + const first = new Date(Date.UTC(year, monthIndex, 1)).getUTCDay(); + return 1 + ((7 - first) % 7) + (nth - 1) * 7; + }; + const dstStart = nthSunday(2, 2); + const dstEnd = nthSunday(10, 1); + const daylight = + (month > 3 && month < 11) || + (month === 3 && day >= dstStart) || + (month === 11 && day < dstEnd); + return daylight ? "-05:00" : "-06:00"; +} + +export function parseStLouisStart(value: string): Date { + const date = parseUsDate(value); + const time = value.match(/(\d{1,2}):(\d{2})\s*([AP]M)/i); + if (!time) throw new Error(`Invalid St. Louis meeting time: ${value}`); + let hour = Number(time[1]) % 12; + if (time[3]!.toUpperCase() === "PM") hour += 12; + return new Date( + `${date}T${String(hour).padStart(2, "0")}:${time[2]}:00${centralOffset(date)}`, + ); +} + +export function parseStLouisCalendar(html: string): StLouisCalendarMeeting[] { + parseActiveCalendarSession(html); + const $ = cheerio.load(html); + const civicIds = new Map(); + const script = $.root().text() + $("script").text(); + const mapping = + /const eventID = "(\d+)";\s*const eventDate = "[^"]*";\s*const civicPlusMeetingID = "([^"]*)";/g; + for (const match of script.matchAll(mapping)) { + if (match[2]) civicIds.set(match[1]!, match[2]); + } + + const meetings = new Map(); + $(".event-list-details").each((_, element) => { + const link = $(element).find('a[href*="Event_ID="]').first(); + const href = link.attr("href"); + if (!href) return; + const canonicalUrl = absoluteUrl(href); + const eventId = new URL(canonicalUrl).searchParams.get("Event_ID"); + if (!eventId) return; + const dateText = cleanText($(element).find(".small").first().text()); + meetings.set(eventId, { + eventId, + civicClerkId: civicIds.get(eventId) ?? null, + title: cleanText(link.text()), + startsAt: parseStLouisStart(dateText), + canonicalUrl, + }); + }); + return [...meetings.values()].sort( + (left, right) => left.startsAt.getTime() - right.startsAt.getTime(), + ); +} + +function typeForLabel(label: string): StLouisDocument["type"] | null { + if (/agenda packet|packet/i.test(label)) return "packet"; + if (/agenda/i.test(label)) return "agenda"; + if (/minutes/i.test(label)) return "minutes"; + return null; +} + +export function parseStLouisAgendaIndex(html: string): StLouisAgendaWeek[] { + const session = parseActiveAgendaSession(html); + const $ = cheerio.load(html); + const rows: StLouisAgendaWeek[] = []; + $("table tbody tr").each((_, row) => { + const detailLink = $(row).find('a[href*="agendaViewID="]').first(); + const href = detailLink.attr("href"); + if (!href) return; + const detailUrl = absoluteUrl(href, AGENDA_INDEX); + const params = new URL(detailUrl).searchParams; + const agendaViewId = params.get("agendaViewID"); + const rowSession = params.get("session"); + const week = Number(params.get("detail")); + if (!agendaViewId || rowSession !== session.id || !Number.isInteger(week)) { + return; + } + const documents: StLouisDocument[] = []; + $(row) + .find("a[href]") + .each((__, anchor) => { + if (anchor === detailLink.get(0)) return; + const label = cleanText($(anchor).text()); + const type = typeForLabel(label); + const documentHref = $(anchor).attr("href"); + if (type && documentHref) { + documents.push( + documentFromLink( + documentHref, + type, + label, + cleanText($(anchor).parent().text()), + ), + ); + } + }); + rows.push({ + agendaViewId, + sessionId: session.id, + week, + weekOf: parseUsDate(cleanText(detailLink.text())), + detailUrl, + documents, + }); + }); + return rows; +} + +function parseLegislationTable( + $: cheerio.CheerioAPI, + selector: string, + kind: StLouisLegislationRef["kind"], +): StLouisLegislationRef[] { + const legislation: StLouisLegislationRef[] = []; + $(`${selector} tbody tr`).each((_, row) => { + const cells = $(row).find("td"); + const link = cells.eq(0).find("a[href]").first(); + const href = link.attr("href"); + if (!href) return; + const sourceUrl = absoluteUrl(href); + const params = new URL(sourceUrl).searchParams; + const externalId = + kind === "board-bill" ? params.get("BBId") : params.get("rsId"); + if (!externalId) return; + const cellText = cleanText(cells.eq(0).text()); + const number = cleanText(link.text()); + const title = cleanText( + cellText.slice(cellText.indexOf(number) + number.length), + ); + const sponsor = cleanText(cells.eq(1).text()); + legislation.push({ + kind, + externalId, + number, + title, + sponsors: sponsor ? [sponsor] : [], + sourceUrl, + documents: [], + }); + }); + return legislation; +} + +export function parseStLouisAgendaDetail( + html: string, + fallback: StLouisAgendaWeek, +): StLouisAgendaDetail { + const $ = cheerio.load(html); + const summary = cleanText($(".page-summary").first().text()); + const sessionLabel = summary.match(/Session\s+(\d{4}-\d{4})/i)?.[1]; + if (!sessionLabel) + throw new Error(`Agenda ${fallback.agendaViewId} has no session metadata`); + const eventHref = $(".event-list-details a") + .filter((_, link) => /full board/i.test($(link).text())) + .first() + .attr("href"); + const eventId = eventHref + ? new URL(absoluteUrl(eventHref)).searchParams.get("Event_ID") + : null; + const documents = [...fallback.documents]; + $(".download a[href]").each((_, link) => { + const href = $(link).attr("href"); + const label = cleanText($(link).text()); + const type = typeForLabel(label); + if ( + href && + type && + !documents.some( + (document) => stableUrlKey(document.url) === stableUrlKey(href), + ) + ) { + documents.push( + documentFromLink(href, type, label, cleanText($(link).parent().text())), + ); + } + }); + return { + ...fallback, + eventId, + documents, + legislation: [ + ...parseLegislationTable( + $, + 'table[summary*="Board bills"]', + "board-bill", + ), + ...parseLegislationTable( + $, + 'table[summary*="Resolutions"]', + "resolution", + ), + ], + }; +} + +function legislationLinks($: cheerio.CheerioAPI): StLouisLegislationRef[] { + const results = new Map(); + $('a[href*="boardbill.cfm"], a[href*="resolution.cfm"]').each((_, link) => { + const href = $(link).attr("href"); + if (!href) return; + const sourceUrl = absoluteUrl(href); + const params = new URL(sourceUrl).searchParams; + const isBill = params.has("BBId"); + const externalId = params.get(isBill ? "BBId" : "rsId"); + if (!externalId) return; + const kind = isBill ? "board-bill" : "resolution"; + const text = cleanText($(link).text()); + const number = text.match(/(?:Number\s+)?([0-9]+[A-Z]*)$/i)?.[1] ?? text; + results.set(`${kind}:${externalId}`, { + kind, + externalId, + number, + title: cleanText($(link).closest("p").next("p").text()), + sponsors: [], + sourceUrl, + documents: [], + }); + }); + return [...results.values()]; +} + +export function parseStLouisEventDetail( + html: string, + eventId: string, +): StLouisEventDetail { + const $ = cheerio.load(html); + const typeText = cleanText($(".MeetingType").text()); + const meetingType = + typeText.match(/Type:\s*(Aldermanic[^]*?Meeting)/i)?.[1] ?? + "Aldermanic Meeting"; + const script = $("script").text(); + const civicClerkId = + script.match(/const civicPlusMeetingID = "([^"]*)"/)?.[1] || null; + const locationText = cleanText($("#contact").next().text()); + const location = + locationText.match(/Location:\s*(.*?)(?:Contact|Hours|$)/i)?.[1]?.trim() || + null; + const title = cleanText($("h1").first().text()); + const marker = `${title} ${$("#description").text()}`; + const videoUrl = $("#EventDisplayBlock a[href]") + .toArray() + .map((link) => $(link).attr("href") ?? "") + .find((href) => /(?:youtube\.com|youtu\.be|stltv\.net)/i.test(href)); + return { + eventId, + civicClerkId, + title, + meetingType: cleanText(meetingType), + location, + isCancelled: /\bcancel(?:led|ed|lation)?\b/i.test(marker), + videoUrl: videoUrl ? absoluteUrl(videoUrl) : null, + legislation: legislationLinks($), + }; +} + +export function parseStLouisLegislationDetail( + html: string, + reference: StLouisLegislationRef, +): StLouisLegislationRef { + const $ = cheerio.load(html); + const heading = cleanText($("h1").first().text()); + const title = cleanText($(".page-summary").first().text()) || reference.title; + const body = cleanText($(".content-block").text()); + const sponsorsText = + body.match(/Primary Sponsors?:\s*(.*?)(?:Latest Activity|$)/i)?.[1] ?? ""; + const sponsors = $("strong") + .filter((_, element) => /Primary Sponsors?/i.test($(element).text())) + .parent() + .find('a[href*="profile.cfm"]') + .toArray() + .map((link) => cleanText($(link).text())) + .filter(Boolean); + if (sponsors.length === 0 && sponsorsText) + sponsors.push(cleanText(sponsorsText)); + const action = body + .match(/Latest Activity:\s*(.*?)(?:Legislative History|$)/i)?.[1] + ?.trim(); + const introduced = body.match(/Introduced:\s*(\d{2}\/\d{2}\/\d{4})/i)?.[1]; + const documents: StLouisDocument[] = []; + $(".download a[href], .content-block a.pdf[href]").each((_, link) => { + const href = $(link).attr("href"); + if (!href || !/\.pdf(?:$|\?)/i.test(href)) return; + const label = cleanText($(link).text()) || `${reference.kind} text`; + documents.push( + documentFromLink( + href, + "bill-text", + label, + cleanText($(link).parent().text()), + ), + ); + }); + if (!heading.includes(reference.number)) { + throw new Error( + `Legislation ${reference.externalId} number changed unexpectedly`, + ); + } + return { + ...reference, + title, + sponsors: sponsors.length > 0 ? sponsors : reference.sponsors, + ...(action ? { action } : {}), + ...(introduced + ? { introducedAt: new Date(`${parseUsDate(introduced)}T12:00:00Z`) } + : {}), + documents, + }; +} + +function civicDocument( + file: z.infer, +): StLouisDocument | null { + const type = typeForLabel(file.type); + if (!type) return null; + const sourceVersion = `${file.fileId}:${file.publishedOn}`; + return { + externalId: `civicclerk:${file.fileId}`, + type, + title: file.name || file.type, + url: file.url, + sourceVersion, + checksum: hash({ + fileId: file.fileId, + publishedOn: file.publishedOn, + name: file.name, + }), + }; +} + +export function civicMeetingDocuments(input: unknown): StLouisDocument[] { + return stLouisCivicMeetingSchema.parse(input).files.flatMap((file) => { + const document = civicDocument(file); + return document ? [document] : []; + }); +} + +function legislativeNumber(text: string): { + kind: StLouisLegislationRef["kind"]; + number: string; +} | null { + const bill = text.match(/Board Bill(?: Number)?\s+([0-9]+[A-Z]*)/i); + if (bill) return { kind: "board-bill", number: bill[1]! }; + const resolution = text.match(/Resolution(?: Number)?\s+([0-9]+[A-Z]*)/i); + return resolution ? { kind: "resolution", number: resolution[1]! } : null; +} + +export function adaptStLouisCivicItems( + input: unknown, + sourceUrl: string, + legislation: StLouisLegislationRef[] = [], +): AdaptedStLouisItem[] { + const meeting = stLouisCivicMeetingSchema.parse(input); + const flattened: { item: StLouisCivicItem; section: string | null }[] = []; + const visit = (items: StLouisCivicItem[], section: string | null) => { + for (const item of items) { + const title = htmlText(item.itemName); + flattened.push({ item, section }); + visit(item.childItems ?? [], item.isSection ? title : section); + } + }; + visit(meeting.items, null); + return flattened.map(({ item, section }, index) => { + const rawTitle = htmlText(item.itemName); + const marker = legislativeNumber(rawTitle); + const law = marker + ? legislation.find( + (entry) => + entry.kind === marker.kind && entry.number === marker.number, + ) + : undefined; + const attachments = item.attachmentsList.map((attachment) => ({ + externalId: `civicclerk:${attachment.id}`, + type: "attachment" as const, + title: attachment.name, + url: attachment.publicUrl, + sourceVersion: String(attachment.id), + checksum: hash({ id: attachment.id, name: attachment.name }), + })); + const itemType = + law?.kind ?? + item.agendaObjItemCategoryTypeDesc ?? + (item.isSection ? "section" : "agenda-item"); + const title = law?.title || rawTitle || "Untitled agenda item"; + const mapped = { + externalId: `civicclerk:${item.id}`, + sequence: index, + itemNumber: law?.number ?? (item.idNumber || null), + section, + itemType, + title, + description: + cleanText(item.agendaObjectItemDescription ?? "") || + (law && rawTitle !== title ? rawTitle : null), + minutesNote: cleanText(item.customTextField8?.value ?? "") || null, + action: law?.action ?? null, + outcome: null, + legislativeId: law ? `${law.kind}:${law.externalId}` : null, + sponsors: law?.sponsors ?? [], + sourceUrl: law?.sourceUrl ?? sourceUrl, + documents: [...attachments, ...(law?.documents ?? [])], + }; + return { + ...mapped, + sourceVersion: `stlouis-civicclerk-v1:${hash(item).slice(0, 24)}`, + contentHash: hash(mapped), + }; + }); +} + +export function legislationAgendaItems( + legislation: StLouisLegislationRef[], + startingSequence: number, +): AdaptedStLouisItem[] { + return legislation.map((law, index) => { + const mapped = { + externalId: `legislation:${law.kind}:${law.externalId}`, + sequence: startingSequence + index, + itemNumber: law.number, + section: law.kind === "board-bill" ? "Board Bills" : "Resolutions", + itemType: law.kind, + title: + law.title || + `${law.kind === "board-bill" ? "Board Bill" : "Resolution"} ${law.number}`, + description: null, + minutesNote: null, + action: law.action ?? null, + outcome: null, + legislativeId: `${law.kind}:${law.externalId}`, + sponsors: law.sponsors, + sourceUrl: law.sourceUrl, + documents: law.documents, + }; + return { + ...mapped, + sourceVersion: `stlouis-legislation-v1:${hash(law).slice(0, 24)}`, + contentHash: hash(mapped), + }; + }); +} diff --git a/apps/scraper/src/scrapers/disabled/st-louis-aldermen.config.ts b/apps/scraper/src/scrapers/disabled/st-louis-aldermen.config.ts new file mode 100644 index 00000000..e01b19a5 --- /dev/null +++ b/apps/scraper/src/scrapers/disabled/st-louis-aldermen.config.ts @@ -0,0 +1,12 @@ +import type { ScraperEnvContract } from "@acme/env"; + +export const stLouisAldermenConfig = { + id: "st-louis-aldermen", + name: "St. Louis Board of Aldermen", + source: + "City of St. Louis active aldermanic session pages and CivicClerk public API", + environment: { + required: ["POSTGRES_URL"], + optional: ["ST_LOUIS_ALDERMEN_MAX_ITEMS"], + }, +} as const satisfies ScraperEnvContract; diff --git a/apps/scraper/src/scrapers/disabled/st-louis-aldermen.ts b/apps/scraper/src/scrapers/disabled/st-louis-aldermen.ts new file mode 100644 index 00000000..570a4840 --- /dev/null +++ b/apps/scraper/src/scrapers/disabled/st-louis-aldermen.ts @@ -0,0 +1,498 @@ +import { z } from "zod/v4"; + +import { and, eq, notInArray } from "@acme/db"; +import { db } from "@acme/db/client"; +import { + LocalGovernmentAgendaItem, + LocalGovernmentDocument, + LocalGovernmentMeeting, +} from "@acme/db/schema"; + +import type { Scraper } from "../../utils/types.js"; +import type { + AdaptedStLouisItem, + StLouisAgendaDetail, + StLouisCalendarMeeting, + StLouisCivicMeeting, + StLouisDocument, + StLouisEventDetail, + StLouisLegislationRef, +} from "./st-louis-aldermen-parser.js"; +import { getItemLimit } from "../../utils/concurrency.js"; +import { setExpectedTotal } from "../../utils/db/metrics.js"; +import { fetchWithRetry } from "../../utils/fetch.js"; +import { createLogger } from "../../utils/log.js"; +import { + adaptStLouisCivicItems, + civicMeetingDocuments, + hash, + legislationAgendaItems, + parseActiveAgendaSession, + parseActiveCalendarSession, + parseStLouisAgendaDetail, + parseStLouisAgendaIndex, + parseStLouisCalendar, + parseStLouisEventDetail, + parseStLouisLegislationDetail, + stLouisCivicMeetingSchema, +} from "./st-louis-aldermen-parser.js"; +import { stLouisAldermenConfig } from "./st-louis-aldermen.config.js"; + +const CITY_BASE = "https://www.stlouis-mo.gov"; +const AGENDA_INDEX = `${CITY_BASE}/government/departments/aldermen/aldermanic-legislative-session.cfm`; +const CALENDAR = `${CITY_BASE}/government/departments/aldermen/events/`; +const CIVIC_CLERK_API = + "https://stlouismo.v8.civicclerk.com/public-api/Meetings"; +const PROVIDER = "stlouis-civicclerk"; +const JURISDICTION = "st-louis-mo"; +const TIMEZONE = "America/Chicago"; +const SOURCE_VERSION = "stlouis-aldermen-v1"; +const logger = createLogger(stLouisAldermenConfig.name); + +interface SourceMeeting { + calendar: StLouisCalendarMeeting; + detail: StLouisEventDetail; + civic: StLouisCivicMeeting | null; + agenda: StLouisAgendaDetail | null; + legislation: StLouisLegislationRef[]; +} + +interface AdaptedMeeting { + meeting: { + source: string; + sourceVersion: string; + jurisdiction: string; + governingBody: string; + externalId: string; + sessionId: string; + agendaViewId: string | null; + title: string; + meetingType: string; + status: string; + startsAt: Date; + timezone: string; + location: string | null; + isCancelled: boolean; + isAmended: boolean; + canonicalUrl: string; + videoUrl: string | null; + contentHash: string; + sourceUpdatedAt: Date | null; + }; + documents: StLouisDocument[]; + items: AdaptedStLouisItem[]; +} + +function mergeDocuments(documents: StLouisDocument[]): StLouisDocument[] { + const merged = new Map(); + for (const document of documents) { + merged.set(`${document.type}:${document.externalId}`, document); + } + return [...merged.values()]; +} + +function mergeLegislation( + references: StLouisLegislationRef[], +): StLouisLegislationRef[] { + const merged = new Map(); + for (const reference of references) { + const key = `${reference.kind}:${reference.externalId}`; + const previous = merged.get(key); + merged.set(key, { + ...previous, + ...reference, + title: reference.title || previous?.title || "", + sponsors: [ + ...new Set([...(previous?.sponsors ?? []), ...reference.sponsors]), + ], + documents: mergeDocuments([ + ...(previous?.documents ?? []), + ...reference.documents, + ]), + }); + } + return [...merged.values()]; +} + +async function fetchText(url: string): Promise { + const response = await fetchWithRetry(url, { + headers: { + "User-Agent": "Billion civic data scraper (https://billion.app)", + }, + timeoutMs: 30_000, + }); + return response.text(); +} + +async function fetchJson(url: URL): Promise { + const response = await fetchWithRetry(url.toString(), { + headers: { + Accept: "application/json", + "User-Agent": "Billion civic data scraper (https://billion.app)", + }, + timeoutMs: 30_000, + }); + return response.json() as Promise; +} + +function localDate(date: Date): string { + const parts = new Intl.DateTimeFormat("en-US", { + timeZone: TIMEZONE, + year: "numeric", + month: "2-digit", + day: "2-digit", + }).formatToParts(date); + const part = (type: Intl.DateTimeFormatPartTypes) => + parts.find((entry) => entry.type === type)?.value; + return `${part("year")}-${part("month")}-${part("day")}`; +} + +async function fetchCivicMeeting( + calendar: StLouisCalendarMeeting, + civicClerkId: string | null, +): Promise { + const id = civicClerkId ?? calendar.civicClerkId; + if (!id) return null; + const url = new URL(CIVIC_CLERK_API); + const date = localDate(calendar.startsAt); + url.searchParams.set("startDate", date); + url.searchParams.set("endDate", date); + const meetings = z + .array(stLouisCivicMeetingSchema) + .parse(await fetchJson(url)); + return meetings.find((meeting) => String(meeting.id) === id) ?? null; +} + +async function discoverSourceMeetings(maxItems: number): Promise<{ + sessionId: string; + sessionLabel: string; + meetings: SourceMeeting[]; +}> { + const [agendaHtml, calendarHtml] = await Promise.all([ + fetchText(AGENDA_INDEX), + fetchText(CALENDAR), + ]); + const agendaSession = parseActiveAgendaSession(agendaHtml); + const calendarSession = parseActiveCalendarSession(calendarHtml); + if ( + agendaSession.id !== calendarSession.id || + agendaSession.label !== calendarSession.label + ) { + throw new Error( + `St. Louis active-session metadata disagrees: agenda=${agendaSession.id}/${agendaSession.label}, calendar=${calendarSession.id}/${calendarSession.label}`, + ); + } + + // Both source URLs default to the selected active session. Deliberately do + // not submit the archive selectors or construct historical session URLs. + const calendarMeetings = parseStLouisCalendar(calendarHtml).slice( + 0, + maxItems, + ); + const agendaWeeks = parseStLouisAgendaIndex(agendaHtml).slice(0, maxItems); + const limit = getItemLimit(); + const agendaDetails = await Promise.all( + agendaWeeks.map((week) => + limit(async () => + parseStLouisAgendaDetail(await fetchText(week.detailUrl), week), + ), + ), + ); + const agendasByEvent = new Map( + agendaDetails.flatMap((agenda) => + agenda.eventId ? [[agenda.eventId, agenda] as const] : [], + ), + ); + + const eventSources = await Promise.all( + calendarMeetings.map((calendar) => + limit(async () => { + const detail = parseStLouisEventDetail( + await fetchText(calendar.canonicalUrl), + calendar.eventId, + ); + if ( + detail.civicClerkId && + calendar.civicClerkId && + detail.civicClerkId !== calendar.civicClerkId + ) { + throw new Error( + `Event ${calendar.eventId} has conflicting CivicClerk IDs ${calendar.civicClerkId}/${detail.civicClerkId}`, + ); + } + const civic = await fetchCivicMeeting(calendar, detail.civicClerkId); + return { + calendar, + detail, + civic, + agenda: agendasByEvent.get(calendar.eventId) ?? null, + }; + }), + ), + ); + + const references = mergeLegislation( + eventSources.flatMap((source) => [ + ...source.detail.legislation, + ...(source.agenda?.legislation ?? []), + ]), + ); + const enriched = await Promise.all( + references.map((reference) => + limit(async () => { + try { + return parseStLouisLegislationDetail( + await fetchText(reference.sourceUrl), + reference, + ); + } catch (error) { + logger.warn( + `Using meeting-page metadata for ${reference.kind} ${reference.number}`, + error, + ); + return reference; + } + }), + ), + ); + const legislationById = new Map( + enriched.map((reference) => [ + `${reference.kind}:${reference.externalId}`, + reference, + ]), + ); + + return { + sessionId: agendaSession.id, + sessionLabel: agendaSession.label, + meetings: eventSources.map((source) => ({ + ...source, + legislation: mergeLegislation( + [ + ...source.detail.legislation, + ...(source.agenda?.legislation ?? []), + ].map( + (reference) => + legislationById.get(`${reference.kind}:${reference.externalId}`) ?? + reference, + ), + ), + })), + }; +} + +function adaptMeeting( + source: SourceMeeting, + sessionId: string, + sessionLabel: string, + now = new Date(), +): AdaptedMeeting { + const civicItems = source.civic + ? adaptStLouisCivicItems( + source.civic, + source.calendar.canonicalUrl, + source.legislation, + ) + : []; + const linkedLegislation = new Set( + civicItems.flatMap((item) => + item.legislativeId ? [item.legislativeId] : [], + ), + ); + const additionalItems = legislationAgendaItems( + source.legislation.filter( + (law) => !linkedLegislation.has(`${law.kind}:${law.externalId}`), + ), + civicItems.length, + ); + const items = [...civicItems, ...additionalItems]; + const documents = mergeDocuments([ + ...(source.agenda?.documents ?? []), + ...(source.civic ? civicMeetingDocuments(source.civic) : []), + ...items.flatMap((item) => item.documents), + ]); + const sourceUpdatedAt = + source.civic?.files.reduce((latest, file) => { + const published = new Date(file.publishedOn); + return !latest || published > latest ? published : latest; + }, null) ?? null; + const sourceVersion = [ + SOURCE_VERSION, + `session:${sessionId}`, + `event:${source.calendar.eventId}`, + `civic:${source.civic?.id ?? source.detail.civicClerkId ?? "none"}`, + `agenda:${source.agenda?.agendaViewId ?? "none"}`, + ].join(":"); + const status = source.detail.isCancelled + ? "cancelled" + : source.calendar.startsAt < now + ? "completed" + : "scheduled"; + const governingBody = /committee/i.test(source.detail.meetingType) + ? source.detail.title + : "St. Louis Board of Aldermen"; + const mapped = { + source: PROVIDER, + sourceVersion, + jurisdiction: JURISDICTION, + governingBody, + externalId: `event:${source.calendar.eventId}`, + sessionId, + agendaViewId: source.agenda?.agendaViewId ?? null, + title: source.detail.title || source.calendar.title, + meetingType: source.detail.meetingType, + status, + startsAt: source.calendar.startsAt, + timezone: TIMEZONE, + location: source.detail.location, + isCancelled: source.detail.isCancelled, + isAmended: documents.some((document) => + /\b(amended|revised|updated|corrected)\b/i.test( + `${document.title} ${document.sourceVersion}`, + ), + ), + canonicalUrl: source.calendar.canonicalUrl, + videoUrl: source.detail.videoUrl, + sourceUpdatedAt, + }; + return { + meeting: { + ...mapped, + contentHash: hash({ + ...mapped, + sessionLabel, + documents: documents.map((document) => ({ + id: document.externalId, + version: document.sourceVersion, + checksum: document.checksum, + })), + items: items.map((item) => item.contentHash), + }), + }, + documents, + items, + }; +} + +async function persistMeeting(adapted: AdaptedMeeting): Promise { + const fetchedAt = new Date(); + await db.transaction(async (tx) => { + const [stored] = await tx + .insert(LocalGovernmentMeeting) + .values({ ...adapted.meeting, fetchedAt }) + .onConflictDoUpdate({ + target: [ + LocalGovernmentMeeting.source, + LocalGovernmentMeeting.jurisdiction, + LocalGovernmentMeeting.externalId, + ], + set: { ...adapted.meeting, fetchedAt }, + }) + .returning({ id: LocalGovernmentMeeting.id }); + if (!stored) { + throw new Error( + `Failed to persist St. Louis meeting ${adapted.meeting.externalId}`, + ); + } + + await tx + .update(LocalGovernmentDocument) + .set({ isCurrent: false }) + .where(eq(LocalGovernmentDocument.meetingId, stored.id)); + for (const document of adapted.documents) { + await tx + .insert(LocalGovernmentDocument) + .values({ + meetingId: stored.id, + externalId: document.externalId, + sourceVersion: document.sourceVersion, + type: document.type, + title: document.title, + url: document.url, + mediaType: "application/pdf", + checksum: document.checksum, + isCurrent: true, + fetchedAt, + }) + .onConflictDoUpdate({ + target: [ + LocalGovernmentDocument.meetingId, + LocalGovernmentDocument.type, + LocalGovernmentDocument.externalId, + ], + set: { + sourceVersion: document.sourceVersion, + title: document.title, + url: document.url, + mediaType: "application/pdf", + checksum: document.checksum, + isCurrent: true, + fetchedAt, + }, + }); + } + + for (const item of adapted.items) { + const { documents: _documents, ...itemRow } = item; + await tx + .insert(LocalGovernmentAgendaItem) + .values({ ...itemRow, meetingId: stored.id }) + .onConflictDoUpdate({ + target: [ + LocalGovernmentAgendaItem.meetingId, + LocalGovernmentAgendaItem.externalId, + ], + set: itemRow, + }); + } + const externalIds = adapted.items.map((item) => item.externalId); + await tx + .delete(LocalGovernmentAgendaItem) + .where( + externalIds.length > 0 + ? and( + eq(LocalGovernmentAgendaItem.meetingId, stored.id), + notInArray(LocalGovernmentAgendaItem.externalId, externalIds), + ) + : eq(LocalGovernmentAgendaItem.meetingId, stored.id), + ); + }); +} + +async function scrape(maxItems: number): Promise { + logger.info( + "Discovering selected active aldermanic session (structured HTML/JSON only; archives, AI, and OCR disabled)", + ); + const discovery = await discoverSourceMeetings(maxItems); + setExpectedTotal(discovery.meetings.length); + logger.info( + `Syncing ${discovery.meetings.length} meetings from active session ${discovery.sessionLabel} (${discovery.sessionId})`, + ); + for (const source of discovery.meetings) { + try { + const adapted = adaptMeeting( + source, + discovery.sessionId, + discovery.sessionLabel, + ); + await persistMeeting(adapted); + logger.success( + `Synced ${adapted.meeting.title} (${adapted.meeting.externalId})`, + ); + } catch (error) { + logger.error( + `Meeting event:${source.calendar.eventId} failed without aborting run`, + error, + ); + } + } +} + +export const stLouisAldermen: Scraper = { + ...stLouisAldermenConfig, + scrape: (options) => + scrape( + (options?.maxItems ?? Number(process.env.ST_LOUIS_ALDERMEN_MAX_ITEMS)) || + 100, + ), +}; diff --git a/apps/scraper/src/scrapers/durham-bocc.config.ts b/apps/scraper/src/scrapers/durham-bocc.config.ts new file mode 100644 index 00000000..882a29a2 --- /dev/null +++ b/apps/scraper/src/scrapers/durham-bocc.config.ts @@ -0,0 +1,11 @@ +import type { ScraperEnvContract } from "@acme/env"; + +export const durhamBoccConfig = { + id: "durham-bocc", + name: "Durham County BOCC", + source: "Durham County Legistar Web API — BOCC meetings and actions", + environment: { + required: ["POSTGRES_URL"], + optional: ["DURHAM_BOCC_MAX_ITEMS"], + }, +} as const satisfies ScraperEnvContract; diff --git a/apps/scraper/src/scrapers/durham-bocc.test.ts b/apps/scraper/src/scrapers/durham-bocc.test.ts new file mode 100644 index 00000000..aab5679a --- /dev/null +++ b/apps/scraper/src/scrapers/durham-bocc.test.ts @@ -0,0 +1,74 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import test from "node:test"; + +import { + adaptDurhamItem, + adaptDurhamMeeting, + adaptDurhamVote, + currentElectionCycleStart, + parseDurhamStart, +} from "./durham-bocc.js"; + +async function fixture(name: string): Promise { + const url = new URL(`./fixtures/durham-bocc/${name}.json`, import.meta.url); + return JSON.parse(await readFile(url, "utf8")) as unknown; +} + +void test("maps regular, work, cancelled, and amended meetings", async () => { + const events = (await fixture("events")) as unknown[]; + const [regular, work, cancelled, amended] = events.map(adaptDurhamMeeting); + + assert.equal(regular?.meetingType, "Regular Session"); + assert.equal(regular?.isCancelled, false); + assert.match(regular?.videoUrl ?? "", /ID1=1539/); + assert.equal(work?.meetingType, "Work Session"); + assert.equal(cancelled?.isCancelled, true); + assert.equal(cancelled?.status, "cancelled"); + assert.equal(amended?.isAmended, true); + assert.equal(amended?.documents[0]?.title, "Amended Agenda"); +}); + +void test("maps actions, consent status, official attachments, and named votes", async () => { + const [rawItem] = (await fixture("items")) as unknown[]; + const [rawVote] = (await fixture("votes")) as unknown[]; + const item = adaptDurhamItem(rawItem); + const vote = adaptDurhamVote(rawVote); + + assert.equal(item.consent, true); + assert.equal(item.action, "Approved"); + assert.equal(item.outcome, "Passed"); + assert.equal(item.voteSummary, "5-0"); + assert.equal(item.documents.length, 2); + assert.equal(item.documents[1]?.language, "es"); + assert.equal(vote.voterName, "Commissioner A"); + assert.equal(vote.value, "Aye"); +}); + +void test("uses stable source ids and changes checksum when an amendment changes", async () => { + const [raw] = (await fixture("events")) as Record[]; + const original = adaptDurhamMeeting(raw); + const replaced = adaptDurhamMeeting({ + ...raw, + EventRowVersion: "replacement-v2", + EventLastModifiedUtc: "2026-01-13T00:00:00.000", + }); + + assert.equal(original.externalId, replaced.externalId); + assert.notEqual(original.contentHash, replaced.contentHash); +}); + +void test("parses Durham local time with DST and bounds to the current cycle", () => { + assert.equal( + parseDurhamStart("2026-01-12T00:00:00", "6:00 PM").toISOString(), + "2026-01-12T23:00:00.000Z", + ); + assert.equal( + parseDurhamStart("2026-07-13T00:00:00", "5:00 PM").toISOString(), + "2026-07-13T21:00:00.000Z", + ); + assert.equal( + currentElectionCycleStart(new Date("2026-07-21T00:00:00Z")).toISOString(), + "2025-01-01T00:00:00.000Z", + ); +}); diff --git a/apps/scraper/src/scrapers/durham-bocc.ts b/apps/scraper/src/scrapers/durham-bocc.ts new file mode 100644 index 00000000..0c64762d --- /dev/null +++ b/apps/scraper/src/scrapers/durham-bocc.ts @@ -0,0 +1,539 @@ +import { createHash } from "node:crypto"; +import { z } from "zod/v4"; + +import { and, eq } from "@acme/db"; +import { db } from "@acme/db/client"; +import { + LocalGovernmentAgendaItem, + LocalGovernmentDocument, + LocalGovernmentMeeting, + LocalGovernmentVote, +} from "@acme/db/schema"; + +import type { Scraper } from "../utils/types.js"; +import { getItemLimit } from "../utils/concurrency.js"; +import { setExpectedTotal } from "../utils/db/metrics.js"; +import { fetchWithRetry } from "../utils/fetch.js"; +import { createLogger } from "../utils/log.js"; +import { durhamBoccConfig } from "./durham-bocc.config.js"; + +const API_BASE = "https://webapi.legistar.com/v1/durhamcounty"; +const SITE_BASE = "https://durhamcounty.legistar.com"; +const PROVIDER = "legistar"; +const JURISDICTION = "durham-county-nc"; +const BOCC_BODY_ID = 138; +const SOURCE_VERSION = "durham-legistar-v1"; +const TIMEZONE = "America/New_York"; +const logger = createLogger(durhamBoccConfig.name); + +const attachmentSchema = z + .object({ + MatterAttachmentId: z.number(), + MatterAttachmentName: z.string(), + MatterAttachmentHyperlink: z.string().nullable(), + MatterAttachmentLastModifiedUtc: z.string(), + MatterAttachmentShowOnInternetPage: z.boolean().optional(), + }) + .passthrough(); + +export const durhamEventSchema = z + .object({ + EventId: z.number(), + EventGuid: z.string(), + EventLastModifiedUtc: z.string(), + EventRowVersion: z.string(), + EventBodyId: z.number(), + EventBodyName: z.string(), + EventDate: z.string(), + EventTime: z.string().nullable(), + EventAgendaStatusName: z.string().nullable(), + EventMinutesStatusName: z.string().nullable(), + EventLocation: z.string().nullable(), + EventAgendaFile: z.string().nullable(), + EventMinutesFile: z.string().nullable(), + EventComment: z.string().nullable(), + EventVideoPath: z.string().nullable(), + EventMedia: z.union([z.string(), z.number()]).nullable().optional(), + EventInSiteURL: z.string().nullable(), + }) + .passthrough(); + +export const durhamItemSchema = z + .object({ + EventItemId: z.number(), + EventItemLastModifiedUtc: z.string(), + EventItemRowVersion: z.string(), + EventItemEventId: z.number(), + EventItemAgendaSequence: z.number(), + EventItemAgendaNumber: z.string().nullable(), + EventItemAgendaNote: z.string().nullable(), + EventItemMinutesNote: z.string().nullable(), + EventItemActionName: z.string().nullable(), + EventItemActionText: z.string().nullable(), + EventItemPassedFlagName: z.string().nullable(), + EventItemRollCallFlag: z.number().nullable(), + EventItemTitle: z.string().nullable(), + EventItemTally: z.string().nullable(), + EventItemConsent: z.number(), + EventItemMover: z.string().nullable(), + EventItemSeconder: z.string().nullable(), + EventItemMatterId: z.number().nullable(), + EventItemMatterAttachments: z.array(attachmentSchema).nullable(), + }) + .passthrough(); + +export const durhamVoteSchema = z + .object({ + VoteId: z.number(), + VoteLastModifiedUtc: z.string(), + VotePersonId: z.number(), + VotePersonName: z.string(), + VoteValueName: z.string(), + VoteSort: z.number(), + VoteEventItemId: z.number(), + }) + .passthrough(); + +type DurhamEvent = z.infer; +type DurhamItem = z.infer; +type DurhamVote = z.infer; +interface DurhamDocument { + kind: "agenda" | "minutes" | "attachment"; + title: string; + url: string; + language?: string; +} + +function hash(value: unknown): string { + return createHash("sha256").update(JSON.stringify(value)).digest("hex"); +} + +function datePart(value: string): string { + const match = /^(\d{4}-\d{2}-\d{2})/.exec(value); + if (!match) throw new Error(`Invalid Legistar date: ${value}`); + return match[1]!; +} + +function nthSunday(year: number, monthIndex: number, nth: number): number { + const first = new Date(Date.UTC(year, monthIndex, 1)).getUTCDay(); + return 1 + ((7 - first) % 7) + (nth - 1) * 7; +} + +function easternUtcOffset(date: string): "-04:00" | "-05:00" { + const [year, month, day] = date.split("-").map(Number) as [ + number, + number, + number, + ]; + const dstStart = nthSunday(year, 2, 2); + const dstEnd = nthSunday(year, 10, 1); + const isDst = + (month > 3 && month < 11) || + (month === 3 && day >= dstStart) || + (month === 11 && day < dstEnd); + return isDst ? "-04:00" : "-05:00"; +} + +export function parseDurhamStart( + eventDate: string, + eventTime: string | null, +): Date { + const date = datePart(eventDate); + const match = eventTime?.trim().match(/^(\d{1,2}):(\d{2})\s*([AP]M)$/i); + let hour = 0; + let minute = 0; + if (match) { + hour = Number(match[1]) % 12; + minute = Number(match[2]); + if (match[3]!.toUpperCase() === "PM") hour += 12; + } + return new Date( + `${date}T${String(hour).padStart(2, "0")}:${String(minute).padStart(2, "0")}:00${easternUtcOffset(date)}`, + ); +} + +export function currentElectionCycleStart(now = new Date()): Date { + const year = now.getUTCFullYear(); + const startYear = year % 2 === 0 ? year - 1 : year; + return new Date(Date.UTC(startYear, 0, 1)); +} + +function documentLanguage(title: string): string | undefined { + return /\b(spanish|español|espanol)\b/i.test(title) ? "es" : undefined; +} + +function visibleAttachmentDocuments(item: DurhamItem): DurhamDocument[] { + const kind = /\bminutes?\b/i.test(item.EventItemTitle ?? "") + ? ("minutes" as const) + : ("attachment" as const); + return (item.EventItemMatterAttachments ?? []) + .filter( + (attachment) => + attachment.MatterAttachmentHyperlink && + attachment.MatterAttachmentShowOnInternetPage !== false, + ) + .map((attachment) => ({ + kind, + title: attachment.MatterAttachmentName, + url: attachment.MatterAttachmentHyperlink!, + ...(documentLanguage(attachment.MatterAttachmentName) + ? { language: documentLanguage(attachment.MatterAttachmentName) } + : {}), + })); +} + +export function adaptDurhamMeeting(input: unknown) { + const event = durhamEventSchema.parse(input); + const comment = event.EventComment?.trim() || "Board meeting"; + const markerText = [ + comment, + event.EventAgendaStatusName, + event.EventMinutesStatusName, + event.EventAgendaFile, + event.EventMinutesFile, + ] + .filter(Boolean) + .join(" "); + const isCancelled = /\bcancel(?:led|ed|lation)?\b/i.test(markerText); + const isAmended = /\b(amend(?:ed|ment)?|revis(?:ed|ion)|corrected)\b/i.test( + markerText, + ); + const documents: DurhamDocument[] = []; + if (event.EventAgendaFile) { + documents.push({ + kind: "agenda", + title: `${isAmended ? "Amended " : ""}Agenda`, + url: event.EventAgendaFile, + }); + } + if (event.EventMinutesFile) { + documents.push({ + kind: "minutes", + title: "Minutes", + url: event.EventMinutesFile, + }); + } + const sourceUrl = + event.EventInSiteURL ?? + `${SITE_BASE}/MeetingDetail.aspx?LEGID=${event.EventId}`; + const videoUrl = event.EventVideoPath + ? event.EventVideoPath + : event.EventMedia + ? `${SITE_BASE}/Video.aspx?Mode=Granicus&ID1=${event.EventMedia}&Mode2=Video` + : null; + const status = isCancelled + ? "cancelled" + : (event.EventMinutesStatusName ?? + event.EventAgendaStatusName ?? + "scheduled"); + const mapped = { + source: PROVIDER, + jurisdiction: JURISDICTION, + externalId: String(event.EventId), + governingBody: event.EventBodyName, + title: comment, + meetingType: comment.replace(/^cancel(?:led|ed)\s*-?\s*/i, ""), + startsAt: parseDurhamStart(event.EventDate, event.EventTime), + timezone: TIMEZONE, + location: event.EventLocation, + status, + isCancelled, + isAmended, + canonicalUrl: sourceUrl, + videoUrl, + documents, + sourceVersion: `${SOURCE_VERSION}:${event.EventRowVersion}`, + sourceUpdatedAt: new Date(event.EventLastModifiedUtc), + }; + return { ...mapped, contentHash: hash(mapped) }; +} + +export function adaptDurhamItem(input: unknown) { + const item = durhamItemSchema.parse(input); + const mapped = { + externalId: String(item.EventItemId), + meetingExternalId: String(item.EventItemEventId), + sequence: item.EventItemAgendaSequence, + itemNumber: item.EventItemAgendaNumber, + itemType: item.EventItemActionName ?? "agenda-item", + title: item.EventItemTitle?.trim() || "Untitled agenda item", + description: item.EventItemAgendaNote, + minutesNote: item.EventItemMinutesNote, + consent: item.EventItemConsent === 1, + action: item.EventItemActionName, + motion: item.EventItemActionText, + outcome: item.EventItemPassedFlagName, + voteSummary: item.EventItemTally, + mover: item.EventItemMover, + seconder: item.EventItemSeconder, + documents: visibleAttachmentDocuments(item), + sourceVersion: `${SOURCE_VERSION}:${item.EventItemRowVersion}`, + sourceUpdatedAt: new Date(item.EventItemLastModifiedUtc), + sourceUrl: `${SITE_BASE}/MeetingDetail.aspx?LEGID=${item.EventItemEventId}&GID=174`, + }; + return { ...mapped, contentHash: hash(mapped) }; +} + +export function adaptDurhamVote(input: unknown) { + const vote = durhamVoteSchema.parse(input); + return { + externalId: String(vote.VoteId), + itemExternalId: String(vote.VoteEventItemId), + voterExternalId: String(vote.VotePersonId), + voterName: vote.VotePersonName, + value: vote.VoteValueName, + sort: vote.VoteSort, + sourceUpdatedAt: new Date(vote.VoteLastModifiedUtc), + }; +} + +async function fetchJson(url: URL): Promise { + const response = await fetchWithRetry(url.toString(), { timeoutMs: 30_000 }); + return response.json() as Promise; +} + +async function upsertMeeting(meeting: ReturnType) { + const fetchedAt = new Date(); + const { documents, ...meetingRow } = meeting; + const [stored] = await db + .insert(LocalGovernmentMeeting) + .values({ ...meetingRow, fetchedAt }) + .onConflictDoUpdate({ + target: [ + LocalGovernmentMeeting.source, + LocalGovernmentMeeting.jurisdiction, + LocalGovernmentMeeting.externalId, + ], + set: { ...meetingRow, fetchedAt }, + }) + .returning({ id: LocalGovernmentMeeting.id }); + if (!stored) + throw new Error(`Failed to persist meeting ${meeting.externalId}`); + await upsertDocuments(stored.id, documents); + return stored.id; +} + +async function upsertDocuments(meetingId: string, documents: DurhamDocument[]) { + for (const document of documents) { + await db + .insert(LocalGovernmentDocument) + .values({ + meetingId, + type: document.kind, + title: document.language + ? `${document.title} (${document.language})` + : document.title, + url: document.url, + fetchedAt: new Date(), + }) + .onConflictDoUpdate({ + target: [ + LocalGovernmentDocument.meetingId, + LocalGovernmentDocument.type, + LocalGovernmentDocument.url, + ], + set: { + title: document.language + ? `${document.title} (${document.language})` + : document.title, + isCurrent: true, + fetchedAt: new Date(), + }, + }); + } +} + +async function upsertItem( + meetingId: string, + item: ReturnType, +) { + const { documents } = item; + const itemRow = { + externalId: item.externalId, + sequence: item.sequence, + itemNumber: item.itemNumber, + itemType: item.itemType, + title: item.title, + description: item.description, + minutesNote: item.minutesNote, + consent: item.consent, + action: item.action, + motion: item.motion, + outcome: item.outcome, + voteSummary: item.voteSummary, + mover: item.mover, + seconder: item.seconder, + sourceVersion: item.sourceVersion, + contentHash: item.contentHash, + sourceUpdatedAt: item.sourceUpdatedAt, + sourceUrl: item.sourceUrl, + }; + const [stored] = await db + .insert(LocalGovernmentAgendaItem) + .values({ ...itemRow, meetingId }) + .onConflictDoUpdate({ + target: [ + LocalGovernmentAgendaItem.meetingId, + LocalGovernmentAgendaItem.externalId, + ], + set: itemRow, + }) + .returning({ id: LocalGovernmentAgendaItem.id }); + if (!stored) throw new Error(`Failed to persist item ${item.externalId}`); + await upsertDocuments(meetingId, documents); + return stored.id; +} + +async function upsertVote( + agendaItemId: string, + vote: ReturnType, +) { + const voteRow = { + externalId: vote.externalId, + voterExternalId: vote.voterExternalId, + voterName: vote.voterName, + value: vote.value, + sort: vote.sort, + sourceUpdatedAt: vote.sourceUpdatedAt, + }; + await db + .insert(LocalGovernmentVote) + .values({ ...voteRow, agendaItemId, fetchedAt: new Date() }) + .onConflictDoUpdate({ + target: [LocalGovernmentVote.agendaItemId, LocalGovernmentVote.voterName], + set: { ...voteRow, fetchedAt: new Date() }, + }); +} + +async function scrapeMeeting(rawEvent: unknown): Promise { + const meeting = adaptDurhamMeeting(rawEvent); + const meetingId = await upsertMeeting(meeting); + + const itemsUrl = new URL( + `${API_BASE}/Events/${meeting.externalId}/EventItems`, + ); + itemsUrl.searchParams.set("AgendaNote", "1"); + itemsUrl.searchParams.set("MinutesNote", "1"); + itemsUrl.searchParams.set("Attachments", "1"); + const rawItems = z.array(z.unknown()).parse(await fetchJson(itemsUrl)); + + for (const rawItem of rawItems) { + try { + const parsed = durhamItemSchema.parse(rawItem); + let enrichedItem: unknown = rawItem; + if ( + parsed.EventItemMatterId && + (parsed.EventItemMatterAttachments?.length ?? 0) === 0 + ) { + try { + const attachmentsUrl = new URL( + `${API_BASE}/Matters/${parsed.EventItemMatterId}/Attachments`, + ); + const attachments = z + .array(attachmentSchema) + .parse(await fetchJson(attachmentsUrl)); + enrichedItem = { + ...parsed, + EventItemMatterAttachments: attachments, + }; + } catch (error) { + logger.warn( + `Attachments unavailable for matter ${parsed.EventItemMatterId}`, + error, + ); + } + } + const item = adaptDurhamItem(enrichedItem); + const agendaItemId = await upsertItem(meetingId, item); + if (parsed.EventItemRollCallFlag === 1) { + const votesUrl = new URL( + `${API_BASE}/EventItems/${item.externalId}/Votes`, + ); + const rawVotes = z.array(z.unknown()).parse(await fetchJson(votesUrl)); + for (const rawVote of rawVotes) { + try { + await upsertVote(agendaItemId, adaptDurhamVote(rawVote)); + } catch (error) { + logger.warn( + `Skipping invalid vote for item ${item.externalId}`, + error, + ); + } + } + } + } catch (error) { + logger.warn( + `Skipping invalid item for meeting ${meeting.externalId}`, + error, + ); + } + } + + logger.success(`Synced ${meeting.title} (${meeting.externalId})`); +} + +async function scrape(maxItems = 100): Promise { + const start = currentElectionCycleStart(); + const eventsUrl = new URL(`${API_BASE}/Events`); + eventsUrl.searchParams.set( + "$filter", + `EventDate ge datetime'${start.toISOString().slice(0, 10)}' and EventBodyId eq ${BOCC_BODY_ID}`, + ); + eventsUrl.searchParams.set("$orderby", "EventDate asc"); + eventsUrl.searchParams.set("$top", String(maxItems)); + + logger.info( + `Syncing current election cycle from ${start.toISOString().slice(0, 10)} (structured source; OCR not required)`, + ); + const rawEvents = z + .array(z.unknown()) + .parse(await fetchJson(eventsUrl)) + .slice(0, maxItems); + setExpectedTotal(rawEvents.length); + + const limit = getItemLimit(); + const results = await Promise.allSettled( + rawEvents.map((rawEvent) => + limit(async () => { + try { + await scrapeMeeting(rawEvent); + } catch (error) { + let sourceId = "unknown"; + const parsed = durhamEventSchema.safeParse(rawEvent); + if (parsed.success) sourceId = String(parsed.data.EventId); + logger.error( + `Meeting ${sourceId} failed without aborting run`, + error, + ); + } + }), + ), + ); + const failed = results.filter((result) => result.status === "rejected"); + if (failed.length > 0) logger.warn(`${failed.length} meeting tasks failed`); + logger.success(`Completed ${rawEvents.length} Durham BOCC meetings`); +} + +export const durhamBocc: Scraper = { + ...durhamBoccConfig, + scrape: (options) => + scrape( + (options?.maxItems ?? Number(process.env.DURHAM_BOCC_MAX_ITEMS)) || 100, + ), +}; + +// Exported for targeted cleanup/diagnostics without widening the public API. +export async function hasDurhamMeeting(sourceId: string): Promise { + const rows = await db + .select({ id: LocalGovernmentMeeting.id }) + .from(LocalGovernmentMeeting) + .where( + and( + eq(LocalGovernmentMeeting.source, PROVIDER), + eq(LocalGovernmentMeeting.jurisdiction, JURISDICTION), + eq(LocalGovernmentMeeting.externalId, sourceId), + ), + ) + .limit(1); + return rows.length > 0; +} diff --git a/apps/scraper/src/scrapers/durham-onbase-parser.test.ts b/apps/scraper/src/scrapers/durham-onbase-parser.test.ts new file mode 100644 index 00000000..d55b0c68 --- /dev/null +++ b/apps/scraper/src/scrapers/durham-onbase-parser.test.ts @@ -0,0 +1,73 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import { describe, it } from "node:test"; + +import { + parseAgendaOutline, + parseItemAttachments, + parseMeetingIndex, +} from "./durham-onbase-parser.js"; + +const fixture = (name: string) => + readFile(new URL(`./__fixtures__/${name}`, import.meta.url), "utf8"); + +describe("Durham OnBase deterministic parsers", () => { + it("parses the embedded meeting index JSON and preserves timezone", async () => { + const meetings = parseMeetingIndex( + await fixture("durham-onbase-index.html"), + ); + assert.equal(meetings.length, 2); + assert.deepEqual( + { + id: meetings[0]?.id, + type: meetings[0]?.meetingType, + iso: meetings[0]?.date.toISOString(), + latestDocumentType: meetings[1]?.latestDocumentType, + }, + { + id: 748, + type: "City Council Meeting Agenda", + iso: "2026-05-18T23:00:00.000Z", + latestDocumentType: 2, + }, + ); + }); + + it("parses sections, items, action text, and vote text", async () => { + const items = parseAgendaOutline( + await fixture("durham-onbase-agenda.html"), + ); + assert.equal(items.length, 2); + assert.deepEqual( + { + externalId: items[0]?.externalId, + section: items[0]?.section, + number: items[0]?.agendaNumber, + title: items[0]?.title, + vote: items[0]?.voteText, + }, + { + externalId: "47768", + section: "Consent Agenda", + number: "1", + title: "Approval of City Council Minutes", + vote: "[Approved by Vote: 7/0]", + }, + ); + assert.match(items[1]?.voteText ?? "", /FAILED by Vote: 0\/7/i); + assert.match(items[1]?.voteText ?? "", /No vote taken/i); + }); + + it("parses stable attachment IDs and absolute official URLs", async () => { + const attachments = parseItemAttachments( + await fixture("durham-onbase-item.html"), + ); + assert.equal(attachments.length, 2); + assert.equal(attachments[0]?.externalId, "272813"); + assert.equal( + new URL(attachments[0]!.url).hostname, + "cityordinances.durhamnc.gov", + ); + assert.equal(attachments[1]?.title, "March 16 City Council Minutes"); + }); +}); diff --git a/apps/scraper/src/scrapers/durham-onbase-parser.ts b/apps/scraper/src/scrapers/durham-onbase-parser.ts new file mode 100644 index 00000000..0764279a --- /dev/null +++ b/apps/scraper/src/scrapers/durham-onbase-parser.ts @@ -0,0 +1,198 @@ +import * as cheerio from "cheerio"; + +export const DURHAM_ONBASE_BASE_URL = + "https://cityordinances.durhamnc.gov/OnBaseAgendaOnline/"; + +export interface OnBaseMeetingIndexItem { + id: number; + name: string; + meetingType: string; + date: Date; + location?: string; + isAgendaAvailable: boolean; + isMinutesAvailable: boolean; + agendaUniqueName: string; + minutesUniqueName: string; + latestDocumentType: 1 | 2 | 3; +} + +export interface ParsedOnBaseAttachment { + externalId: string; + title: string; + url: string; +} + +export interface ParsedOnBaseAgendaItem { + externalId: string; + section?: string; + agendaNumber?: string; + title: string; + actionText?: string; + voteText?: string; + attachments: ParsedOnBaseAttachment[]; + sortOrder: number; +} + +interface RawMeeting { + ID?: unknown; + Name?: unknown; + MeetingTypeName?: unknown; + Time?: unknown; + Location?: unknown; + IsAgendaAvailable?: unknown; + IsMinutesAvailable?: unknown; + AgendaUniqueName?: unknown; + MinutesUniqueName?: unknown; + LatestDocumentType?: unknown; +} + +function cleanText(value: string): string { + return value + .replace(/\u00a0/g, " ") + .replace(/\s+/g, " ") + .trim(); +} + +function extractJsonObject(source: string, start: number): string { + let depth = 0; + let inString = false; + let escaped = false; + + for (let index = start; index < source.length; index++) { + const char = source[index]!; + if (inString) { + if (escaped) escaped = false; + else if (char === "\\") escaped = true; + else if (char === '"') inString = false; + continue; + } + if (char === '"') inString = true; + else if (char === "{") depth++; + else if (char === "}" && --depth === 0) + return source.slice(start, index + 1); + } + + throw new Error("Unterminated OnBase meeting index JSON"); +} + +export function parseMeetingIndex(html: string): OnBaseMeetingIndexItem[] { + const marker = "showSearchResults(new SearchResults("; + const markerIndex = html.indexOf(marker); + if (markerIndex < 0) + throw new Error("OnBase meeting index payload not found"); + const jsonStart = html.indexOf("{", markerIndex + marker.length); + if (jsonStart < 0) throw new Error("OnBase meeting index JSON not found"); + + const payload = JSON.parse(extractJsonObject(html, jsonStart)) as { + Meetings?: RawMeeting[]; + }; + + return (payload.Meetings ?? []).flatMap((raw) => { + if ( + typeof raw.ID !== "number" || + typeof raw.Name !== "string" || + typeof raw.MeetingTypeName !== "string" || + typeof raw.Time !== "string" || + typeof raw.AgendaUniqueName !== "string" || + typeof raw.MinutesUniqueName !== "string" + ) { + return []; + } + const date = new Date(raw.Time); + if (Number.isNaN(date.getTime())) return []; + const latest = + raw.LatestDocumentType === 2 || raw.LatestDocumentType === 3 + ? raw.LatestDocumentType + : 1; + + return [ + { + id: raw.ID, + name: cleanText(raw.Name), + meetingType: cleanText(raw.MeetingTypeName), + date, + location: + typeof raw.Location === "string" && cleanText(raw.Location) + ? cleanText(raw.Location) + : undefined, + isAgendaAvailable: raw.IsAgendaAvailable === true, + isMinutesAvailable: raw.IsMinutesAvailable === true, + agendaUniqueName: raw.AgendaUniqueName, + minutesUniqueName: raw.MinutesUniqueName, + latestDocumentType: latest, + }, + ]; + }); +} + +function extractVoteText(actionText: string): string | undefined { + const matches = actionText.match( + /\[(?:approved|failed)[^\]]*\]|\[motion referred[^\]]*\]|no vote (?:was )?taken\.?|ayes:\s*[^.]*\.?|nays:\s*[^.]*\.?/gi, + ); + return matches ? cleanText(matches.join(" ")) : undefined; +} + +export function parseAgendaOutline(html: string): ParsedOnBaseAgendaItem[] { + const $ = cheerio.load(html); + const items: ParsedOnBaseAgendaItem[] = []; + let currentSection: string | undefined; + + $("a[href*='loadAgendaItem']").each((_, element) => { + const anchor = $(element); + const href = anchor.attr("href") ?? ""; + const match = /loadAgendaItem\((\d+),(true|false)\)/.exec(href); + if (!match) return; + const title = cleanText(anchor.text()); + if (!title) return; + if (match[2] === "true") { + currentSection = title; + return; + } + + const externalId = match[1]!; + const agendaMatch = /^(\d+[A-Za-z]?)\.\s*(.+)$/.exec(title); + const displayTitle = agendaMatch?.[2] ?? title; + const tableText = cleanText(anchor.closest("table").text()); + const actionText = cleanText( + tableText.startsWith(title) ? tableText.slice(title.length) : tableText, + ); + + items.push({ + externalId, + section: currentSection, + agendaNumber: agendaMatch?.[1], + title: displayTitle, + actionText: actionText || undefined, + voteText: actionText ? extractVoteText(actionText) : undefined, + attachments: [], + sortOrder: items.length, + }); + }); + + return items; +} + +export function parseItemAttachments( + html: string, + baseUrl = DURHAM_ONBASE_BASE_URL, +): ParsedOnBaseAttachment[] { + const $ = cheerio.load(html); + return $("a[href*='/Documents/DownloadFile/']") + .toArray() + .flatMap((element) => { + const anchor = $(element); + const href = anchor.attr("href"); + if (!href) return []; + const url = new URL(href, baseUrl); + const externalId = + url.searchParams.get("publishId") ?? + (anchor.attr("id")?.match(/(\d+)$/)?.[1] || url.toString()); + return [ + { + externalId, + title: cleanText(anchor.text()), + url: url.toString(), + }, + ]; + }); +} diff --git a/apps/scraper/src/scrapers/durham-onbase.config.ts b/apps/scraper/src/scrapers/durham-onbase.config.ts new file mode 100644 index 00000000..9297fbf7 --- /dev/null +++ b/apps/scraper/src/scrapers/durham-onbase.config.ts @@ -0,0 +1,11 @@ +import type { ScraperEnvContract } from "@acme/env"; + +export const durhamOnBaseConfig = { + id: "durham-onbase", + name: "Durham City Council OnBase", + source: "City of Durham OnBase Agenda Online", + environment: { + required: ["POSTGRES_URL"], + optional: ["DURHAM_ONBASE_MAX_ITEMS", "DURHAM_ONBASE_CACHE_TTL_HOURS"], + }, +} as const satisfies ScraperEnvContract; diff --git a/apps/scraper/src/scrapers/durham-onbase.ts b/apps/scraper/src/scrapers/durham-onbase.ts new file mode 100644 index 00000000..385963f1 --- /dev/null +++ b/apps/scraper/src/scrapers/durham-onbase.ts @@ -0,0 +1,302 @@ +import { createHash } from "node:crypto"; +import { and, eq, notInArray } from "drizzle-orm"; + +import { db } from "@acme/db/client"; +import { + LocalGovernmentAgendaItem, + LocalGovernmentDocument, + LocalGovernmentMeeting, +} from "@acme/db/schema"; + +import type { Scraper } from "../utils/types.js"; +import { getItemLimit } from "../utils/concurrency.js"; +import { setExpectedTotal } from "../utils/db/metrics.js"; +import { fetchWithRetry } from "../utils/fetch.js"; +import { createLogger } from "../utils/log.js"; +import { + DURHAM_ONBASE_BASE_URL, + parseAgendaOutline, + parseItemAttachments, + parseMeetingIndex, +} from "./durham-onbase-parser.js"; +import { durhamOnBaseConfig } from "./durham-onbase.config.js"; + +const PROVIDER = "onbase"; +const JURISDICTION = "durham-nc"; +const GOVERNING_BODY = "Durham City Council"; +const SOURCE_VERSION = "onbase-agenda-online-v1"; +const MIN_REQUEST_INTERVAL_MS = 250; +const logger = createLogger(durhamOnBaseConfig.name); +let nextRequestAt = 0; +let requestGate = Promise.resolve(); + +async function fetchOnBase(path: string): Promise { + const turn = requestGate.then(async () => { + const delay = Math.max(0, nextRequestAt - Date.now()); + if (delay) await new Promise((resolve) => setTimeout(resolve, delay)); + nextRequestAt = Date.now() + MIN_REQUEST_INTERVAL_MS; + }); + requestGate = turn.catch(() => undefined); + await turn; + const response = await fetchWithRetry( + new URL(path, DURHAM_ONBASE_BASE_URL).toString(), + { + timeoutMs: 30_000, + headers: { + "User-Agent": + "Billion civic-data scraper (contact: support@billion.app)", + }, + }, + ); + return response.text(); +} + +function durhamCalendarYear(date: Date): number { + return Number( + new Intl.DateTimeFormat("en-US", { + year: "numeric", + timeZone: "America/New_York", + }).format(date), + ); +} + +function documentTypeName(documentType: 1 | 2 | 3): string { + if (documentType === 2) return "minutes"; + if (documentType === 3) return "summary"; + return "agenda"; +} + +function pdfUrl( + uniqueName: string, + documentType: number, + meetingId: number, +): string { + const path = `Documents/DownloadFile/${encodeURIComponent(uniqueName)}.pdf`; + const url = new URL(path, DURHAM_ONBASE_BASE_URL); + url.searchParams.set("documentType", String(documentType)); + url.searchParams.set("meetingId", String(meetingId)); + return url.toString(); +} + +function contentHash(value: unknown): string { + return createHash("sha256").update(JSON.stringify(value)).digest("hex"); +} + +async function scrape(maxItems = 100): Promise { + const cycleYear = new Date().getFullYear(); + const ttlHours = Number(process.env.DURHAM_ONBASE_CACHE_TTL_HOURS ?? 24); + const cacheCutoff = new Date(Date.now() - ttlHours * 60 * 60 * 1000); + logger.info(`Fetching current ${cycleYear} council cycle`); + + const indexHtml = await fetchOnBase(""); + const meetings = parseMeetingIndex(indexHtml) + .filter((meeting) => durhamCalendarYear(meeting.date) === cycleYear) + .slice(0, maxItems); + setExpectedTotal(meetings.length); + + const limit = getItemLimit(); + const results = await Promise.allSettled( + meetings.map((meeting) => + limit(async () => { + const externalId = String(meeting.id); + const [cached] = await db + .select({ fetchedAt: LocalGovernmentMeeting.fetchedAt }) + .from(LocalGovernmentMeeting) + .where( + and( + eq(LocalGovernmentMeeting.source, PROVIDER), + eq(LocalGovernmentMeeting.jurisdiction, JURISDICTION), + eq(LocalGovernmentMeeting.externalId, externalId), + ), + ) + .limit(1); + if (cached && cached.fetchedAt >= cacheCutoff) { + logger.info(`Cached: ${meeting.name}`); + return; + } + + const documentType = meeting.latestDocumentType; + const type = documentTypeName(documentType); + const outlineHtml = await fetchOnBase( + `Documents/ViewAgenda?meetingId=${meeting.id}&type=${type}&doctype=${documentType}`, + ); + const items = parseAgendaOutline(outlineHtml); + + for (const item of items) { + const detailHtml = await fetchOnBase( + `Meetings/ViewMeetingAgendaItem?meetingId=${meeting.id}&itemId=${item.externalId}&isSection=false&type=${type}`, + ); + item.attachments = parseItemAttachments(detailHtml); + } + + const sourceUrl = new URL( + `Meetings/ViewMeeting?doctype=${documentType}&id=${meeting.id}`, + DURHAM_ONBASE_BASE_URL, + ).toString(); + const [storedMeeting] = await db + .insert(LocalGovernmentMeeting) + .values({ + source: PROVIDER, + sourceVersion: SOURCE_VERSION, + jurisdiction: JURISDICTION, + governingBody: GOVERNING_BODY, + externalId, + title: meeting.name, + meetingType: meeting.meetingType, + status: documentType === 2 ? "minutes-published" : "published", + startsAt: meeting.date, + location: meeting.location, + canonicalUrl: sourceUrl, + contentHash: contentHash({ meeting, items }), + fetchedAt: new Date(), + }) + .onConflictDoUpdate({ + target: [ + LocalGovernmentMeeting.source, + LocalGovernmentMeeting.jurisdiction, + LocalGovernmentMeeting.externalId, + ], + set: { + sourceVersion: SOURCE_VERSION, + governingBody: GOVERNING_BODY, + title: meeting.name, + meetingType: meeting.meetingType, + status: documentType === 2 ? "minutes-published" : "published", + startsAt: meeting.date, + location: meeting.location, + canonicalUrl: sourceUrl, + contentHash: contentHash({ meeting, items }), + fetchedAt: new Date(), + }, + }) + .returning({ id: LocalGovernmentMeeting.id }); + if (!storedMeeting) + throw new Error(`Failed to persist meeting ${externalId}`); + + const documents = [ + ...(meeting.isAgendaAvailable + ? [ + { + type: "agenda", + title: `${meeting.name} agenda`, + url: pdfUrl(meeting.agendaUniqueName, 1, meeting.id), + }, + ] + : []), + ...(meeting.isMinutesAvailable + ? [ + { + type: "minutes", + title: `${meeting.name} minutes`, + url: pdfUrl(meeting.minutesUniqueName, 2, meeting.id), + }, + ] + : []), + ...items.flatMap((item) => + item.attachments.map((attachment) => ({ + type: "attachment", + title: attachment.title, + url: attachment.url, + })), + ), + ]; + for (const document of documents) { + await db + .insert(LocalGovernmentDocument) + .values({ + meetingId: storedMeeting.id, + ...document, + mediaType: document.url.toLowerCase().endsWith(".pdf") + ? "application/pdf" + : undefined, + fetchedAt: new Date(), + }) + .onConflictDoUpdate({ + target: [ + LocalGovernmentDocument.meetingId, + LocalGovernmentDocument.type, + LocalGovernmentDocument.url, + ], + set: { + title: document.title, + isCurrent: true, + fetchedAt: new Date(), + }, + }); + } + + for (const item of items) { + const itemSourceUrl = new URL( + `Meetings/ViewMeetingAgendaItem?meetingId=${meeting.id}&itemId=${item.externalId}&isSection=false&type=${type}`, + DURHAM_ONBASE_BASE_URL, + ).toString(); + await db + .insert(LocalGovernmentAgendaItem) + .values({ + meetingId: storedMeeting.id, + externalId: item.externalId, + sequence: item.sortOrder, + itemNumber: item.agendaNumber, + section: item.section, + itemType: "agenda-item", + title: item.title, + motion: item.actionText, + voteSummary: item.voteText, + sourceUrl: itemSourceUrl, + }) + .onConflictDoUpdate({ + target: [ + LocalGovernmentAgendaItem.meetingId, + LocalGovernmentAgendaItem.externalId, + ], + set: { + sequence: item.sortOrder, + itemNumber: item.agendaNumber, + section: item.section, + title: item.title, + motion: item.actionText, + voteSummary: item.voteText, + sourceUrl: itemSourceUrl, + }, + }); + } + + if (items.length) { + await db.delete(LocalGovernmentAgendaItem).where( + and( + eq(LocalGovernmentAgendaItem.meetingId, storedMeeting.id), + notInArray( + LocalGovernmentAgendaItem.externalId, + items.map((item) => item.externalId), + ), + ), + ); + } else { + await db + .delete(LocalGovernmentAgendaItem) + .where(eq(LocalGovernmentAgendaItem.meetingId, storedMeeting.id)); + } + logger.success(`Scraped ${meeting.name} (${items.length} items)`); + }), + ), + ); + + const failures = results.filter( + (result): result is PromiseRejectedResult => result.status === "rejected", + ); + if (failures.length) { + throw new AggregateError( + failures.map((failure) => failure.reason), + `${failures.length} Durham meeting(s) failed`, + ); + } + logger.success(`Completed ${meetings.length} current-cycle meetings`); +} + +export const durhamOnBase: Scraper = { + ...durhamOnBaseConfig, + scrape: (options) => + scrape( + options?.maxItems ?? Number(process.env.DURHAM_ONBASE_MAX_ITEMS ?? 100), + ), +}; diff --git a/apps/scraper/src/scrapers/fixtures/civicengage/agenda.txt b/apps/scraper/src/scrapers/fixtures/civicengage/agenda.txt new file mode 100644 index 00000000..b83450c8 --- /dev/null +++ b/apps/scraper/src/scrapers/fixtures/civicengage/agenda.txt @@ -0,0 +1,10 @@ +CITY COUNCIL AGENDA +Consent Agenda +D.1 Approval Of Minutes From The Regular Scheduled City Council Meeting On December 18, 2025. +E.1 Second Reading And Approval Of An Ordinance Calling And Ordering A Special Called Election To Be Held On May 2, 2026. +F.1 A Resolution To Adopt The Cedar Park Transportation Criteria Manual (TCM). +Public Hearings +G.1 First Reading And Public Hearing Of An Ordinance For A Future Land Use Plan Amendment. +Regular Agenda (Non-Consent) +H.1 Consideration Of A Resolution Authorizing A Grant Agreement With The Lower Colorado River Authority In The Amount Of $100,000. +H.2 Consider Action, If Any, On Items Discussed In Executive Session. diff --git a/apps/scraper/src/scrapers/fixtures/civicengage/meetings.html b/apps/scraper/src/scrapers/fixtures/civicengage/meetings.html new file mode 100644 index 00000000..c907d64f --- /dev/null +++ b/apps/scraper/src/scrapers/fixtures/civicengage/meetings.html @@ -0,0 +1,44 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
City Council Mtg.2/12/20266:00 PMCEDAR PARK CITY HALL COUNCIL CHAMBERSPDF Agenda for February 12, 2026 City Council Mtg.PDF Packet for February 12, 2026 City Council Mtg.PDF Minutes for February 12, 2026 City Council Mtg.
City Council - Special Called2/25/20265:00 PMCITY HALLAgenda
City Council Mtg. - CANCELLED3/12/20266:00 PMCITY HALLAgenda
City Council Mtg. - Amended3/13/20257:00 PMCITY HALLAgenda
City Council Mtg.3/13/20257:00 PMCITY HALLPrevious Agenda
Planning and Zoning Commission3/1/20266:00 PMCITY HALL
diff --git a/apps/scraper/src/scrapers/fixtures/civicengage/minutes.txt b/apps/scraper/src/scrapers/fixtures/civicengage/minutes.txt new file mode 100644 index 00000000..32f53f9b --- /dev/null +++ b/apps/scraper/src/scrapers/fixtures/civicengage/minutes.txt @@ -0,0 +1,24 @@ +CITY COUNCIL MINUTES +Consent Agenda +D.1 Approval Of Minutes From The Regular Scheduled City Council Meeting On December 18, 2025. +Approved under the Consent Agenda. +E.1 Second Reading And Approval Of An Ordinance Calling And Ordering A Special Called Election To Be Held On May 2, 2026. +Approved under the Consent Agenda. +F.1 A Resolution To Adopt The Cedar Park Transportation Criteria Manual (TCM). +Motion to approve Agenda Item F.1 as presented. +Movant: Councilmember Kirkland +Second: Councilmember Frezza +Vote: 6-0 with Mayor Pro Tem Boyce absent from meeting +Public Hearings +G.1 First Reading And Public Hearing Of An Ordinance For A Future Land Use Plan Amendment. +Mayor opened the Public Hearing. No Public Comment. Mayor closed the Public Hearing. +Regular Agenda (Non-Consent) +H.1 Consideration Of A Resolution Authorizing A Grant Agreement With The Lower Colorado River Authority In The Amount Of $100,000. +Motion to approve Agenda Item H.1 as presented. +Movant: Councilmember Kirkland +Second: Councilmember Duffy +Vote: 5-1. +Voting Aye: Kirkland, Duffy, Harris, Frezza, and Penniman-Morin +Voting Nay: Darby +H.2 Consider Action, If Any, On Items Discussed In Executive Session. +No action taken. diff --git a/apps/scraper/src/scrapers/fixtures/durham-bocc/events.json b/apps/scraper/src/scrapers/fixtures/durham-bocc/events.json new file mode 100644 index 00000000..fe275d86 --- /dev/null +++ b/apps/scraper/src/scrapers/fixtures/durham-bocc/events.json @@ -0,0 +1,78 @@ +[ + { + "EventId": 1439, + "EventGuid": "4AF37432-715B-4E88-8C23-C24F0ABF3D98", + "EventLastModifiedUtc": "2026-01-12T20:19:16.093", + "EventRowVersion": "AAAAAACFTNc=", + "EventBodyId": 138, + "EventBodyName": "Board of County Commissioners", + "EventDate": "2026-01-12T00:00:00", + "EventTime": "6:00 PM", + "EventAgendaStatusName": "Final", + "EventMinutesStatusName": "Draft", + "EventLocation": "Commissioners' Chambers", + "EventAgendaFile": "https://durhamcounty.legistar1.com/durhamcounty/meetings/2026/1/1439_Agenda.pdf", + "EventMinutesFile": null, + "EventComment": "Regular Session", + "EventVideoPath": null, + "EventMedia": "1539", + "EventInSiteURL": "https://durhamcounty.legistar.com/MeetingDetail.aspx?LEGID=1439" + }, + { + "EventId": 1441, + "EventGuid": "AA819B77-C3F4-49A5-8BF0-29BCDDD09B25", + "EventLastModifiedUtc": "2026-02-02T12:55:38.687", + "EventRowVersion": "AAAAAACFhUA=", + "EventBodyId": 138, + "EventBodyName": "Board of County Commissioners", + "EventDate": "2026-02-02T00:00:00", + "EventTime": "9:00 AM", + "EventAgendaStatusName": "Final", + "EventMinutesStatusName": "Draft", + "EventLocation": "Commissioners' Chambers", + "EventAgendaFile": "https://durhamcounty.legistar1.com/durhamcounty/meetings/2026/2/1441_Agenda.pdf", + "EventMinutesFile": null, + "EventComment": "Work Session", + "EventVideoPath": null, + "EventMedia": null, + "EventInSiteURL": "https://durhamcounty.legistar.com/MeetingDetail.aspx?LEGID=1441" + }, + { + "EventId": 1440, + "EventGuid": "418F96F6-84B8-40B6-9BD8-F409555EDFD7", + "EventLastModifiedUtc": "2026-01-26T01:33:13.067", + "EventRowVersion": "AAAAAACFIbU=", + "EventBodyId": 138, + "EventBodyName": "Board of County Commissioners", + "EventDate": "2026-01-26T00:00:00", + "EventTime": "7:00 PM", + "EventAgendaStatusName": "Final", + "EventMinutesStatusName": "Draft", + "EventLocation": "Commissioners' Chambers", + "EventAgendaFile": "https://durhamcounty.legistar1.com/durhamcounty/meetings/2026/1/1440_Agenda.pdf", + "EventMinutesFile": null, + "EventComment": "Cancelled - Regular Session", + "EventVideoPath": null, + "EventMedia": null, + "EventInSiteURL": "https://durhamcounty.legistar.com/MeetingDetail.aspx?LEGID=1440" + }, + { + "EventId": 1499, + "EventGuid": "EDC0142F-3923-4A79-AAA7-9319E811C112", + "EventLastModifiedUtc": "2026-05-04T15:00:00.000", + "EventRowVersion": "AMENDED-V2", + "EventBodyId": 138, + "EventBodyName": "Board of County Commissioners", + "EventDate": "2026-05-04T00:00:00", + "EventTime": "9:00 AM", + "EventAgendaStatusName": "Final", + "EventMinutesStatusName": "Draft", + "EventLocation": "Commissioners' Chambers", + "EventAgendaFile": "https://durhamcounty.legistar1.com/durhamcounty/meetings/2026/5/1499_Amended_Agenda.pdf", + "EventMinutesFile": null, + "EventComment": "Amended Work Session", + "EventVideoPath": null, + "EventMedia": "1599", + "EventInSiteURL": "https://durhamcounty.legistar.com/MeetingDetail.aspx?LEGID=1499" + } +] diff --git a/apps/scraper/src/scrapers/fixtures/durham-bocc/items.json b/apps/scraper/src/scrapers/fixtures/durham-bocc/items.json new file mode 100644 index 00000000..cfc2a009 --- /dev/null +++ b/apps/scraper/src/scrapers/fixtures/durham-bocc/items.json @@ -0,0 +1,38 @@ +[ + { + "EventItemId": 30280, + "EventItemLastModifiedUtc": "2026-04-14T13:52:55.327", + "EventItemRowVersion": "ITEM-V1", + "EventItemEventId": 1439, + "EventItemAgendaSequence": 12, + "EventItemAgendaNumber": "6.", + "EventItemAgendaNote": "Approve the consent agenda.", + "EventItemMinutesNote": "The motion carried unanimously.", + "EventItemActionName": "Approved", + "EventItemActionText": "Adopt the consent agenda", + "EventItemPassedFlagName": "Passed", + "EventItemRollCallFlag": 1, + "EventItemTitle": "Consent Agenda", + "EventItemTally": "5-0", + "EventItemConsent": 1, + "EventItemMover": "Commissioner A", + "EventItemSeconder": "Commissioner B", + "EventItemMatterId": 9181, + "EventItemMatterAttachments": [ + { + "MatterAttachmentId": 22223, + "MatterAttachmentName": "Budget memorandum", + "MatterAttachmentHyperlink": "https://durhamcounty.legistar1.com/durhamcounty/attachments/budget.pdf", + "MatterAttachmentLastModifiedUtc": "2026-04-09T14:39:50.21", + "MatterAttachmentShowOnInternetPage": true + }, + { + "MatterAttachmentId": 22224, + "MatterAttachmentName": "Presupuesto - Español", + "MatterAttachmentHyperlink": "https://durhamcounty.legistar1.com/durhamcounty/attachments/presupuesto.pdf", + "MatterAttachmentLastModifiedUtc": "2026-04-09T14:40:50.21", + "MatterAttachmentShowOnInternetPage": true + } + ] + } +] diff --git a/apps/scraper/src/scrapers/fixtures/durham-bocc/votes.json b/apps/scraper/src/scrapers/fixtures/durham-bocc/votes.json new file mode 100644 index 00000000..55e7c2c7 --- /dev/null +++ b/apps/scraper/src/scrapers/fixtures/durham-bocc/votes.json @@ -0,0 +1,11 @@ +[ + { + "VoteId": 9001, + "VoteLastModifiedUtc": "2026-04-14T14:00:00.000", + "VotePersonId": 101, + "VotePersonName": "Commissioner A", + "VoteValueName": "Aye", + "VoteSort": 1, + "VoteEventItemId": 30280 + } +] diff --git a/apps/scraper/src/scrapers/fixtures/kansas-city-council/events.json b/apps/scraper/src/scrapers/fixtures/kansas-city-council/events.json new file mode 100644 index 00000000..cdf72665 --- /dev/null +++ b/apps/scraper/src/scrapers/fixtures/kansas-city-council/events.json @@ -0,0 +1,86 @@ +[ + { + "EventId": 19001, + "EventGuid": "11111111-1111-4111-8111-111111111111", + "EventLastModifiedUtc": "2026-01-09T22:00:00.000", + "EventRowVersion": "kc-event-v1", + "EventBodyId": 138, + "EventBodyName": "Council", + "EventDate": "2026-01-08T00:00:00", + "EventTime": "2:00 PM", + "EventAgendaStatusName": "Final", + "EventMinutesStatusName": "Final", + "EventLocation": "Council Chambers", + "EventAgendaFile": "https://kansascity.legistar1.com/meetings/19001-agenda.pdf", + "EventMinutesFile": "https://kansascity.legistar1.com/meetings/19001-minutes.pdf", + "EventAgendaLastPublishedUTC": "2026-01-08T17:00:00.000", + "EventMinutesLastPublishedUTC": "2026-01-09T22:00:00.000", + "EventComment": "Webinar Link: https://example.test/council", + "EventVideoPath": null, + "EventMedia": "14001", + "EventInSiteURL": "https://kansascity.legistar.com/MeetingDetail.aspx?LEGID=19001" + }, + { + "EventId": 19002, + "EventGuid": "22222222-2222-4222-8222-222222222222", + "EventLastModifiedUtc": "2026-07-03T20:00:00.000", + "EventRowVersion": "kc-special-v1", + "EventBodyId": 138, + "EventBodyName": "Council", + "EventDate": "2026-07-02T00:00:00", + "EventTime": "12:30 PM", + "EventAgendaStatusName": "Final", + "EventMinutesStatusName": "Draft", + "EventLocation": "KCPD Headquarters - Community Room", + "EventAgendaFile": "https://kansascity.legistar1.com/meetings/19002-agenda.pdf", + "EventMinutesFile": null, + "EventAgendaLastPublishedUTC": "2026-07-02T16:00:00.000", + "EventMinutesLastPublishedUTC": null, + "EventComment": "Special Council Meeting", + "EventVideoPath": "https://videos.kcmo.gov/19002", + "EventMedia": null, + "EventInSiteURL": "https://kansascity.legistar.com/MeetingDetail.aspx?LEGID=19002" + }, + { + "EventId": 19003, + "EventGuid": "33333333-3333-4333-8333-333333333333", + "EventLastModifiedUtc": "2026-02-12T18:00:00.000", + "EventRowVersion": "kc-cancelled-v1", + "EventBodyId": 138, + "EventBodyName": "Council", + "EventDate": "2026-02-12T00:00:00", + "EventTime": "2:00 PM", + "EventAgendaStatusName": "Cancelled", + "EventMinutesStatusName": null, + "EventLocation": "Council Chambers", + "EventAgendaFile": null, + "EventMinutesFile": null, + "EventAgendaLastPublishedUTC": null, + "EventMinutesLastPublishedUTC": null, + "EventComment": "Meeting cancelled", + "EventVideoPath": null, + "EventMedia": null, + "EventInSiteURL": "https://kansascity.legistar.com/MeetingDetail.aspx?LEGID=19003" + }, + { + "EventId": 19004, + "EventGuid": "44444444-4444-4444-8444-444444444444", + "EventLastModifiedUtc": "2026-04-16T21:30:00.000", + "EventRowVersion": "kc-revised-v2", + "EventBodyId": 138, + "EventBodyName": "Council", + "EventDate": "2026-04-16T00:00:00", + "EventTime": "2:00 PM", + "EventAgendaStatusName": "Final-Revised", + "EventMinutesStatusName": "Final-Revised", + "EventLocation": "26th Floor, Council Chambers", + "EventAgendaFile": "https://kansascity.legistar1.com/meetings/19004-agenda-revised.pdf", + "EventMinutesFile": "https://kansascity.legistar1.com/meetings/19004-minutes-revised.pdf", + "EventAgendaLastPublishedUTC": "2026-04-16T20:30:00.000", + "EventMinutesLastPublishedUTC": "2026-04-16T21:30:00.000", + "EventComment": "Webinar Link: https://example.test/council", + "EventVideoPath": null, + "EventMedia": null, + "EventInSiteURL": "https://kansascity.legistar.com/MeetingDetail.aspx?LEGID=19004" + } +] diff --git a/apps/scraper/src/scrapers/fixtures/kansas-city-council/items.json b/apps/scraper/src/scrapers/fixtures/kansas-city-council/items.json new file mode 100644 index 00000000..b24796ea --- /dev/null +++ b/apps/scraper/src/scrapers/fixtures/kansas-city-council/items.json @@ -0,0 +1,75 @@ +[ + { + "EventItemId": 106909, + "EventItemLastModifiedUtc": "2026-07-16T21:12:40.197", + "EventItemRowVersion": "kc-item-v2", + "EventItemEventId": 19004, + "EventItemAgendaSequence": 15, + "EventItemAgendaNumber": null, + "EventItemVersion": "2", + "EventItemAgendaNote": "Committee substitute", + "EventItemMinutesNote": "The motion carried.", + "EventItemActionId": 409, + "EventItemActionName": "Passed as Substituted", + "EventItemActionText": "A motion was made that this Ordinance be Passed as Substituted.", + "EventItemPassedFlagName": "Pass", + "EventItemRollCallFlag": 0, + "EventItemTitle": "Approving funding for the 1815 Paseo Project.", + "EventItemTally": null, + "EventItemConsent": 0, + "EventItemMover": "Councilmember A", + "EventItemSeconder": "Councilmember B", + "EventItemMatterId": 50729, + "EventItemMatterGuid": "B41F898F-F92D-4A4E-B3B1-33F53A59EA56", + "EventItemMatterFile": "260582", + "EventItemMatterName": null, + "EventItemMatterType": "Ordinance", + "EventItemMatterStatus": "Passed", + "EventItemMatterAttachments": [ + { + "MatterAttachmentId": 104574, + "MatterAttachmentLastModifiedUtc": "2026-07-06T17:39:27.393", + "MatterAttachmentRowVersion": "attachment-v1", + "MatterAttachmentName": "Council Agenda Packet", + "MatterAttachmentHyperlink": "https://kansascity.legistar1.com/attachments/packet.pdf", + "MatterAttachmentShowOnInternetPage": true + }, + { + "MatterAttachmentId": 104575, + "MatterAttachmentLastModifiedUtc": "2026-07-08T15:58:22.030", + "MatterAttachmentRowVersion": "attachment-v1", + "MatterAttachmentName": "Docket Memo", + "MatterAttachmentHyperlink": "https://kansascity.legistar1.com/attachments/docket-memo.pdf", + "MatterAttachmentShowOnInternetPage": true + } + ] + }, + { + "EventItemId": 106906, + "EventItemLastModifiedUtc": "2026-07-16T20:07:18.743", + "EventItemRowVersion": "kc-roll-call-v1", + "EventItemEventId": 19004, + "EventItemAgendaSequence": 4, + "EventItemAgendaNumber": null, + "EventItemVersion": null, + "EventItemAgendaNote": null, + "EventItemMinutesNote": null, + "EventItemActionId": null, + "EventItemActionName": null, + "EventItemActionText": null, + "EventItemPassedFlagName": null, + "EventItemRollCallFlag": 1, + "EventItemTitle": "ROLL CALL:", + "EventItemTally": null, + "EventItemConsent": 0, + "EventItemMover": null, + "EventItemSeconder": null, + "EventItemMatterId": null, + "EventItemMatterGuid": null, + "EventItemMatterFile": null, + "EventItemMatterName": null, + "EventItemMatterType": null, + "EventItemMatterStatus": null, + "EventItemMatterAttachments": [] + } +] diff --git a/apps/scraper/src/scrapers/fixtures/kansas-city-council/votes.json b/apps/scraper/src/scrapers/fixtures/kansas-city-council/votes.json new file mode 100644 index 00000000..b9ac84c5 --- /dev/null +++ b/apps/scraper/src/scrapers/fixtures/kansas-city-council/votes.json @@ -0,0 +1,20 @@ +[ + { + "VoteId": 89823, + "VoteLastModifiedUtc": "2026-07-16T21:05:22.227", + "VotePersonId": 194, + "VotePersonName": "Quinton Lucas", + "VoteValueName": "Nay", + "VoteSort": 63, + "VoteEventItemId": 106909 + }, + { + "VoteId": 89824, + "VoteLastModifiedUtc": "2026-07-16T21:05:22.237", + "VotePersonId": 196, + "VotePersonName": "Kevin O'Neill", + "VoteValueName": "Aye", + "VoteSort": 64, + "VoteEventItemId": 106909 + } +] diff --git a/apps/scraper/src/scrapers/fixtures/missouri-bill-list.xml b/apps/scraper/src/scrapers/fixtures/missouri-bill-list.xml new file mode 100644 index 00000000..bf5d66aa --- /dev/null +++ b/apps/scraper/src/scrapers/fixtures/missouri-bill-list.xml @@ -0,0 +1,4 @@ + + HB422026Rhttps://documents.house.mo.gov/xml/261-HB42.xml07-22-2026 10:30:48.253 + HJR72026Rhttps://documents.house.mo.gov/xml/261-HJR7.xml07-21-2026 09:20:10.010 + diff --git a/apps/scraper/src/scrapers/fixtures/missouri-hb42.xml b/apps/scraper/src/scrapers/fixtures/missouri-hb42.xml new file mode 100644 index 00000000..734f73f3 --- /dev/null +++ b/apps/scraper/src/scrapers/fixtures/missouri-hb42.xml @@ -0,0 +1,16 @@ + + HB42HCS HB 42 + <ShortTitle>PUBLIC SAFETY</ShortTitle><LongTitle>Modifies provisions relating to public safety</LongTitle> + 2026-08-2804/03/2026 - Reported Do Pass (H) + https://house.mo.gov/bill.aspx?bill=HB42&year=2026&code=RIntroduced and Read First Time (H)1001002026-01-08 + https://house.mo.gov/bill.aspx?bill=HB42&year=2026&code=RReported Do Pass (H)2002002026-04-031021 + SponsorJamie Example12 + Co-SponsorAlex Example13 + https://documents.house.mo.gov/billtracking/HB42P.pdf1000H.01PPerfected + https://documents.house.mo.gov/billtracking/HB42-summary.pdfPerfected + https://documents.house.mo.gov/billtracking/HB42-fiscal.pdf + https://documents.house.mo.gov/billtracking/HB42-amendment.pdfHA 1 + Crime Prevention and Public Safety2026-02-01 + PUBLIC SAFETY + https://documents.house.mo.gov/billtracking/HB42-witness.pdfPublic testimony + diff --git a/apps/scraper/src/scrapers/fixtures/missouri-senate-actions.xml b/apps/scraper/src/scrapers/fixtures/missouri-senate-actions.xml new file mode 100644 index 00000000..f7b4971d --- /dev/null +++ b/apps/scraper/src/scrapers/fixtures/missouri-senate-actions.xml @@ -0,0 +1,7 @@ + + SB9HEALTH CAREModifies provisions relating to health care2026-08-28T00:00:00 + Sample, Sam04/10/2026 - Reported Do Pass (H) + https://house.mo.gov/bill.aspx?bill=SB9&year=2026&code=RReported to the House and First Read (H)3001002026-03-20 + Health and Mental Health + https://documents.house.mo.gov/billtracking/SB9-summary.pdfCommittee + diff --git a/apps/scraper/src/scrapers/fixtures/missouri-session-set.js b/apps/scraper/src/scrapers/fixtures/missouri-session-set.js new file mode 100644 index 00000000..192df650 --- /dev/null +++ b/apps/scraper/src/scrapers/fixtures/missouri-session-set.js @@ -0,0 +1,7 @@ +var sessionyearcode = "261"; +var baseURL = "https://documents.house.mo.gov/xml/261-"; +var specsessionyearcode = "263"; +var specbaseURL = "https://documents.house.mo.gov/xml/263-"; +var specsession2yearcode = "264"; +var specbaseURL2 = "https://documents.house.mo.gov/xml/264-"; +var showSpec = "true"; diff --git a/apps/scraper/src/scrapers/fixtures/st-louis-aldermen/agenda-detail.html b/apps/scraper/src/scrapers/fixtures/st-louis-aldermen/agenda-detail.html new file mode 100644 index 00000000..e9c11d13 --- /dev/null +++ b/apps/scraper/src/scrapers/fixtures/st-louis-aldermen/agenda-detail.html @@ -0,0 +1,12 @@ + +

Agenda for Week of July 20, 2026

+

Week 14, Session 2026-2027

+
Meeting Agenda
+ + +
63
Youthbuild Award		Shameem Clark Hubbard
+ + +
77
Journey of Hope		Thomas Oldenburg
+
Full Board of Aldermen Meeting
+ diff --git a/apps/scraper/src/scrapers/fixtures/st-louis-aldermen/agenda-index.html b/apps/scraper/src/scrapers/fixtures/st-louis-aldermen/agenda-index.html new file mode 100644 index 00000000..09969d7a --- /dev/null +++ b/apps/scraper/src/scrapers/fixtures/st-louis-aldermen/agenda-index.html @@ -0,0 +1,21 @@ + + +

2026-2027

+ + + + + + + + + + + + + +
07/20/202614Agenda (07/20/2026)
07/06/202613Agenda (07/10/2026)Minutes
+ diff --git a/apps/scraper/src/scrapers/fixtures/st-louis-aldermen/board-bill.html b/apps/scraper/src/scrapers/fixtures/st-louis-aldermen/board-bill.html new file mode 100644 index 00000000..914b80d6 --- /dev/null +++ b/apps/scraper/src/scrapers/fixtures/st-louis-aldermen/board-bill.html @@ -0,0 +1,12 @@ + +

Board Bill Number 63 In Session 2026-2027

+

Youthbuild Award

+
Introduced
+
+

Session: 2026-2027

+

Introduced: 07/20/2026

+

Primary Sponsors: Shameem Clark Hubbard

+

Latest Activity: Committee Assignment

+
+

Legislative History

+ diff --git a/apps/scraper/src/scrapers/fixtures/st-louis-aldermen/calendar.html b/apps/scraper/src/scrapers/fixtures/st-louis-aldermen/calendar.html new file mode 100644 index 00000000..325dd396 --- /dev/null +++ b/apps/scraper/src/scrapers/fixtures/st-louis-aldermen/calendar.html @@ -0,0 +1,27 @@ + + 2026-2027 Aldermanic Calendar + +
+ + + diff --git a/apps/scraper/src/scrapers/fixtures/st-louis-aldermen/civic-meetings.json b/apps/scraper/src/scrapers/fixtures/st-louis-aldermen/civic-meetings.json new file mode 100644 index 00000000..6badbb9b --- /dev/null +++ b/apps/scraper/src/scrapers/fixtures/st-louis-aldermen/civic-meetings.json @@ -0,0 +1,58 @@ +[ + { + "id": 2409, + "name": "Health and Human Development Committee", + "meetingDate": "2026-07-23T09:00:00", + "files": [ + { + "fileId": 3680, + "type": "Agenda", + "publishedOn": "2026-07-20T11:42:35.730", + "name": "Health and Human Development", + "url": "https://civicclerk.blob.core.windows.net/stream/STLOUISMO/agenda.pdf?sig=first" + }, + { + "fileId": 3681, + "type": "Agenda Packet", + "publishedOn": "2026-07-20T11:42:35.730", + "name": "Health and Human Development Packet", + "url": "https://civicclerk.blob.core.windows.net/stream/STLOUISMO/packet.pdf?sig=first" + } + ], + "items": [ + { + "id": 12326, + "idNumber": "", + "itemName": "Board Bills for Review", + "agendaObjectItemOutlineNumber": "IV.", + "agendaObjItemCategoryTypeDesc": null, + "agendaObjectItemDescription": "The committee will discuss the following.", + "isSection": true, + "customTextField8": { "name": "Minutes API Call", "value": "" }, + "recommendedActions": [], + "attachmentsList": [], + "childItems": [ + { + "id": 12336, + "idNumber": "2026-1798", + "itemName": "Item Number 1
Board Bill Number 63
Introduced by Alderwoman Clark Hubbard
YouthBuild grant program.", + "agendaObjectItemOutlineNumber": "", + "agendaObjItemCategoryTypeDesc": "Board Bills", + "agendaObjectItemDescription": "", + "isSection": false, + "customTextField8": { "name": "Minutes API Call", "value": "[DISCUSSION]" }, + "recommendedActions": [], + "attachmentsList": [ + { + "id": 3724, + "name": "BB63 Combined", + "publicUrl": "https://civicclerk.blob.core.windows.net/stream/STLOUISMO/bill.pdf?sig=first" + } + ], + "childItems": null + } + ] + } + ] + } +] diff --git a/apps/scraper/src/scrapers/fixtures/st-louis-aldermen/event-detail.html b/apps/scraper/src/scrapers/fixtures/st-louis-aldermen/event-detail.html new file mode 100644 index 00000000..8eb8926d --- /dev/null +++ b/apps/scraper/src/scrapers/fixtures/st-louis-aldermen/event-detail.html @@ -0,0 +1,14 @@ + +

Health and Human Development Committee

+
+

Type: Aldermanic Committee Meeting
Health and Human Development

+

Description

+

You may watch on YouTube.

+

Board Bills

+

Board Bill Number 63

+

YouthBuild grant program

+
+

Contacts and Location

+
Location:
City Hall Room 230
+ + diff --git a/apps/scraper/src/scrapers/fixtures/texas-hb9-history.xml b/apps/scraper/src/scrapers/fixtures/texas-hb9-history.xml new file mode 100644 index 00000000..b92063fa --- /dev/null +++ b/apps/scraper/src/scrapers/fixtures/texas-hb9-history.xml @@ -0,0 +1,78 @@ + + + Relating to an exemption from ad valorem taxation. + Meyer | Bonnen + Bernal + Bettencourt + + + Taxation--Property-Exemptions (I0793) + Business & Commerce--General (I0050) + + + + + + + + + + + Introduced + https://capitol.texas.gov/tlodocs/89R/billtext/html/HB00009I.htm + https://capitol.texas.gov/tlodocs/89R/billtext/pdf/HB00009I.pdf + ftp://ftp.legis.state.tx.us/bills/89R/billtext/HTML/house_bills/HB00001_HB00099/HB00009I.HTM + ftp://ftp.legis.state.tx.us/bills/89R/billtext/PDF/house_bills/HB00001_HB00099/HB00009I.PDF + + + Enrolled + https://capitol.texas.gov/tlodocs/89R/billtext/html/HB00009F.htm + https://capitol.texas.gov/tlodocs/89R/billtext/pdf/HB00009F.pdf + ftp://ftp.legis.state.tx.us/bills/89R/billtext/HTML/house_bills/HB00001_HB00099/HB00009F.HTM + ftp://ftp.legis.state.tx.us/bills/89R/billtext/PDF/house_bills/HB00001_HB00099/HB00009F.PDF + + + + + + + House Committee Report + https://capitol.texas.gov/tlodocs/89R/analysis/html/HB00009H.htm + https://capitol.texas.gov/tlodocs/89R/analysis/pdf/HB00009H.pdf + ftp://ftp.legis.state.tx.us/bills/89R/analysis/HTML/house_bills/HB00001_HB00099/HB00009H.HTM + ftp://ftp.legis.state.tx.us/bills/89R/analysis/PDF/house_bills/HB00001_HB00099/HB00009H.PDF + + + + + + + Introduced + https://capitol.texas.gov/tlodocs/89R/fiscalnotes/html/HB00009I.htm + https://capitol.texas.gov/tlodocs/89R/fiscalnotes/pdf/HB00009I.pdf + ftp://ftp.legis.state.tx.us/bills/89R/fiscalNotes/HTML/house_bills/HB00001_HB00099/HB00009I.HTM + ftp://ftp.legis.state.tx.us/bills/89R/fiscalNotes/PDF/house_bills/HB00001_HB00099/HB00009I.PDF + + + + + + + + 06/12/2025 + E100 + Effective immediately + + + 11/12/2024 + H001 + Filed + + + 05/19/2025 + H630 + Record vote + RV#2999 + + + diff --git a/apps/scraper/src/scrapers/kansas-city-council.test.ts b/apps/scraper/src/scrapers/kansas-city-council.test.ts new file mode 100644 index 00000000..1e6f0c66 --- /dev/null +++ b/apps/scraper/src/scrapers/kansas-city-council.test.ts @@ -0,0 +1,112 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import test from "node:test"; + +import { + adaptKansasCityItem, + adaptKansasCityMeeting, + adaptKansasCityVote, + currentKansasCityCouncilCycleStart, + isDiscoverableKansasCityEvent, + parseKansasCityStart, +} from "./disabled/kansas-city-council.js"; + +async function fixture(name: string): Promise { + const url = new URL( + `./fixtures/kansas-city-council/${name}.json`, + import.meta.url, + ); + return JSON.parse(await readFile(url, "utf8")) as unknown; +} + +void test("maps regular, special, cancelled, and revised Council meetings", async () => { + const events = (await fixture("events")) as unknown[]; + const [regular, special, cancelled, revised] = events.map( + adaptKansasCityMeeting, + ); + + assert.equal(regular?.meetingType, "Regular Meeting"); + assert.match(regular?.videoUrl ?? "", /ID1=14001/); + assert.equal(special?.meetingType, "Special Meeting"); + assert.equal(special?.location, "KCPD Headquarters - Community Room"); + assert.equal(cancelled?.isCancelled, true); + assert.equal(cancelled?.status, "cancelled"); + assert.equal(revised?.isAmended, true); + assert.equal(revised?.documents[0]?.title, "Revised Agenda"); + assert.equal(revised?.documents[1]?.title, "Revised Minutes"); +}); + +void test("maps legislation references, actions, packets, and named votes", async () => { + const [rawItem, rawRollCall] = (await fixture("items")) as unknown[]; + const [rawNay, rawAye] = (await fixture("votes")) as unknown[]; + const item = adaptKansasCityItem(rawItem); + const rollCall = adaptKansasCityItem(rawRollCall); + const nay = adaptKansasCityVote(rawNay); + const aye = adaptKansasCityVote(rawAye); + + assert.equal(item.itemNumber, "260582"); + assert.equal(item.itemType, "Ordinance"); + assert.equal(item.action, "Passed as Substituted"); + assert.equal(item.outcome, "Pass"); + assert.equal(item.shouldFetchVotes, true); + assert.equal(item.documents[0]?.type, "packet"); + assert.equal(item.documents[1]?.type, "attachment"); + assert.equal(rollCall.section, "ROLL CALL"); + assert.equal(rollCall.shouldFetchVotes, true); + assert.deepEqual( + [nay.voterName, nay.value, aye.voterName, aye.value], + ["Quinton Lucas", "Nay", "Kevin O'Neill", "Aye"], + ); +}); + +void test("keeps stable IDs while revisions and document replacements change hashes", async () => { + const [raw] = (await fixture("events")) as Record[]; + const original = adaptKansasCityMeeting(raw); + const replacement = adaptKansasCityMeeting({ + ...raw, + EventAgendaFile: + "https://kansascity.legistar1.com/meetings/19001-agenda-v2.pdf", + EventAgendaLastPublishedUTC: "2026-01-08T19:00:00.000", + EventLastModifiedUtc: "2026-01-08T19:01:00.000", + EventRowVersion: "kc-event-v2", + }); + + assert.equal(original.externalId, replacement.externalId); + assert.notEqual(original.sourceVersion, replacement.sourceVersion); + assert.notEqual(original.contentHash, replacement.contentHash); + assert.notEqual( + original.documents[0]?.checksum, + replacement.documents[0]?.checksum, + ); +}); + +void test("uses Central Time DST and the active four-year Council term", () => { + assert.equal( + parseKansasCityStart("2026-01-08T00:00:00", "2:00 PM").toISOString(), + "2026-01-08T20:00:00.000Z", + ); + assert.equal( + parseKansasCityStart("2026-07-02T00:00:00", "2:00 PM").toISOString(), + "2026-07-02T19:00:00.000Z", + ); + assert.equal( + currentKansasCityCouncilCycleStart( + new Date("2026-07-22T00:00:00Z"), + ).toISOString(), + "2023-08-01T00:00:00.000Z", + ); +}); + +void test("excludes hidden test meetings from primary-body discovery", async () => { + const [raw] = (await fixture("events")) as Record[]; + assert.equal(isDiscoverableKansasCityEvent(raw), true); + assert.equal( + isDiscoverableKansasCityEvent({ + ...raw, + EventId: 99999, + EventAgendaStatusName: "Hidden", + EventComment: "Test Meeting", + }), + false, + ); +}); diff --git a/apps/scraper/src/scrapers/missouri-legislature.test.ts b/apps/scraper/src/scrapers/missouri-legislature.test.ts new file mode 100644 index 00000000..0882a72b --- /dev/null +++ b/apps/scraper/src/scrapers/missouri-legislature.test.ts @@ -0,0 +1,118 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import test from "node:test"; + +import { CreateBillSchema } from "@acme/db/schema"; + +import { + missouriCommitteesFromActions, + missouriEffectiveDateFromActions, + parseMissouriBill, + parseMissouriBillList, + parseMissouriSenateActionList, + parseMissouriSessions, +} from "./disabled/missouri-legislature-parser.js"; +import { missouriRefreshExpiresAt } from "./disabled/missouri-legislature-source.js"; + +const fixture = (name: string) => + readFile(new URL(`./fixtures/${name}`, import.meta.url), "utf8"); + +void test("uses an exact 30-minute durable refresh expiration", () => { + assert.equal( + missouriRefreshExpiresAt( + new Date("2026-07-22T18:00:00.000Z"), + ).toISOString(), + "2026-07-22T18:30:00.000Z", + ); +}); + +void test("discovers enabled regular and special sessions from SessionSet.js", async () => { + assert.deepEqual( + parseMissouriSessions(await fixture("missouri-session-set.js")), + [ + { + code: "261", + baseUrl: "https://documents.house.mo.gov/xml/261-", + kind: "regular", + }, + { + code: "263", + baseUrl: "https://documents.house.mo.gov/xml/263-", + kind: "special", + }, + { + code: "264", + baseUrl: "https://documents.house.mo.gov/xml/264-", + kind: "special", + }, + ], + ); +}); + +void test("parses stable bill-list versions for changed-row filtering", async () => { + const entries = parseMissouriBillList( + await fixture("missouri-bill-list.xml"), + ); + assert.equal(entries[0]?.billNumber, "HB 42"); + assert.equal(entries[0]?.sourceVersion, "07-22-2026 10:30:48.253"); + assert.equal( + entries[0]?.sourceUpdatedAt?.toISOString(), + "2026-07-22T15:30:48.253Z", + ); + const persisted = new Map([["HB 42", entries[0]!.sourceVersion]]); + assert.deepEqual( + entries + .filter( + (entry) => persisted.get(entry.billNumber) !== entry.sourceVersion, + ) + .map((entry) => entry.billNumber), + ["HJR 7"], + ); +}); + +void test("normalizes Missouri sponsors, actions, committees, dates, votes, and documents", async () => { + const bill = parseMissouriBill(await fixture("missouri-hb42.xml"), { + session: "261", + sourceVersion: "07-22-2026 10:30:48.253", + sourceUpdatedAt: new Date("2026-07-22T15:30:48.253Z"), + coverage: "complete_house_export", + }); + assert.equal(bill.billNumber, "HB 42"); + assert.equal(bill.sponsor, "Jamie Example"); + assert.equal(missouriEffectiveDateFromActions(bill.actions), "2026-08-28"); + assert.deepEqual(missouriCommitteesFromActions(bill.actions), [ + "Crime Prevention and Public Safety", + ]); + assert.deepEqual(bill.votes[0]?.counts, [ + { option: "Yes", value: 10 }, + { option: "No", value: 2 }, + { option: "Present", value: 1 }, + ]); + assert.deepEqual( + bill.documents.map((document) => document.type), + ["bill_text", "analysis", "fiscal_note", "analysis", "analysis"], + ); + assert.equal( + CreateBillSchema.safeParse({ + ...bill, + sourceWebsite: "documents.house.mo.gov", + }).success, + true, + ); +}); + +void test("labels SenateActList rows as House-actions-only coverage", async () => { + const [bill] = parseMissouriSenateActionList( + await fixture("missouri-senate-actions.xml"), + "261", + "sha256:fixture", + new Date("2026-07-22T18:00:00.000Z"), + ); + assert.equal(bill?.billNumber, "SB 9"); + assert.equal(bill?.chamber, "Senate"); + assert.equal(bill?.versions[0]?.changes, "senate_with_house_actions_only"); + assert.equal(bill?.versions[0]?.updatedAt, "2026-07-22T18:00:00.000Z"); + assert.deepEqual(missouriCommitteesFromActions(bill?.actions ?? []), [ + "Health and Mental Health", + ]); +}); diff --git a/apps/scraper/src/scrapers/missouri-sos-parsers.test.ts b/apps/scraper/src/scrapers/missouri-sos-parsers.test.ts new file mode 100644 index 00000000..6db41233 --- /dev/null +++ b/apps/scraper/src/scrapers/missouri-sos-parsers.test.ts @@ -0,0 +1,157 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import test from "node:test"; + +import { + discoverMissouriResultsUrl, + parseMissouriBallotMeasures, + parseMissouriCalendar, + parseMissouriCandidateDiscovery, + parseMissouriCandidateOffice, + parseMissouriResults, + parseMissouriWithdrawals, +} from "./disabled/missouri-sos-parsers.js"; + +const fixture = (name: string) => + readFile( + new URL(`../fixtures/missouri-sos/${name}`, import.meta.url), + "utf8", + ); + +void test("discovers the active primary and dynamic certified candidate endpoints", async () => { + const active = parseMissouriCalendar( + await fixture("calendar.html"), + new Date("2026-07-22T12:00:00Z"), + ); + assert.equal(active.type, "primary"); + assert.equal(active.electionDate, "2026-08-04"); + assert.equal(active.finalCertificationDate, "2026-05-26"); + assert.equal( + parseMissouriCalendar( + await fixture("calendar.html"), + new Date("2026-08-10T12:00:00Z"), + undefined, + "primary", + ).type, + "primary", + ); + + const discovery = parseMissouriCandidateDiscovery( + await fixture("candidate-index.html"), + ); + assert.equal(discovery.electionCode, "750006905"); + assert.equal(discovery.offices.length, 2); + assert.match(discovery.candidatesUrl, /ElectionCode=750006905/); + assert.match(discovery.withdrawalsUrl, /CandidatesRemoved/); +}); + +void test("parses ballot order, parties, districts, and excludes candidate addresses", async () => { + const candidates = parseMissouriCandidateOffice( + await fixture("candidates.html"), + "https://s1.sos.mo.gov/candidatesonweb/cumulative", + ); + assert.deepEqual( + candidates.map(({ name, party, office, district, ballotOrder }) => ({ + name, + party, + office, + district, + ballotOrder, + })), + [ + { + name: "Alice First", + party: "Republican", + office: "State Senator", + district: "2", + ballotOrder: 1, + }, + { + name: "Bob Second", + party: "Republican", + office: "State Senator", + district: "2", + ballotOrder: 2, + }, + { + name: "Carol Third", + party: "Democratic", + office: "State Senator", + district: "2", + ballotOrder: 1, + }, + { + name: "Dana Fourth", + party: "Libertarian", + office: "State Representative", + district: "11", + ballotOrder: 1, + }, + ], + ); + assert.doesNotMatch( + JSON.stringify(candidates), + /Secret|Private|Hidden|Confidential/, + ); +}); + +void test("parses withdrawals without retaining their address nodes", async () => { + const candidates = parseMissouriWithdrawals( + await fixture("withdrawals.html"), + "https://s1.sos.mo.gov/candidatesonweb/withdrawals", + ); + assert.equal(candidates[0]?.status, "withdrawn"); + assert.equal(candidates[0]?.withdrawalDate, "2026-06-09"); + assert.equal(candidates[1]?.status, "removed"); + assert.doesNotMatch(JSON.stringify(candidates), /Private Street|PO Box/); +}); + +void test("parses only certified 2026 measures with language, fiscal text, and source files", async () => { + const measures = parseMissouriBallotMeasures(await fixture("measures.html")); + assert.equal(measures.length, 2); + assert.deepEqual( + measures.map((measure) => [measure.officialTitle, measure.electionDate]), + [ + ["Amendment 1", "2026-08-04"], + ["Amendment 3", "2026-11-03"], + ], + ); + assert.match(measures[0]?.officialBallotLanguage ?? "", /Constitution/); + assert.match(measures[0]?.fairBallotLanguage ?? "", /yes/); + assert.match(measures[0]?.fiscalStatement ?? "", /costs or savings/); + assert.match(measures[0]?.certificateUrl ?? "", /certificate-amendment-1/); +}); + +void test("fails soft when current results are absent and detects deterministic updates", async () => { + const active = parseMissouriCalendar( + await fixture("calendar.html"), + new Date("2026-07-22T12:00:00Z"), + ); + assert.equal( + discoverMissouriResultsUrl( + await fixture("showmo-unavailable.html"), + active, + ), + null, + ); + assert.equal( + discoverMissouriResultsUrl( + '
August 4, 2026 Primary Election Results
', + active, + ), + "https://www.sos.mo.gov/elections/2026-primary-results", + ); + + const layout = await fixture("historical-results-layout.html"); + const first = parseMissouriResults( + layout, + "https://example.test/fixture-only", + ); + const updated = parseMissouriResults( + layout.replace("1,250", "1,300").replace("11:45 PM", "11:55 PM"), + "https://example.test/fixture-only", + ); + assert.equal(first.contests[0]?.totalVotes, 2399); + assert.equal(updated.contests[0]?.totalVotes, 2449); + assert.notEqual(first.updatedAt, updated.updatedAt); +}); diff --git a/apps/scraper/src/scrapers/ncsbe-parsers.test.ts b/apps/scraper/src/scrapers/ncsbe-parsers.test.ts new file mode 100644 index 00000000..c9abd4d0 --- /dev/null +++ b/apps/scraper/src/scrapers/ncsbe-parsers.test.ts @@ -0,0 +1,58 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import test from "node:test"; + +import { + parseCandidateCsv, + parseReferendumLines, + parseResultsTsv, + restrictToCycle, +} from "./ncsbe-parsers.js"; + +const fixture = (name: string) => + readFile(new URL(`../fixtures/ncsbe/${name}`, import.meta.url), "utf8"); + +test("parses current-cycle candidate CSV without private contact fields", async () => { + const rows = parseCandidateCsv(await fixture("candidates-2026.csv")); + assert.equal(rows.length, 2); + assert.deepEqual(rows[0], { + electionDate: "2026-03-03", + county: "DURHAM", + contest: "US HOUSE OF REPRESENTATIVES DISTRICT 04", + name: "Nida Allam", + party: "DEM", + voteFor: 1, + termYears: 2, + hasPrimary: true, + isPartisan: true, + }); + assert.equal("email" in rows[0]!, false); + assert.deepEqual( + rows.map((row) => row.county), + ["DURHAM", "WAKE"], + ); +}); + +test("parses NCSBE result layouts from 2026 and 2024 fixtures", async () => { + const current = parseResultsTsv(await fixture("results-2026.tsv")); + const legacy = parseResultsTsv(await fixture("results-2024.tsv")); + assert.equal(current.length, 2); + assert.equal(current[0]?.totalVotes, 371); + assert.equal(current[0]?.earlyVotingVotes, 0); + assert.equal(legacy.length, 2); + assert.equal(legacy[0]?.choice, "Kamala D. Harris"); + assert.deepEqual(restrictToCycle([...current, ...legacy], 2026), current); +}); + +test("parses current-cycle referendum PDF text across counties", async () => { + const rows = parseReferendumLines( + (await fixture("referendums-2026.txt")).split(/\r?\n/), + ); + assert.equal(rows.length, 4); + assert.deepEqual( + [...new Set(rows.map((row) => row.county))], + ["GATES", "GRANVILLE"], + ); + assert.equal(rows[0]?.electionDate, "2026-03-03"); + assert.equal(rows[0]?.choice, "For"); +}); diff --git a/apps/scraper/src/scrapers/ncsbe-parsers.ts b/apps/scraper/src/scrapers/ncsbe-parsers.ts new file mode 100644 index 00000000..bcfa3d76 --- /dev/null +++ b/apps/scraper/src/scrapers/ncsbe-parsers.ts @@ -0,0 +1,328 @@ +import { inflateRawSync } from "node:zlib"; + +export const NCSBE_STRUCTURE_VERSION = "ncsbe-public-election-v1"; + +export interface NcsbeCandidateRecord { + electionDate: string; + county: string; + contest: string; + name: string; + party: string | null; + voteFor: number | null; + termYears: number | null; + hasPrimary: boolean | null; + isPartisan: boolean | null; +} + +export interface NcsbeReferendumRecord { + electionDate: string; + county: string; + contest: string; + choice: string; + description: string | null; +} + +export interface NcsbeResultRecord { + electionDate: string; + county: string; + precinct: string; + contestId: string | null; + contestType: string | null; + contest: string; + choice: string; + party: string | null; + voteFor: number | null; + electionDayVotes: number; + earlyVotingVotes: number; + absenteeMailVotes: number; + provisionalVotes: number; + totalVotes: number; + realPrecinct: boolean | null; +} + +function clean(value: string | undefined): string { + return (value ?? "").replace(/^\uFEFF/, "").trim(); +} + +function nullable(value: string | undefined): string | null { + const result = clean(value); + return result ? result : null; +} + +function integer(value: string | undefined): number | null { + const normalized = clean(value).replace(/,/g, ""); + if (!normalized) return null; + const result = Number(normalized); + return Number.isSafeInteger(result) ? result : null; +} + +function boolean(value: string | undefined): boolean | null { + const normalized = clean(value).toLowerCase(); + if (["true", "t", "yes", "y", "1"].includes(normalized)) return true; + if (["false", "f", "no", "n", "0"].includes(normalized)) return false; + return null; +} + +export function normalizeElectionDate(value: string): string | null { + const normalized = clean(value); + const slash = /^(\d{1,2})\/(\d{1,2})\/(\d{4})$/.exec(normalized); + if (slash) { + return `${slash[3]}-${slash[1]!.padStart(2, "0")}-${slash[2]!.padStart(2, "0")}`; + } + return /^\d{4}-\d{2}-\d{2}$/.test(normalized) ? normalized : null; +} + +/** Small RFC 4180 parser; NCSBE files use commas, quoted fields and CRLF. */ +export function parseDelimited(text: string, delimiter: string): string[][] { + const rows: string[][] = []; + let row: string[] = []; + let field = ""; + let quoted = false; + + for (let i = 0; i < text.length; i++) { + const char = text[i]!; + if (quoted) { + if (char === '"' && text[i + 1] === '"') { + field += '"'; + i++; + } else if (char === '"') { + quoted = false; + } else { + field += char; + } + } else if (char === '"') { + quoted = true; + } else if (char === delimiter) { + row.push(field); + field = ""; + } else if (char === "\n") { + row.push(field.replace(/\r$/, "")); + if (row.some((cell) => cell.length > 0)) rows.push(row); + row = []; + field = ""; + } else { + field += char; + } + } + if (field.length > 0 || row.length > 0) { + row.push(field.replace(/\r$/, "")); + if (row.some((cell) => cell.length > 0)) rows.push(row); + } + return rows; +} + +function records(text: string, delimiter: string): Record[] { + const [rawHeaders, ...rows] = parseDelimited(text, delimiter); + if (!rawHeaders) return []; + const headers = rawHeaders.map((header) => clean(header).toLowerCase()); + return rows.map((row) => + Object.fromEntries( + headers.map((header, index) => [header, row[index] ?? ""]), + ), + ); +} + +function first(row: Record, ...keys: string[]): string { + for (const key of keys) { + if (row[key] !== undefined) return row[key]; + } + return ""; +} + +export function parseCandidateCsv(text: string): NcsbeCandidateRecord[] { + return records(text, ",").flatMap((row) => { + const electionDate = normalizeElectionDate( + first(row, "election_dt", "election_date"), + ); + const county = clean(first(row, "county_name", "county")); + const contest = clean(first(row, "contest_name", "contest")); + const name = clean( + first(row, "name_on_ballot", "candidate_name", "choice"), + ); + if (!electionDate || !county || !contest || !name) return []; + return [ + { + electionDate, + county, + contest, + name, + party: nullable( + first(row, "party_candidate", "candidate_party", "party"), + ), + voteFor: integer(first(row, "vote_for")), + termYears: integer(first(row, "term", "term_years")), + hasPrimary: boolean(first(row, "has_primary")), + isPartisan: boolean(first(row, "is_partisan")), + }, + ]; + }); +} + +export function parseResultsTsv(text: string): NcsbeResultRecord[] { + return records(text, "\t").flatMap((row) => { + const electionDate = normalizeElectionDate( + first(row, "election date", "election_date"), + ); + const county = clean(first(row, "county")); + const precinct = clean(first(row, "precinct")); + const contest = clean(first(row, "contest name", "contest_name")); + const choice = clean(first(row, "choice", "candidate")); + const totals = [ + integer(first(row, "election day", "election_day")), + integer(first(row, "early voting", "early_voting")), + integer(first(row, "absentee by mail", "absentee_by_mail")), + integer(first(row, "provisional")), + integer(first(row, "total votes", "total_votes")), + ]; + if ( + !electionDate || + !county || + !precinct || + !contest || + !choice || + totals.some((n) => n === null) + ) { + return []; + } + return [ + { + electionDate, + county, + precinct, + contestId: nullable(first(row, "contest group id", "contest_group_id")), + contestType: nullable(first(row, "contest type", "contest_type")), + contest, + choice, + party: nullable(first(row, "choice party", "choice_party", "party")), + voteFor: integer(first(row, "vote for", "vote_for")), + electionDayVotes: totals[0]!, + earlyVotingVotes: totals[1]!, + absenteeMailVotes: totals[2]!, + provisionalVotes: totals[3]!, + totalVotes: totals[4]!, + realPrecinct: boolean(first(row, "real precinct", "real_precinct")), + }, + ]; + }); +} + +const NC_COUNTIES = new Set( + `ALAMANCE ALEXANDER ALLEGHANY ANSON ASHE AVERY BEAUFORT BERTIE BLADEN BRUNSWICK BUNCOMBE BURKE CABARRUS CALDWELL CAMDEN CARTERET CASWELL CATAWBA CHATHAM CHEROKEE CHOWAN CLAY CLEVELAND COLUMBUS CRAVEN CUMBERLAND CURRITUCK DARE DAVIDSON DAVIE DUPLIN DURHAM EDGECOMBE FORSYTH FRANKLIN GASTON GATES GRAHAM GRANVILLE GREENE GUILFORD HALIFAX HARNETT HAYWOOD HENDERSON HERTFORD HOKE HYDE IREDELL JACKSON JOHNSTON JONES LEE LENOIR LINCOLN MACON MADISON MARTIN MCDOWELL MECKLENBURG MITCHELL MONTGOMERY MOORE NASH NEW HANOVER NORTHAMPTON ONSLOW ORANGE PAMLICO PASQUOTANK PENDER PERQUIMANS PERSON PITT POLK RANDOLPH RICHMOND ROBESON ROCKINGHAM ROWAN RUTHERFORD SAMPSON SCOTLAND STANLY STOKES SURRY SWAIN TRANSYLVANIA TYRRELL UNION VANCE WAKE WARREN WASHINGTON WATAUGA WAYNE WILKES WILSON YADKIN YANCEY`.split( + " ", + ), +); + +/** Parse position-sorted PDF text lines from NCSBE referendum reports. */ +export function parseReferendumLines( + lines: readonly string[], + sourceElectionDate?: string, +): NcsbeReferendumRecord[] { + let electionDate = sourceElectionDate ?? null; + let county: string | null = null; + let contest: string | null = null; + const output: NcsbeReferendumRecord[] = []; + + for (const raw of lines) { + const line = clean(raw).replace(/\s+/g, " "); + if (!line) continue; + const criteriaDate = /Election:\s*(\d{1,2}\/\d{1,2}\/\d{4})/i.exec( + line, + )?.[1]; + if (criteriaDate) electionDate = normalizeElectionDate(criteriaDate); + const headerCounty = /^([A-Z ]+) BOARD OF ELECTIONS$/ + .exec(line)?.[1] + ?.trim(); + if (headerCounty && NC_COUNTIES.has(headerCounty)) { + county = headerCounty; + contest = null; + continue; + } + if (NC_COUNTIES.has(line)) { + county = line; + contest = null; + continue; + } + if ( + /^(?:REFERENDUM CHOICES|CHOICE DESCRIPTION|CRITERIA:|Page \d+|\w{3} \d{1,2}, \d{4})/i.test( + line, + ) + ) { + continue; + } + const choice = /^(For|Against|Yes|No)\b[\s:.-]*(.*)$/i.exec(line); + if (choice && county && contest && electionDate) { + output.push({ + electionDate, + county, + contest, + choice: + choice[1]![0]!.toUpperCase() + choice[1]!.slice(1).toLowerCase(), + description: nullable(choice[2]), + }); + continue; + } + if (county && /^[A-Z0-9][A-Z0-9 '&().,/%-]+$/.test(line)) contest = line; + } + return output; +} + +/** Read the first text/CSV entry from a conventional NCSBE ZIP archive. */ +export function extractFirstTextFileFromZip(bytes: Uint8Array): string { + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + let eocd = -1; + for ( + let i = bytes.length - 22; + i >= Math.max(0, bytes.length - 65_557); + i-- + ) { + if (view.getUint32(i, true) === 0x06054b50) { + eocd = i; + break; + } + } + if (eocd < 0) + throw new Error("ZIP end-of-central-directory record not found"); + const entries = view.getUint16(eocd + 10, true); + let offset = view.getUint32(eocd + 16, true); + + for (let entry = 0; entry < entries; entry++) { + if (view.getUint32(offset, true) !== 0x02014b50) + throw new Error("Invalid ZIP central directory"); + const method = view.getUint16(offset + 10, true); + const compressedSize = view.getUint32(offset + 20, true); + const fileNameLength = view.getUint16(offset + 28, true); + const extraLength = view.getUint16(offset + 30, true); + const commentLength = view.getUint16(offset + 32, true); + const localOffset = view.getUint32(offset + 42, true); + const fileName = new TextDecoder().decode( + bytes.subarray(offset + 46, offset + 46 + fileNameLength), + ); + offset += 46 + fileNameLength + extraLength + commentLength; + if (!/\.(?:txt|csv|tsv)$/i.test(fileName)) continue; + if (view.getUint32(localOffset, true) !== 0x04034b50) + throw new Error("Invalid ZIP local header"); + const localNameLength = view.getUint16(localOffset + 26, true); + const localExtraLength = view.getUint16(localOffset + 28, true); + const start = localOffset + 30 + localNameLength + localExtraLength; + const compressed = bytes.subarray(start, start + compressedSize); + const content = + method === 0 + ? compressed + : method === 8 + ? inflateRawSync(compressed) + : null; + if (!content) + throw new Error(`Unsupported ZIP compression method ${method}`); + return new TextDecoder().decode(content); + } + throw new Error("ZIP contains no text election-results file"); +} + +export function restrictToCycle( + rows: readonly T[], + cycleYear: number, +): T[] { + return rows.filter( + (row) => Number(row.electionDate.slice(0, 4)) === cycleYear, + ); +} diff --git a/apps/scraper/src/scrapers/ncsbe.config.ts b/apps/scraper/src/scrapers/ncsbe.config.ts new file mode 100644 index 00000000..4689c441 --- /dev/null +++ b/apps/scraper/src/scrapers/ncsbe.config.ts @@ -0,0 +1,11 @@ +import type { ScraperEnvContract } from "@acme/env"; + +export const ncsbeConfig = { + id: "ncsbe", + name: "North Carolina State Board of Elections", + source: "NCSBE candidate, referendum, and election-results files", + environment: { + required: ["POSTGRES_URL"], + optional: ["NCSBE_MAX_ITEMS"], + }, +} as const satisfies ScraperEnvContract; diff --git a/apps/scraper/src/scrapers/ncsbe.test.ts b/apps/scraper/src/scrapers/ncsbe.test.ts new file mode 100644 index 00000000..c07d555c --- /dev/null +++ b/apps/scraper/src/scrapers/ncsbe.test.ts @@ -0,0 +1,20 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { discoverCurrentCycleFiles } from "./ncsbe.js"; + +test("discovers structured current-cycle files without accepting history", () => { + const candidates = ` + 2026 Candidate List Spreadsheet (CSV) + 2024 Candidate List Spreadsheet (CSV) + 2026 Primary Referendum List`; + const results = ` + 2026 Mar 03 Election - Results (ZIP) + 2024 Nov 05 Election - Results (ZIP)`; + const files = discoverCurrentCycleFiles(candidates, results, 2026); + assert.deepEqual( + files.map((file) => file.kind), + ["candidates", "referenda", "results"], + ); + assert.ok(files.every((file) => file.url.includes("2026"))); +}); diff --git a/apps/scraper/src/scrapers/ncsbe.ts b/apps/scraper/src/scrapers/ncsbe.ts new file mode 100644 index 00000000..544d7d15 --- /dev/null +++ b/apps/scraper/src/scrapers/ncsbe.ts @@ -0,0 +1,382 @@ +/** + * Current-cycle NCSBE public election-data importer. + * + * Runtime discovery is intentionally limited to links for the current calendar + * year. Candidate contact/address fields and all voter-history products are out + * of scope and are never represented in the normalized parser output. + */ +import { createHash } from "node:crypto"; +import { load } from "cheerio"; +import { and, eq, gte, lt, or } from "drizzle-orm"; +import { getDocumentProxy } from "unpdf"; + +import { db } from "@acme/db/client"; +import { + ElectionCandidate, + ElectionReferendum, + ElectionResult, + ElectionSource, +} from "@acme/db/schema"; + +import type { Scraper } from "../utils/types.js"; +import type { + NcsbeCandidateRecord, + NcsbeReferendumRecord, + NcsbeResultRecord, +} from "./ncsbe-parsers.js"; +import { fetchWithRetry } from "../utils/fetch.js"; +import { createLogger } from "../utils/log.js"; +import { + extractFirstTextFileFromZip, + NCSBE_STRUCTURE_VERSION, + normalizeElectionDate, + parseCandidateCsv, + parseReferendumLines, + parseResultsTsv, + restrictToCycle, +} from "./ncsbe-parsers.js"; +import { ncsbeConfig } from "./ncsbe.config.js"; + +const logger = createLogger("ncsbe"); +const CANDIDATE_LIST_URL = "https://www.ncsbe.gov/results-data/candidate-lists"; +const RESULTS_LIST_URL = + "https://www.ncsbe.gov/results-data/election-results/historical-election-results-data"; +const USER_AGENT = "BillionCivicBot/1.0 (+https://billion.app)"; +const BATCH_SIZE = 750; + +type SourceKind = "candidates" | "referenda" | "results"; +type NcsbeRecord = + | NcsbeCandidateRecord + | NcsbeReferendumRecord + | NcsbeResultRecord; + +export interface NcsbeSourceFile { + kind: SourceKind; + url: string; + label: string; +} + +function currentCycleYear(now = new Date()): number { + return now.getUTCFullYear(); +} + +function absoluteUrl(href: string, base: string): string | null { + try { + return new URL(href, base).toString(); + } catch { + return null; + } +} + +function links(html: string, base: string): { label: string; url: string }[] { + const $ = load(html); + return $("a[href]") + .toArray() + .flatMap((anchor) => { + const url = absoluteUrl($(anchor).attr("href") ?? "", base); + return url + ? [{ label: $(anchor).text().replace(/\s+/g, " ").trim(), url }] + : []; + }); +} + +/** Discover only official files whose label/path identifies the current year. */ +export function discoverCurrentCycleFiles( + candidateHtml: string, + resultsHtml: string, + year = currentCycleYear(), +): NcsbeSourceFile[] { + const yearText = String(year); + const discovered: NcsbeSourceFile[] = []; + for (const link of links(candidateHtml, CANDIDATE_LIST_URL)) { + const haystack = `${link.label} ${decodeURIComponent(link.url)}`; + if (!haystack.includes(yearText)) continue; + if (/candidate/i.test(link.label) && /\.csv(?:$|\?)/i.test(link.url)) { + discovered.push({ kind: "candidates", ...link }); + } else if ( + /referendum/i.test(haystack) && + /\.pdf(?:$|\?)/i.test(link.url) + ) { + discovered.push({ kind: "referenda", ...link }); + } + } + for (const link of links(resultsHtml, RESULTS_LIST_URL)) { + const haystack = `${link.label} ${decodeURIComponent(link.url)}`; + if ( + haystack.includes(yearText) && + /results/i.test(link.label) && + /\.zip(?:$|\?)/i.test(link.url) && + !/precinct sort/i.test(link.label) + ) { + discovered.push({ kind: "results", ...link }); + } + } + return [ + ...new Map( + discovered.map((file) => [`${file.kind}:${file.url}`, file]), + ).values(), + ].sort((a, b) => a.kind.localeCompare(b.kind) || a.url.localeCompare(b.url)); +} + +async function fetchBytes(url: string): Promise { + const response = await fetchWithRetry(url, { + headers: { "User-Agent": USER_AGENT }, + maxRetries: 3, + timeoutMs: 45_000, + }); + return new Uint8Array(await response.arrayBuffer()); +} + +async function fetchPage(url: string): Promise { + return new TextDecoder().decode(await fetchBytes(url)); +} + +export async function extractPdfLines(bytes: Uint8Array): Promise { + const pdf = await getDocumentProxy(bytes); + const lines: string[] = []; + for (let pageNumber = 1; pageNumber <= pdf.numPages; pageNumber++) { + const page = await pdf.getPage(pageNumber); + const content = await page.getTextContent(); + const items = (content.items as unknown[]) + .flatMap((raw) => { + const item = raw as { str?: string; transform?: number[] }; + return typeof item.str === "string" && item.transform + ? [ + { + text: item.str.trim(), + x: item.transform[4] ?? 0, + y: item.transform[5] ?? 0, + }, + ] + : []; + }) + .filter((item) => item.text); + items.sort((a, b) => (Math.abs(a.y - b.y) > 2 ? b.y - a.y : a.x - b.x)); + let activeY: number | null = null; + let active: typeof items = []; + const flush = () => { + if (active.length) + lines.push( + active + .sort((a, b) => a.x - b.x) + .map((item) => item.text) + .join(" "), + ); + active = []; + }; + for (const item of items) { + if (activeY !== null && Math.abs(activeY - item.y) > 2) flush(); + activeY = item.y; + active.push(item); + } + flush(); + } + return lines; +} + +export async function parseReferendumPdf( + bytes: Uint8Array, + sourceUrl: string, +): Promise { + return parseReferendumLines( + await extractPdfLines(bytes), + sourceDateFromUrl(sourceUrl) ?? undefined, + ); +} + +function sha256(bytes: Uint8Array): string { + return createHash("sha256").update(bytes).digest("hex"); +} + +function sourceDateFromUrl(url: string): string | null { + const compact = /(?:_|referendums_)(20\d{2})(\d{2})(\d{2})/.exec(url); + return compact + ? normalizeElectionDate(`${compact[2]}/${compact[3]}/${compact[1]}`) + : null; +} + +function groupsByDate( + rows: readonly T[], +): Map { + const groups = new Map(); + for (const row of rows) + groups.set(row.electionDate, [ + ...(groups.get(row.electionDate) ?? []), + row, + ]); + return groups; +} + +async function insertBatches( + rows: readonly T[], + insert: (batch: T[]) => Promise, +): Promise { + for (let start = 0; start < rows.length; start += BATCH_SIZE) { + await insert(rows.slice(start, start + BATCH_SIZE)); + } +} + +async function persistDateGroup( + file: NcsbeSourceFile, + electionDate: string, + checksum: string, + fetchedAt: Date, + rows: NcsbeRecord[], +): Promise { + await db.transaction(async (tx) => { + const [cached] = await tx + .select({ + id: ElectionSource.id, + checksum: ElectionSource.checksum, + structureVersion: ElectionSource.structureVersion, + }) + .from(ElectionSource) + .where( + and( + eq(ElectionSource.provider, "ncsbe"), + eq(ElectionSource.sourceKind, file.kind), + eq(ElectionSource.electionDate, electionDate), + eq(ElectionSource.sourceUrl, file.url), + ), + ) + .limit(1); + if ( + cached?.checksum === checksum && + cached.structureVersion === NCSBE_STRUCTURE_VERSION + ) { + await tx + .update(ElectionSource) + .set({ fetchedAt }) + .where(eq(ElectionSource.id, cached.id)); + return; + } + + const [source] = await tx + .insert(ElectionSource) + .values({ + provider: "ncsbe", + sourceKind: file.kind, + electionDate, + sourceUrl: file.url, + checksum, + structureVersion: NCSBE_STRUCTURE_VERSION, + certificationStatus: + file.kind === "results" ? "official_not_certified" : "not_applicable", + fetchedAt, + }) + .onConflictDoUpdate({ + target: [ + ElectionSource.provider, + ElectionSource.sourceKind, + ElectionSource.electionDate, + ElectionSource.sourceUrl, + ], + set: { checksum, structureVersion: NCSBE_STRUCTURE_VERSION, fetchedAt }, + }) + .returning({ id: ElectionSource.id }); + if (!source) throw new Error(`Unable to upsert source ${file.url}`); + + if (file.kind === "candidates") { + await tx + .delete(ElectionCandidate) + .where(eq(ElectionCandidate.sourceId, source.id)); + const candidates = rows as NcsbeCandidateRecord[]; + await insertBatches(candidates, (batch) => + tx + .insert(ElectionCandidate) + .values(batch.map((row) => ({ ...row, sourceId: source.id }))), + ); + } else if (file.kind === "referenda") { + await tx + .delete(ElectionReferendum) + .where(eq(ElectionReferendum.sourceId, source.id)); + const referenda = rows as NcsbeReferendumRecord[]; + await insertBatches(referenda, (batch) => + tx + .insert(ElectionReferendum) + .values(batch.map((row) => ({ ...row, sourceId: source.id }))), + ); + } else { + await tx + .delete(ElectionResult) + .where(eq(ElectionResult.sourceId, source.id)); + const results = rows as NcsbeResultRecord[]; + await insertBatches(results, (batch) => + tx + .insert(ElectionResult) + .values(batch.map((row) => ({ ...row, sourceId: source.id }))), + ); + } + }); +} + +async function importFile( + file: NcsbeSourceFile, + year: number, +): Promise { + const fetchedAt = new Date(); + const bytes = await fetchBytes(file.url); + const checksum = sha256(bytes); + let rows: NcsbeRecord[]; + if (file.kind === "candidates") { + rows = parseCandidateCsv(new TextDecoder().decode(bytes)); + } else if (file.kind === "results") { + rows = parseResultsTsv(extractFirstTextFileFromZip(bytes)); + } else { + rows = await parseReferendumPdf(bytes, file.url); + } + rows = restrictToCycle(rows, year); + if (rows.length === 0) + throw new Error(`${file.url} produced no current-cycle records`); + for (const [electionDate, dateRows] of groupsByDate(rows)) { + await persistDateGroup(file, electionDate, checksum, fetchedAt, dateRows); + } + return rows.length; +} + +export async function scrapeNcsbe( + maxItems = 4, + now = new Date(), +): Promise { + const year = currentCycleYear(now); + logger.info(`Discovering NCSBE public election files for the ${year} cycle…`); + const [candidateHtml, resultsHtml] = await Promise.all([ + fetchPage(CANDIDATE_LIST_URL), + fetchPage(RESULTS_LIST_URL), + ]); + const files = discoverCurrentCycleFiles( + candidateHtml, + resultsHtml, + year, + ).slice(0, maxItems); + if (files.length === 0) + throw new Error(`No NCSBE files discovered for ${year}`); + for (const file of files) { + const count = await importFile(file, year); + logger.success(`${file.kind}: persisted ${count} records from ${file.url}`); + } + // Keep persistence bounded to the product's current-cycle scope. Cascading + // foreign keys remove normalized rows only after every current import succeeds. + const stale = await db + .delete(ElectionSource) + .where( + and( + eq(ElectionSource.provider, "ncsbe"), + or( + lt(ElectionSource.electionDate, `${year}-01-01`), + gte(ElectionSource.electionDate, `${year + 1}-01-01`), + ), + ), + ) + .returning({ id: ElectionSource.id }); + if (stale.length) + logger.info(`Removed ${stale.length} out-of-cycle NCSBE source snapshots.`); +} + +export const ncsbe: Scraper = { + ...ncsbeConfig, + scrape: (options) => + scrapeNcsbe( + (options?.maxItems ?? Number(process.env.NCSBE_MAX_ITEMS)) || 4, + ), +}; diff --git a/apps/scraper/src/scrapers/st-louis-aldermen-parser.test.ts b/apps/scraper/src/scrapers/st-louis-aldermen-parser.test.ts new file mode 100644 index 00000000..4703fafa --- /dev/null +++ b/apps/scraper/src/scrapers/st-louis-aldermen-parser.test.ts @@ -0,0 +1,111 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import test from "node:test"; + +import { + adaptStLouisCivicItems, + civicMeetingDocuments, + parseActiveAgendaSession, + parseActiveCalendarSession, + parseStLouisAgendaDetail, + parseStLouisAgendaIndex, + parseStLouisCalendar, + parseStLouisEventDetail, + parseStLouisLegislationDetail, +} from "./disabled/st-louis-aldermen-parser.js"; + +const fixture = (name: string) => + readFile( + new URL(`./fixtures/st-louis-aldermen/${name}`, import.meta.url), + "utf8", + ); + +void test("discovers only the selected active session metadata", async () => { + const agenda = await fixture("agenda-index.html"); + const calendar = await fixture("calendar.html"); + assert.deepEqual(parseActiveAgendaSession(agenda), { + id: "202", + label: "2026-2027", + }); + assert.deepEqual(parseActiveCalendarSession(calendar), { + id: "202", + label: "2026-2027", + }); + assert.equal(parseStLouisAgendaIndex(agenda).length, 2); +}); + +void test("maps current-session full-board and committee meeting identities", async () => { + const meetings = parseStLouisCalendar(await fixture("calendar.html")); + assert.equal(meetings[0]?.eventId, "53891"); + assert.equal(meetings[0]?.civicClerkId, "2408"); + assert.equal(meetings[0]?.startsAt.toISOString(), "2026-07-20T15:00:00.000Z"); + assert.equal(meetings[1]?.civicClerkId, "2409"); +}); + +void test("preserves agendaViewID, session, revised documents, bill IDs, and sponsors", async () => { + const [week] = parseStLouisAgendaIndex(await fixture("agenda-index.html")); + assert.ok(week); + const detail = parseStLouisAgendaDetail( + await fixture("agenda-detail.html"), + week, + ); + assert.equal(detail.agendaViewId, "12765"); + assert.equal(detail.sessionId, "202"); + assert.equal(detail.eventId, "53891"); + assert.equal(detail.documents.length, 2); + assert.equal(detail.legislation[0]?.externalId, "17901"); + assert.deepEqual(detail.legislation[0]?.sponsors, ["Shameem Clark Hubbard"]); +}); + +void test("links CivicClerk agenda items to official legislative metadata", async () => { + const civic = JSON.parse(await fixture("civic-meetings.json")) as unknown[]; + const event = parseStLouisEventDetail( + await fixture("event-detail.html"), + "53958", + ); + const bill = parseStLouisLegislationDetail( + await fixture("board-bill.html"), + event.legislation[0]!, + ); + const items = adaptStLouisCivicItems( + civic[0], + event.legislation[0]!.sourceUrl, + [bill], + ); + assert.equal(event.meetingType, "Aldermanic Committee Meeting"); + assert.equal(event.civicClerkId, "2409"); + assert.match(event.videoUrl ?? "", /youtube\.com/); + assert.equal(items[1]?.legislativeId, "board-bill:17901"); + assert.equal(items[1]?.title, "Youthbuild Award"); + assert.deepEqual(items[1]?.sponsors, ["Shameem Clark Hubbard"]); + assert.equal(items[1]?.action, "Committee Assignment"); + assert.equal(items[1]?.documents[0]?.externalId, "civicclerk:3724"); +}); + +void test("uses stable file IDs while allowing signed URLs and revisions to change", async () => { + const [meeting] = JSON.parse(await fixture("civic-meetings.json")) as Record< + string, + unknown + >[]; + assert.ok(meeting); + const original = civicMeetingDocuments(meeting); + const rotated = civicMeetingDocuments({ + ...meeting, + files: (meeting.files as Record[]).map((file) => ({ + ...file, + url: `${String(file.url).split("?")[0]}?sig=rotated`, + })), + }); + const revised = civicMeetingDocuments({ + ...meeting, + files: (meeting.files as Record[]).map((file, index) => ({ + ...file, + fileId: Number(file.fileId) + 100 + index, + publishedOn: "2026-07-21T09:00:00.000", + })), + }); + assert.equal(original[0]?.externalId, rotated[0]?.externalId); + assert.equal(original[0]?.checksum, rotated[0]?.checksum); + assert.notEqual(original[0]?.externalId, revised[0]?.externalId); + assert.notEqual(original[0]?.checksum, revised[0]?.checksum); +}); diff --git a/apps/scraper/src/scrapers/texas-current-election.config.ts b/apps/scraper/src/scrapers/texas-current-election.config.ts new file mode 100644 index 00000000..4366309a --- /dev/null +++ b/apps/scraper/src/scrapers/texas-current-election.config.ts @@ -0,0 +1,11 @@ +import type { ScraperEnvContract } from "@acme/env"; + +export const texasCurrentElectionConfig = { + id: "texas-current-election", + name: "Texas SOS/TLC current election cycle", + source: "Texas Secretary of State and Texas Legislative Council", + environment: { + required: ["POSTGRES_URL"], + optional: ["TX_SOS_MAX_ITEMS"], + }, +} as const satisfies ScraperEnvContract; diff --git a/apps/scraper/src/scrapers/texas-current-election.test.ts b/apps/scraper/src/scrapers/texas-current-election.test.ts new file mode 100644 index 00000000..0c546f17 --- /dev/null +++ b/apps/scraper/src/scrapers/texas-current-election.test.ts @@ -0,0 +1,17 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { discoverLatestTlcAnalysis } from "./texas-current-election.js"; + +void test("TLC discovery chooses the newest full report and skips condensed PDFs", () => { + const html = ` + November 7, 2023 + Condensed + 2025 + `; + assert.deepEqual(discoverLatestTlcAnalysis(html), { + year: 2025, + title: "2025", + url: "https://tlc.texas.gov/docs/amendments/analyses25.pdf", + }); +}); diff --git a/apps/scraper/src/scrapers/texas-current-election.ts b/apps/scraper/src/scrapers/texas-current-election.ts new file mode 100644 index 00000000..6a2dc07b --- /dev/null +++ b/apps/scraper/src/scrapers/texas-current-election.ts @@ -0,0 +1,285 @@ +/** + * Current-cycle Texas statewide election ingestion. + * + * The SOS Civix application publishes base64-wrapped JSON for election + * discovery, contests/results, reporting status, and county totals. TLC's + * publications page links the latest cycle-specific constitutional-amendment + * analysis PDF. This scraper discovers both rather than constructing year URLs, + * parses them deterministically, and stores separate provider snapshots. + */ + +import { createHash } from "node:crypto"; +import * as cheerio from "cheerio"; +import { getDocumentProxy } from "unpdf"; + +import type { + TexasElectionDefinition, + TexasSosElection, + TexasSosSnapshotData, + TexasTlcSnapshotData, + TlcTextPage, +} from "@acme/api/lib/texas-election-data"; +import { + parseTexasSosDiscovery, + parseTexasSosElection, + parseTexasTlcAnalysis, + TEXAS_CURRENT_SCOPE, + TEXAS_RESULTS_URL, + TEXAS_SOS_PROVIDER, + TEXAS_TLC_PROVIDER, + TEXAS_TLC_PUBLICATIONS_URL, +} from "@acme/api/lib/texas-election-data"; +import { db } from "@acme/db/client"; +import { ElectionSourceSnapshot } from "@acme/db/schema"; + +import type { Scraper } from "../utils/types.js"; +import { fetchWithRetry } from "../utils/fetch.js"; +import { createLogger } from "../utils/log.js"; +import { texasCurrentElectionConfig } from "./texas-current-election.config.js"; + +const logger = createLogger("texas-current-election"); +const API_BASE = + "https://goelect.txelections.civixapps.com/api-ivis-system/api/s3/enr"; +const USER_AGENT = + "Mozilla/5.0 (compatible; BillionCivicBot/1.0; +https://billion.app)"; + +interface TlcPublication { + year: number; + title: string; + url: string; +} + +function sha256(value: string | Uint8Array): string { + return createHash("sha256").update(value).digest("hex"); +} + +async function fetchJson(url: string): Promise { + const response = await fetchWithRetry(url, { + headers: { Accept: "application/json", "User-Agent": USER_AGENT }, + }); + return response.json() as Promise; +} + +async function fetchText(url: string): Promise { + const response = await fetchWithRetry(url, { + headers: { Accept: "text/html", "User-Agent": USER_AGENT }, + }); + return response.text(); +} + +async function fetchBytes(url: string): Promise { + const response = await fetchWithRetry(url, { + headers: { Accept: "application/pdf", "User-Agent": USER_AGENT }, + timeoutMs: 60_000, + }); + return new Uint8Array(await response.arrayBuffer()); +} + +/** Discover the newest full TLC analysis PDF from the publications index. */ +export function discoverLatestTlcAnalysis( + html: string, + baseUrl = TEXAS_TLC_PUBLICATIONS_URL, +): TlcPublication | null { + const $ = cheerio.load(html); + const candidates: TlcPublication[] = []; + $("a[href]").each((_index, element) => { + const href = $(element).attr("href"); + if (!href || /condensed/i.test(href)) return; + const match = /\/analyses(\d{2}|\d{4})\.pdf(?:$|[?#])/i.exec(href); + if (!match?.[1]) return; + const short = Number.parseInt(match[1], 10); + const year = short < 100 ? 2000 + short : short; + if (!Number.isInteger(year) || year > new Date().getFullYear()) return; + candidates.push({ + year, + title: + $(element).text().replace(/\s+/g, " ").trim() || + `Analyses of Proposed Constitutional Amendments (${year})`, + url: new URL(href, baseUrl).toString(), + }); + }); + return candidates.sort((a, b) => b.year - a.year)[0] ?? null; +} + +/** Extract each PDF page independently so citations retain real page numbers. */ +export async function extractTlcPages( + bytes: Uint8Array, +): Promise { + const pdf = await getDocumentProxy(bytes); + const pages: TlcTextPage[] = []; + for (let pageNumber = 1; pageNumber <= pdf.numPages; pageNumber++) { + const page = await pdf.getPage(pageNumber); + const content = await page.getTextContent(); + const text = (content.items as Array<{ str?: string }>) + .map((item) => item.str?.trim() ?? "") + .filter(Boolean) + .join(" "); + pages.push({ page: pageNumber, text }); + } + return pages; +} + +async function loadSosElection( + definition: TexasElectionDefinition, +): Promise { + try { + const [election, counties] = await Promise.all([ + fetchJson(`${API_BASE}/election/${definition.id}`), + fetchJson(`${API_BASE}/election/countyInfo/${definition.id}`).catch( + () => undefined, + ), + ]); + return parseTexasSosElection(election, definition, counties); + } catch (error) { + logger.warn(`SOS election ${definition.id} could not be parsed:`, error); + return null; + } +} + +async function upsertSnapshot(input: { + cycleYear: number; + provider: string; + sourceVersion: string; + data: Record; + diagnostics: string[]; + sourceUrls: string[]; +}): Promise { + const contentHash = sha256(JSON.stringify(input.data)); + const now = new Date(); + await db + .insert(ElectionSourceSnapshot) + .values({ + jurisdiction: "TX", + cycleYear: input.cycleYear, + provider: input.provider, + scope: TEXAS_CURRENT_SCOPE, + sourceVersion: input.sourceVersion, + contentHash, + data: input.data, + diagnostics: input.diagnostics, + sourceUrls: input.sourceUrls, + fetchedAt: now, + }) + .onConflictDoUpdate({ + target: [ + ElectionSourceSnapshot.jurisdiction, + ElectionSourceSnapshot.cycleYear, + ElectionSourceSnapshot.provider, + ElectionSourceSnapshot.scope, + ], + set: { + sourceVersion: input.sourceVersion, + contentHash, + data: input.data, + diagnostics: input.diagnostics, + sourceUrls: input.sourceUrls, + fetchedAt: now, + }, + }); +} + +function selectedDefinitions( + definitions: TexasElectionDefinition[], + amendmentOnly: boolean, + remaining: number, +): TexasElectionDefinition[] { + const filtered = amendmentOnly + ? definitions.filter((definition) => + /constitutional amendment/i.test(definition.name), + ) + : definitions; + return filtered.slice(0, Math.max(0, remaining)); +} + +async function scrape(maxItems = 12): Promise { + logger.info("Discovering current Texas SOS elections and TLC analysis…"); + const [constantsRaw, publicationsHtml] = await Promise.all([ + fetchJson(`${API_BASE}/electionConstants`), + fetchText(TEXAS_TLC_PUBLICATIONS_URL), + ]); + const discovery = parseTexasSosDiscovery(constantsRaw); + const tlcPublication = discoverLatestTlcAnalysis(publicationsHtml); + const diagnostics: string[] = []; + if (!tlcPublication) + diagnostics.push("No current TLC amendment analysis PDF discovered"); + + let processed = 0; + const years = new Set([discovery.cycleYear]); + if (tlcPublication) years.add(tlcPublication.year); + for (const year of [...years].sort((a, b) => b - a)) { + const definitions = selectedDefinitions( + discovery.electionsByYear[year] ?? [], + year !== discovery.cycleYear, + maxItems - processed, + ); + const elections: TexasSosElection[] = []; + for (const definition of definitions) { + const election = await loadSosElection(definition); + processed++; + if (election) elections.push(election); + } + if (!elections.length) { + diagnostics.push(`No SOS election payloads parsed for ${year}`); + continue; + } + const data: TexasSosSnapshotData = { cycleYear: year, elections }; + await upsertSnapshot({ + cycleYear: year, + provider: TEXAS_SOS_PROVIDER, + sourceVersion: elections + .map((election) => election.sourceVersion) + .join(","), + data: data as unknown as Record, + diagnostics: diagnostics.filter((item) => item.includes(String(year))), + sourceUrls: [TEXAS_RESULTS_URL], + }); + logger.success( + `Texas SOS ${year}: persisted ${elections.length} election(s).`, + ); + } + + if (tlcPublication && processed < maxItems) { + try { + const bytes = await fetchBytes(tlcPublication.url); + const pages = await extractTlcPages(bytes); + const parsed: TexasTlcSnapshotData = parseTexasTlcAnalysis( + pages, + tlcPublication.url, + ); + parsed.publicationTitle = tlcPublication.title; + const tlcDiagnostics = [ + ...diagnostics.filter((item) => item.includes("TLC")), + ...parsed.measures.flatMap((measure) => + measure.diagnostics.map( + (item) => `Proposition ${measure.propositionNumber}: ${item}`, + ), + ), + ]; + if (!parsed.measures.length) { + tlcDiagnostics.push("TLC PDF contained no parseable propositions"); + } + await upsertSnapshot({ + cycleYear: parsed.cycleYear, + provider: TEXAS_TLC_PROVIDER, + sourceVersion: `tlc:${parsed.cycleYear}:${sha256(bytes).slice(0, 16)}`, + data: parsed as unknown as Record, + diagnostics: tlcDiagnostics, + sourceUrls: [TEXAS_TLC_PUBLICATIONS_URL, tlcPublication.url], + }); + logger.success( + `Texas TLC ${parsed.cycleYear}: persisted ${parsed.measures.length} proposition analyses.`, + ); + } catch (error) { + logger.warn( + "TLC analysis could not be parsed; SOS data remains available:", + error, + ); + } + } +} + +export const texasCurrentElection: Scraper = { + ...texasCurrentElectionConfig, + scrape: (options) => + scrape((options?.maxItems ?? Number(process.env.TX_SOS_MAX_ITEMS)) || 12), +}; diff --git a/apps/scraper/src/scrapers/texas-legislature-parser.ts b/apps/scraper/src/scrapers/texas-legislature-parser.ts new file mode 100644 index 00000000..b7f63569 --- /dev/null +++ b/apps/scraper/src/scrapers/texas-legislature-parser.ts @@ -0,0 +1,344 @@ +import { load } from "cheerio"; + +export const TEXAS_JURISDICTION = + "ocd-jurisdiction/country:us/state:tx/government"; + +export interface TexasDocument { + type: "bill_text" | "analysis" | "fiscal_note"; + description: string; + htmlUrl?: string; + pdfUrl?: string; + ftpHtmlUrl?: string; + ftpPdfUrl?: string; + text?: string; +} + +export interface TexasVote { + identifier: string; + date?: string; + chamber?: "House" | "Senate"; + motion?: string; + result?: string; + sourceUrl?: string; + counts: { option: string; value: number }[]; + votes: { option: string; voterName: string; openStatesId?: string }[]; +} + +export interface ParsedTexasBill { + billNumber: string; + title: string; + sponsor?: string; + status?: string; + introducedDate?: Date; + chamber: "House" | "Senate"; + jurisdiction: typeof TEXAS_JURISDICTION; + legislativeSession: string; + subjects: string[]; + sponsorships: { + name: string; + classification: "primary" | "cosponsor"; + chamber: "House" | "Senate"; + }[]; + documents: TexasDocument[]; + votes: TexasVote[]; + actions: { date: string; text: string; type?: string }[]; + url: string; +} + +function normalizeSpace(value: string | undefined): string | undefined { + const normalized = value?.replace(/\s+/g, " ").trim(); + return normalized || undefined; +} + +function parseTexasDate(value: string | undefined): string | undefined { + const text = normalizeSpace(value); + if (!text) return undefined; + const match = /^(\d{1,2})\/(\d{1,2})\/(\d{4})$/.exec(text); + if (match) { + const [, month, day, year] = match; + return `${year}-${month!.padStart(2, "0")}-${day!.padStart(2, "0")}`; + } + const parsed = new Date(text); + return Number.isNaN(parsed.valueOf()) + ? undefined + : parsed.toISOString().slice(0, 10); +} + +export function normalizeTexasBillNumber(value: string): string { + const match = /(HCR|HJR|HR|HB|SCR|SJR|SR|SB)\s*0*(\d+)/i.exec(value); + if (!match) throw new Error(`Unrecognized Texas bill identifier: ${value}`); + return `${match[1]!.toUpperCase()} ${Number(match[2])}`; +} + +function chamberFor(identifier: string): "House" | "Senate" { + return identifier.startsWith("H") ? "House" : "Senate"; +} + +function splitNames(value: string | undefined): string[] { + return (value ?? "") + .split("|") + .map((name) => normalizeSpace(name)) + .filter((name): name is string => Boolean(name)); +} + +function firstText( + $: ReturnType, + selectors: readonly string[], +): string | undefined { + for (const selector of selectors) { + const value = normalizeSpace($(selector).first().text()); + if (value) return value; + } + return undefined; +} + +function parseDocuments($: ReturnType): TexasDocument[] { + const documents: TexasDocument[] = []; + const groups = [ + ["billtext > docTypes > bill > versions > version", "bill_text"], + ["billtext > docTypes > analysis > versions > version", "analysis"], + ["billtext > docTypes > fiscalNote > versions > version", "fiscal_note"], + ] as const; + + for (const [selector, type] of groups) { + $(selector).each((_, element) => { + const version = $(element); + const description = + normalizeSpace(version.find("versionDescription").first().text()) ?? + type.replace("_", " "); + const htmlUrl = normalizeSpace(version.find("WebHTMLURL").first().text()); + const pdfUrl = normalizeSpace(version.find("WebPDFURL").first().text()); + const ftpHtmlUrl = normalizeSpace( + version.find("FTPHTMLURL").first().text(), + ); + const ftpPdfUrl = normalizeSpace( + version.find("FTPPDFURL").first().text(), + ); + if (htmlUrl || pdfUrl || ftpHtmlUrl || ftpPdfUrl) { + documents.push({ + type, + description, + ...(htmlUrl && { htmlUrl }), + ...(pdfUrl && { pdfUrl }), + ...(ftpHtmlUrl && { ftpHtmlUrl }), + ...(ftpPdfUrl && { ftpPdfUrl }), + }); + } + }); + } + return documents; +} + +function parseVotes($: ReturnType): TexasVote[] { + const votes: TexasVote[] = []; + $("votes > vote, recordVotes > vote, voteHistory > vote").each( + (_, element) => { + const vote = $(element); + const text = (selectors: string[]) => { + for (const selector of selectors) { + const value = normalizeSpace(vote.find(selector).first().text()); + if (value) return value; + } + return undefined; + }; + const identifier = text([ + "identifier", + "voteNumber", + "rollCallId", + "actionNumber", + ]); + if (!identifier) return; + const chamberText = text(["chamber", "organization"]); + const chamber = chamberText + ? chamberText.toLowerCase().startsWith("h") + ? "House" + : chamberText.toLowerCase().startsWith("s") + ? "Senate" + : undefined + : undefined; + const counts: TexasVote["counts"] = []; + vote.find("counts > count, totals > total").each((__, countElement) => { + const count = $(countElement); + const option = normalizeSpace( + count.attr("option") ?? count.find("option").first().text(), + ); + const value = Number( + normalizeSpace( + count.attr("value") ?? count.find("value").first().text(), + ), + ); + if (option && Number.isInteger(value)) counts.push({ option, value }); + }); + const memberVotes: TexasVote["votes"] = []; + vote + .find("voters > voter, memberVotes > vote") + .each((__, voterElement) => { + const voter = $(voterElement); + const voterName = normalizeSpace( + voter.attr("name") ?? + voter.find("name, voterName, memberName").first().text(), + ); + const option = normalizeSpace( + voter.attr("option") ?? voter.find("option, vote").first().text(), + ); + if (voterName && option) memberVotes.push({ voterName, option }); + }); + votes.push({ + identifier, + ...(parseTexasDate(text(["date", "voteDate"])) && { + date: parseTexasDate(text(["date", "voteDate"])), + }), + ...(chamber && { chamber }), + ...(text(["motion", "motionText", "description"]) && { + motion: text(["motion", "motionText", "description"]), + }), + ...(text(["result", "outcome"]) && { + result: text(["result", "outcome"]), + }), + ...(text(["sourceUrl", "url"]) && { + sourceUrl: text(["sourceUrl", "url"]), + }), + counts, + votes: memberVotes, + }); + }, + ); + $("actions > action").each((_, element) => { + const action = $(element); + if ( + normalizeSpace(action.find("description").first().text()) !== + "Record vote" + ) { + return; + } + const identifier = normalizeSpace(action.find("comment").first().text()); + if (!identifier) return; + const actionNumber = normalizeSpace( + action.find("actionNumber").first().text(), + ); + votes.push({ + identifier, + ...(parseTexasDate(action.find("date").first().text()) && { + date: parseTexasDate(action.find("date").first().text()), + }), + ...(actionNumber?.startsWith("H") && { chamber: "House" }), + ...(actionNumber?.startsWith("S") && { chamber: "Senate" }), + motion: "Record vote", + counts: [], + votes: [], + }); + }); + $("committees > house, committees > senate").each((_, element) => { + const committee = $(element); + const name = normalizeSpace(committee.attr("name")); + if (!name) return; + const chamber = + element.tagName.toLowerCase() === "house" ? "House" : "Senate"; + const countAttributes = [ + ["Yea", "ayeVotes"], + ["Nay", "nayVotes"], + ["Present not voting", "presentNotVotingVotes"], + ["Absent", "absentVotes"], + ] as const; + const counts = countAttributes.flatMap(([option, attribute]) => { + const value = Number(committee.attr(attribute)); + return Number.isInteger(value) ? [{ option, value }] : []; + }); + if (counts.length === 0) return; + votes.push({ + identifier: `committee:${chamber.toLowerCase()}:${name}`, + chamber, + motion: `${name} committee vote`, + result: normalizeSpace(committee.attr("status")), + counts, + votes: [], + }); + }); + return votes; +} + +export function parseTexasBillHistory( + xml: string, + session: string, +): ParsedTexasBill { + const $ = load(xml, { xml: true }); + const root = $.root().children().first(); + const rawIdentifier = root.attr("bill") ?? firstText($, ["billNumber"]); + if (!rawIdentifier) + throw new Error("Texas bill history is missing bill identity"); + const billNumber = normalizeTexasBillNumber(rawIdentifier); + const title = firstText($, ["caption"]); + if (!title || title.includes("Bill does not exist")) { + throw new Error(`${billNumber} does not contain a usable caption`); + } + const chamber = chamberFor(billNumber); + const otherChamber = chamber === "House" ? "Senate" : "House"; + const sponsorships: ParsedTexasBill["sponsorships"] = []; + for (const [selector, classification, sponsorChamber] of [ + ["authors", "primary", chamber], + ["coauthors", "cosponsor", chamber], + ["sponsors", "primary", otherChamber], + ["cosponsors", "cosponsor", otherChamber], + ] as const) { + for (const name of splitNames($(selector).first().text())) { + sponsorships.push({ name, classification, chamber: sponsorChamber }); + } + } + + const actions: ParsedTexasBill["actions"] = []; + $("actions > action").each((_, element) => { + const action = $(element); + const date = parseTexasDate(action.find("date").first().text()); + const text = normalizeSpace(action.find("description").first().text()); + if (!date || !text) return; + const actionNumber = normalizeSpace( + action.find("actionNumber").first().text(), + ); + actions.push({ date, text, ...(actionNumber && { type: actionNumber }) }); + }); + actions.sort((left, right) => left.date.localeCompare(right.date)); + + const sponsor = sponsorships + .filter((item) => item.classification === "primary") + .map((item) => item.name) + .join(" | ") + .slice(0, 256); + const billSlug = billNumber.replace(/\s+/g, ""); + return { + billNumber, + title, + ...(sponsor && { sponsor }), + ...(actions.at(-1)?.text && { status: actions.at(-1)!.text.slice(0, 100) }), + ...(actions[0]?.date && { + introducedDate: new Date(`${actions[0].date}T12:00:00.000Z`), + }), + chamber, + jurisdiction: TEXAS_JURISDICTION, + legislativeSession: session.toUpperCase(), + subjects: $("subjects > subject") + .map((_, element) => normalizeSpace($(element).text())) + .get() + .filter((value): value is string => Boolean(value)), + sponsorships, + documents: parseDocuments($), + votes: parseVotes($), + actions, + url: `https://capitol.texas.gov/BillLookup/History.aspx?LegSess=${encodeURIComponent(session.toUpperCase())}&Bill=${billSlug}`, + }; +} + +export function htmlToText(html: string): string { + const $ = load(html); + $("script, style, nav, header, footer").remove(); + $("br, p, div, h1, h2, h3, h4, h5, h6, li, tr").each((_, element) => { + $(element).append(" "); + }); + return $.root().text().replace(/\s+/g, " ").trim(); +} + +export function openStatesSessionName(session: string): string { + const normalized = session.toUpperCase(); + if (/^\d{2}R$/.test(normalized)) return normalized.slice(0, 2); + if (/^\d{3}$/.test(normalized)) return normalized; + throw new Error(`Invalid Texas legislative session: ${session}`); +} diff --git a/apps/scraper/src/scrapers/texas-legislature-source.ts b/apps/scraper/src/scrapers/texas-legislature-source.ts new file mode 100644 index 00000000..4eba3d35 --- /dev/null +++ b/apps/scraper/src/scrapers/texas-legislature-source.ts @@ -0,0 +1,134 @@ +import { Writable } from "node:stream"; + +import { Client } from "basic-ftp"; + +import type { TexasDocument } from "./texas-legislature-parser.js"; + +export interface BulkEntry { + name: string; + isDirectory: boolean; +} + +export interface TexasBulkClient { + list(path: string): Promise; + download(path: string): Promise; + close(): void; +} + +export class TexasFtpClient implements TexasBulkClient { + private readonly client = new Client(30_000); + + async connect(): Promise { + await this.client.access({ + host: "ftp.legis.state.tx.us", + user: "anonymous", + password: "billion@example.invalid", + secure: false, + }); + } + + async list(path: string): Promise { + return (await this.client.list(path)).map((entry) => ({ + name: entry.name, + isDirectory: entry.isDirectory, + })); + } + + async download(path: string): Promise { + const chunks: Buffer[] = []; + const sink = new Writable({ + write(chunk: Buffer | string, _encoding, callback) { + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + callback(); + }, + }); + await this.client.downloadTo(sink, path); + return Buffer.concat(chunks); + } + + close(): void { + this.client.close(); + } +} + +export function selectCurrentTexasSession(names: readonly string[]): string { + const sessions = names + .map((name) => name.toUpperCase()) + .filter((name) => /^\d{2}(?:R|\d)$/.test(name)) + .sort((left, right) => { + const legislature = Number(left.slice(0, 2)) - Number(right.slice(0, 2)); + if (legislature !== 0) return legislature; + const rank = (value: string) => + value[2] === "R" ? 0 : Number(value[2]) || 0; + return rank(left) - rank(right); + }); + const session = sessions.at(-1); + if (!session) throw new Error("No Texas legislative sessions found in /bills"); + return session; +} + +export async function listFilesRecursively( + client: TexasBulkClient, + root: string, +): Promise { + const entries = (await client.list(root)).sort((a, b) => + a.name.localeCompare(b.name), + ); + const files: string[] = []; + for (const entry of entries) { + if (entry.name === "." || entry.name === "..") continue; + const child = `${root.replace(/\/$/, "")}/${entry.name}`; + if (entry.isDirectory) files.push(...(await listFilesRecursively(client, child))); + else files.push(child); + } + return files; +} + +const folderByPrefix: Record = { + HB: "house_bills", + HCR: "house_concurrent_resolutions", + HJR: "house_joint_resolutions", + HR: "house_resolutions", + SB: "senate_bills", + SCR: "senate_concurrent_resolutions", + SJR: "senate_joint_resolutions", + SR: "senate_resolutions", +}; + +const rootByType: Record = { + bill_text: "billtext", + analysis: "analysis", + fiscal_note: "fiscalnotes", +}; + +export function bulkHtmlPath( + session: string, + document: Pick, +): string | undefined { + if (document.ftpHtmlUrl) { + try { + return decodeURIComponent(new URL(document.ftpHtmlUrl).pathname); + } catch { + return undefined; + } + } + if (!document.htmlUrl) return undefined; + let filename: string; + try { + filename = new URL(document.htmlUrl).pathname.split("/").at(-1) ?? ""; + } catch { + return undefined; + } + const match = /^(HCR|HJR|HR|HB|SCR|SJR|SR|SB)(\d{5})[A-Z0-9]*\.html?$/i.exec( + filename, + ); + if (!match) return undefined; + const prefix = match[1]!.toUpperCase(); + const number = Number(match[2]); + const start = number < 100 ? 1 : Math.floor(number / 100) * 100; + const end = number < 100 ? 99 : start + 99; + const pad = (value: number) => String(value).padStart(5, "0"); + const documentRoot = + document.type === "fiscal_note" ? "fiscalNotes" : rootByType[document.type]; + return `/bills/${session.toUpperCase()}/${documentRoot}/HTML/${folderByPrefix[prefix]}/${prefix}${pad(start)}_${prefix}${pad(end)}/${filename}`; +} diff --git a/apps/scraper/src/scrapers/texas-legislature.config.ts b/apps/scraper/src/scrapers/texas-legislature.config.ts new file mode 100644 index 00000000..8e4be629 --- /dev/null +++ b/apps/scraper/src/scrapers/texas-legislature.config.ts @@ -0,0 +1,16 @@ +import type { ScraperEnvContract } from "@acme/env"; + +export const texasLegislatureConfig = { + id: "texas-legislature", + name: "Texas Legislature Online", + source: + "Texas Legislative Council anonymous FTP — current-session history XML and bulk documents", + environment: { + required: ["POSTGRES_URL"], + optional: [ + "OPEN_STATES_API_KEY", + "TEXAS_LEGISLATURE_SESSION", + "TEXAS_LEGISLATURE_MAX_ITEMS", + ], + }, +} as const satisfies ScraperEnvContract; diff --git a/apps/scraper/src/scrapers/texas-legislature.test.ts b/apps/scraper/src/scrapers/texas-legislature.test.ts new file mode 100644 index 00000000..e51d886c --- /dev/null +++ b/apps/scraper/src/scrapers/texas-legislature.test.ts @@ -0,0 +1,105 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import test from "node:test"; + +import { CreateBillSchema } from "@acme/db/schema"; + +import { + htmlToText, + openStatesSessionName, + parseTexasBillHistory, +} from "./texas-legislature-parser.js"; +import { + bulkHtmlPath, + listFilesRecursively, + selectCurrentTexasSession, + type TexasBulkClient, +} from "./texas-legislature-source.js"; + +const fixture = await readFile( + new URL("./fixtures/texas-hb9-history.xml", import.meta.url), + "utf8", +); + +void test("parses official-style Texas bill history deterministically", () => { + const bill = parseTexasBillHistory(fixture, "89R"); + assert.equal(bill.billNumber, "HB 9"); + assert.equal(bill.chamber, "House"); + assert.equal(bill.legislativeSession, "89R"); + assert.equal(bill.sponsor, "Meyer | Bonnen | Bettencourt"); + assert.equal(bill.status, "Effective immediately"); + assert.equal(bill.introducedDate?.toISOString(), "2024-11-12T12:00:00.000Z"); + assert.deepEqual( + bill.documents.map((document) => document.type), + ["bill_text", "bill_text", "analysis", "fiscal_note"], + ); + assert.deepEqual(bill.votes[0], { + identifier: "RV#2999", + date: "2025-05-19", + chamber: "House", + motion: "Record vote", + counts: [], + votes: [], + }); + assert.deepEqual(bill.votes[1]?.counts, [ + { option: "Yea", value: 9 }, + { option: "Nay", value: 1 }, + { option: "Present not voting", value: 0 }, + { option: "Absent", value: 1 }, + ]); +}); + +void test("maps public document names to the official FTP bulk tree", () => { + const bill = parseTexasBillHistory(fixture, "89R"); + assert.equal( + bulkHtmlPath("89R", bill.documents[0]!), + "/bills/89R/billtext/HTML/house_bills/HB00001_HB00099/HB00009I.HTM", + ); + assert.equal( + bulkHtmlPath("89R", bill.documents.at(-1)!), + "/bills/89R/fiscalNotes/HTML/house_bills/HB00001_HB00099/HB00009I.HTM", + ); +}); + +void test("selects only the latest Texas session and maps Open States names", () => { + assert.equal(selectCurrentTexasSession(["88R", "89R", "891", "892"]), "892"); + assert.equal(openStatesSessionName("89R"), "89"); + assert.equal(openStatesSessionName("892"), "892"); +}); + +void test("walks bulk directories in stable lexical order", async () => { + const listings: Record = { + "/root": [ + { name: "z.xml", isDirectory: false }, + { name: "a", isDirectory: true }, + ], + "/root/a": [{ name: "b.xml", isDirectory: false }], + }; + const client: TexasBulkClient = { + list: async (path) => listings[path] ?? [], + download: async () => Buffer.alloc(0), + close: () => undefined, + }; + assert.deepEqual(await listFilesRecursively(client, "/root"), [ + "/root/a/b.xml", + "/root/z.xml", + ]); +}); + +void test("extracts readable deterministic text from bulk HTML", () => { + assert.equal( + htmlToText("

Bill text

Section 1.

"), + "Bill text Section 1.", + ); +}); + +void test("parsed Texas data satisfies the shared persisted bill contract", () => { + const parsed = parseTexasBillHistory(fixture, "89R"); + assert.equal( + CreateBillSchema.safeParse({ + ...parsed, + sourceWebsite: "capitol.texas.gov", + }).success, + true, + ); +}); diff --git a/apps/scraper/src/scrapers/texas-legislature.ts b/apps/scraper/src/scrapers/texas-legislature.ts new file mode 100644 index 00000000..92a26c58 --- /dev/null +++ b/apps/scraper/src/scrapers/texas-legislature.ts @@ -0,0 +1,164 @@ +import type { BillData, Scraper } from "../utils/types.js"; +import type { TexasBulkClient } from "./texas-legislature-source.js"; +import { setExpectedTotal } from "../utils/db/metrics.js"; +import { upsertContent } from "../utils/db/operations.js"; +import { createLogger } from "../utils/log.js"; +import { + htmlToText, + openStatesSessionName, + parseTexasBillHistory, + TEXAS_JURISDICTION, +} from "./texas-legislature-parser.js"; +import { + bulkHtmlPath, + listFilesRecursively, + selectCurrentTexasSession, + TexasFtpClient, +} from "./texas-legislature-source.js"; +import { texasLegislatureConfig } from "./texas-legislature.config.js"; + +const logger = createLogger("texas-legislature"); + +interface OpenStatesSearchResponse { + results?: { id: string; identifier: string; session: string }[]; +} + +export async function matchOpenStatesBillId( + session: string, + billNumber: string, + apiKey = process.env.OPEN_STATES_API_KEY, +): Promise { + if (!apiKey) return undefined; + const url = new URL("https://v3.openstates.org/bills"); + url.searchParams.set("jurisdiction", TEXAS_JURISDICTION); + url.searchParams.set("session", openStatesSessionName(session)); + url.searchParams.set("q", billNumber); + url.searchParams.set("per_page", "5"); + const response = await fetch(url, { + headers: { Accept: "application/json", "X-API-KEY": apiKey }, + }); + if (!response.ok) return undefined; + const payload = (await response.json()) as OpenStatesSearchResponse; + return payload.results?.find( + (bill) => + bill.identifier.replace(/\s+/g, "").toUpperCase() === + billNumber.replace(/\s+/g, "").toUpperCase() && + bill.session === openStatesSessionName(session), + )?.id; +} + +async function enrichDocuments( + client: TexasBulkClient, + session: string, + documents: ReturnType["documents"], +) { + return Promise.all( + documents.map(async (document) => { + const path = bulkHtmlPath(session, document); + if (!path) return document; + try { + const html = (await client.download(path)).toString("utf8"); + const text = htmlToText(html); + return text ? { ...document, text } : document; + } catch (error) { + logger.debug( + `Optional bulk HTML unavailable at ${path}: ${error instanceof Error ? error.message : error}`, + ); + return document; + } + }), + ); +} + +export async function scrapeTexasLegislature( + client: TexasBulkClient, + options: { maxItems: number; session?: string }, +): Promise { + const availableSessions = (await client.list("/bills")) + .filter((entry) => entry.isDirectory) + .map((entry) => entry.name); + const currentSession = selectCurrentTexasSession(availableSessions); + const session = options.session?.toUpperCase() ?? currentSession; + if (session !== currentSession) { + throw new Error( + `Texas ingestion is current-session only: requested ${session}, latest bulk session is ${currentSession}`, + ); + } + + logger.info(`Reading official Texas bulk data for ${session}...`); + const paths = ( + await listFilesRecursively(client, `/bills/${session}/billhistory`) + ) + .filter( + (path) => + path.toLowerCase().endsWith(".xml") && + !/\/history(?:_periodic)?\.xml$/i.test(path), + ) + .sort() + .slice(0, options.maxItems); + setExpectedTotal(paths.length); + let persisted = 0; + + for (const path of paths) { + try { + const parsed = parseTexasBillHistory( + (await client.download(path)).toString("utf8"), + session, + ); + const documents = await enrichDocuments( + client, + session, + parsed.documents, + ); + const latestBillText = documents + .filter((document) => document.type === "bill_text" && document.text) + .at(-1)?.text; + let openStatesId: string | undefined; + try { + openStatesId = await matchOpenStatesBillId(session, parsed.billNumber); + } catch (error) { + logger.debug( + `Open States identity match skipped for ${parsed.billNumber}: ${error instanceof Error ? error.message : error}`, + ); + } + const data: BillData = { + ...parsed, + documents, + ...(latestBillText && { fullText: latestBillText }), + ...(openStatesId && { openStatesId }), + sourceWebsite: "capitol.texas.gov", + }; + await upsertContent({ type: "bill", data }, { skipEnrichment: true }); + persisted += 1; + } catch (error) { + logger.error( + `Failed to process ${path}: ${error instanceof Error ? error.message : error}`, + ); + } + } + logger.success( + `Persisted ${persisted}/${paths.length} Texas bills from ${session}.`, + ); +} + +async function scrape(options?: { maxItems?: number }): Promise { + const client = new TexasFtpClient(); + try { + await client.connect(); + await scrapeTexasLegislature(client, { + maxItems: + options?.maxItems ?? + (Number(process.env.TEXAS_LEGISLATURE_MAX_ITEMS) || 100), + ...(process.env.TEXAS_LEGISLATURE_SESSION && { + session: process.env.TEXAS_LEGISLATURE_SESSION, + }), + }); + } finally { + client.close(); + } +} + +export const texasLegislature: Scraper = { + ...texasLegislatureConfig, + scrape, +}; diff --git a/apps/scraper/src/utils/db/helpers.ts b/apps/scraper/src/utils/db/helpers.ts index 96280057..ac261c54 100644 --- a/apps/scraper/src/utils/db/helpers.ts +++ b/apps/scraper/src/utils/db/helpers.ts @@ -20,6 +20,7 @@ const logger = createLogger("db"); export async function checkExistingBill( billNumber: string, sourceWebsite: string, + legislativeSession = "", ): Promise { try { const [existing] = await db @@ -30,7 +31,13 @@ export async function checkExistingBill( thumbnailUrl: Bill.thumbnailUrl, }) .from(Bill) - .where(and(eq(Bill.billNumber, billNumber), eq(Bill.sourceWebsite, sourceWebsite))) + .where( + and( + eq(Bill.billNumber, billNumber), + eq(Bill.sourceWebsite, sourceWebsite), + eq(Bill.legislativeSession, legislativeSession), + ), + ) .limit(1); if (!existing) { diff --git a/apps/scraper/src/utils/db/operations.ts b/apps/scraper/src/utils/db/operations.ts index 5e2a54b4..c3d0bc9e 100644 --- a/apps/scraper/src/utils/db/operations.ts +++ b/apps/scraper/src/utils/db/operations.ts @@ -76,6 +76,15 @@ function hashFields(input: ContentData): string { status: input.data.status, summary: input.data.summary, fullText: input.data.fullText, + jurisdiction: input.data.jurisdiction, + legislativeSession: input.data.legislativeSession, + openStatesId: input.data.openStatesId, + subjects: input.data.subjects, + sponsorships: input.data.sponsorships, + documents: input.data.documents, + votes: input.data.votes, + actions: input.data.actions, + versions: input.data.versions, }); case "government_content": return JSON.stringify({ @@ -96,7 +105,11 @@ function hashFields(input: ContentData): string { async function checkExisting(input: ContentData) { switch (input.type) { case "bill": - return checkExistingBill(input.data.billNumber, input.data.sourceWebsite); + return checkExistingBill( + input.data.billNumber, + input.data.sourceWebsite, + input.data.legislativeSession, + ); case "government_content": return checkExistingGovernmentContent(input.data.url); case "court_case": @@ -117,7 +130,7 @@ function getUpdateTable(input: ContentData) { export async function upsertContent( input: ContentData, - options?: { newItemLimiter?: NewItemLimiter }, + options?: { newItemLimiter?: NewItemLimiter; skipEnrichment?: boolean }, ) { const newContentHash = createContentHash(hashFields(input)); const existing = await checkExisting(input); @@ -270,10 +283,10 @@ export async function upsertContent( ...d, description: preGeneratedDescription || description, contentHash: newContentHash, - versions: [], + versions: d.versions ?? [], }) .onConflictDoUpdate({ - target: [Bill.billNumber, Bill.sourceWebsite], + target: [Bill.billNumber, Bill.sourceWebsite, Bill.legislativeSession], set: { title: d.title, description, @@ -282,8 +295,17 @@ export async function upsertContent( introducedDate: d.introducedDate, congress: d.congress, chamber: d.chamber, + jurisdiction: d.jurisdiction, + legislativeSession: d.legislativeSession, + openStatesId: d.openStatesId, + subjects: d.subjects, + sponsorships: d.sponsorships, + documents: d.documents, + votes: d.votes, summary: d.summary, fullText: d.fullText, + actions: d.actions, + ...(d.versions && { versions: d.versions }), url: d.url, contentHash: newContentHash, sourceUpdatedAt: d.sourceUpdatedAt, @@ -354,6 +376,15 @@ export async function upsertContent( return result; } + if (options?.skipEnrichment) { + tickProgress({ + newEntries: progressKind === "new" ? 1 : 0, + unchanged: progressKind === "unchanged" ? 1 : 0, + changed: progressKind === "changed" ? 1 : 0, + }); + return result; + } + // Phase 2: AI enrichment — skipped entirely if rate-limited try { const existingDescription = sourceDescription || persistedDescription; diff --git a/docs/README.md b/docs/README.md index 16378248..d0bf3643 100644 --- a/docs/README.md +++ b/docs/README.md @@ -4,16 +4,17 @@ Start with [CONTRIBUTING.md](../CONTRIBUTING.md) for dev setup. These docs go de ## How the system works -| Doc | What it covers | -| ---------------------------------------------------- | ---------------------------------------------------------------------------------------------------- | -| [Architecture overview](./architecture.md) | The big picture: system diagram, monorepo layout, build tooling, and why the key decisions were made | -| [Data layer](./data-layer.md) | Drizzle + Supabase Postgres, the schema (~20 tables), migrations | -| [API layer](./api.md) | The tRPC router, civic data integrations, request path, LLM provider | -| [Ballot-measure enrichment](./measure-enrichment.md) | How measure summaries are cross-validated across official sources, adapter by adapter | -| [Candidate enrichment](./candidate-enrichment.md) | The same cross-validation pattern applied to candidate bios/photos/contact info | -| [Scraper pipeline](./scraper.md) | The standalone content scraper: sources, change detection, AI generation | -| [Article generation](./article-generation.md) | How a bill becomes a readable brief: the structured schema, quote verification, framing lint | -| [Frontend apps](./frontend.md) | Expo mobile app, Next.js web, shared UI, cross-platform auth | +| Doc | What it covers | +| ---------------------------------------------------------- | ---------------------------------------------------------------------------------------------------- | +| [Architecture overview](./architecture.md) | The big picture: system diagram, monorepo layout, build tooling, and why the key decisions were made | +| [Data layer](./data-layer.md) | Drizzle + Supabase Postgres, the schema (~20 tables), migrations | +| [API layer](./api.md) | The tRPC router, civic data integrations, request path, LLM provider | +| [Ballot-measure enrichment](./measure-enrichment.md) | How measure summaries are cross-validated across official sources, adapter by adapter | +| [Candidate enrichment](./candidate-enrichment.md) | The same cross-validation pattern applied to candidate bios/photos/contact info | +| [Texas current-election data](./texas-current-election.md) | Current-cycle SOS results/candidates and separately cited TLC amendment analyses | +| [Scraper pipeline](./scraper.md) | The standalone content scraper: sources, change detection, AI generation | +| [Article generation](./article-generation.md) | How a bill becomes a readable brief: the structured schema, quote verification, framing lint | +| [Frontend apps](./frontend.md) | Expo mobile app, Next.js web, shared UI, cross-platform auth | ## How to do things diff --git a/docs/api.md b/docs/api.md index 2f6381a2..0025ebb1 100644 --- a/docs/api.md +++ b/docs/api.md @@ -13,17 +13,17 @@ The API is a [tRPC v11](https://trpc.io/) router in `packages/api/`, served by N The root router (`packages/api/src/root.ts`) composes **nine** sub-routers: -| Router | Procedures (Q = query, M = mutation, 🔒 = protected) | -| ------------ | ---------------------------------------------------------------------------------------------------------------------------------------------- | -| `auth` | `getSession` (Q), `getSecretMessage` (Q 🔒) | -| `civic` | `getElections`, `getVoterInfo`, `getRepresentatives`, `getRepresentativesEnriched` (all Q) — Google Civic + measure/candidate cross-validation | -| `places` | `autocomplete` (Q), `details` (M) — Google Places address autocomplete for the ballot lookup | -| `legistar` | `getLocalBills`, `getMeetings`, `getAgenda`, `getVotes`, `getBodies`, `getMeetingVotes` (all Q) — local councils | -| `openStates` | `searchBills`, `getBillDetails`, `getLegislators`, `getBillVotes` (all Q) — CA state legislature (Open States v3) | -| `content` | `getAll`, `getByType`, `getById` (all Q) — aggregates bill / government_content / court_case | -| `video` | `getInfinite` (Q) — cursor-paginated feed; converts `bytea` images to data URIs | -| `post` | `all`, `byId` (Q); `create`, `delete` (M 🔒) | -| `user` | preferences, blocked content, settings, profile, and saved-article CRUD (all 🔒) | +| Router | Procedures (Q = query, M = mutation, 🔒 = protected) | +| ------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `auth` | `getSession` (Q), `getSecretMessage` (Q 🔒) | +| `civic` | `getElections`, `getVoterInfo`, `getTexasCurrentElection`, `getRepresentatives`, `getRepresentativesEnriched` (all Q) — Google Civic + official current-cycle Texas data + enrichment | +| `places` | `autocomplete` (Q), `details` (M) — Google Places address autocomplete for the ballot lookup | +| `legistar` | `getLocalBills`, `getMeetings`, `getAgenda`, `getVotes`, `getBodies`, `getMeetingVotes` (all Q) — local councils | +| `openStates` | `searchBills`, `getBillDetails`, `getLegislators`, `getBillVotes` (all Q) — CA state legislature (Open States v3) | +| `content` | `getAll`, `getByType`, `getById` (all Q) — aggregates bill / government_content / court_case | +| `video` | `getInfinite` (Q) — cursor-paginated feed; converts `bytea` images to data URIs | +| `post` | `all`, `byId` (Q); `create`, `delete` (M 🔒) | +| `user` | preferences, blocked content, settings, profile, and saved-article CRUD (all 🔒) | ## Civic Data & External Sources @@ -32,11 +32,19 @@ The `civic` router calls the **Google Civic Information API** (`GOOGLE_CIVIC_API Other live civic integrations: - **`legistar`** — scrapes Legistar instances for San Jose, Santa Clara County, and Sunnyvale (no key required). +- **`localGovernment`** — reads persisted provider-neutral meetings. `listMeetings` returns bounded meeting summaries and official links; `getMeeting` returns agenda items, actions, attachments, and named votes without live source calls. - **`openStates`** — California bills, legislators, and votes via the Open States v3 API (`OPEN_STATES_API_KEY`). - **`places`** — Google **Places Autocomplete (New)** for the ballot address entry (`packages/api/src/lib/places.ts`). `autocomplete` returns US street-address predictions (biased `includedRegionCodes: ["us"]`, `includedPrimaryTypes: street_address/premise/subpremise`) for queries ≥3 chars; `details` resolves a `placeId` to its full `formattedAddress` (the ZIP the prediction omits, which Civic wants). A **session token** (UUID stable across one address entry) bundles all keystroke calls plus the closing `details` into a single billed unit. Reuses `GOOGLE_PLACES_API_KEY` → `GOOGLE_API_KEY` → `GOOGLE_CIVIC_API_KEY`; with no key it serves a small mock list so the dropdown still works in dev (same fallback pattern as `civic`). Ballot measures and candidates returned by `getVoterInfo` are run through cross-validation engines that merge multiple public-record sources by trust tier — see [Ballot-measure enrichment](./measure-enrichment.md) and [Candidate enrichment](./candidate-enrichment.md). Key/access setup for every source is in [Civic data source setup](./civic-data-sources.md). +`civic.getTexasCurrentElection` is an input-free reader over persisted current +Texas SOS/TLC snapshots. It returns current statewide/federal/district +candidates and results plus the latest constitutional-amendment analyses. SOS +facts and TLC explanation are separately cited. Historical election browsing is +intentionally not part of this procedure; see +[Texas current-election data](./texas-current-election.md). + ## LLM Provider `packages/api/src/lib/ai-provider.ts` exports a single swappable `llm` via the Vercel AI SDK: **Groq** when `GROQ_API_KEY` is set, then **OpenRouter** when `OPENROUTER_API_KEY` is present (using `OPENROUTER_MODEL`, default `deepseek/deepseek-v4-flash`), then **OpenAI `gpt-4o-mini`**, then deprecated direct **DeepSeek `deepseek-v4-flash`** during migration, and `null` otherwise. Callers treat `null` as "AI unavailable" and skip generation rather than throw. diff --git a/docs/civic-data-sources.md b/docs/civic-data-sources.md index ba126200..82c9400a 100644 --- a/docs/civic-data-sources.md +++ b/docs/civic-data-sources.md @@ -9,6 +9,7 @@ How to obtain keys/access for every civic integration. For local dev, copy `.env | Google Places (address autocomplete) | Yes | Pay-as-you-go | `GOOGLE_PLACES_API_KEY` (→ `GOOGLE_API_KEY` → `GOOGLE_CIVIC_API_KEY`) | | Vote Smart | Yes | Free (org tier) | `VOTE_SMART_API_KEY` | | Legistar (local councils) | No | Free | — | +| Cedar Park council records | No | Free | — | | VOTE411 / LWV (scraper) | No | Free | — | | CA SOS Voter Guide (scraper) | No | Free | — | | Santa Clara measure pipeline (scraper) | No | Free | — | @@ -27,6 +28,15 @@ How to obtain keys/access for every civic integration. For local dev, copy `.env **Legistar Web API** — local council meetings, legislation, votes, agendas; no key, unlimited public data. Supported jurisdictions (client id): San Jose (`sanjose`), Santa Clara County (`santaclara`), Sunnyvale (`sunnyvale`). Add a city by extracting its `*.legistar.com` subdomain into the `JURISDICTIONS` constant in `packages/api/src/integrations/legistar.ts`. Base: (OData-compatible). +**Cedar Park City Council** — the official CivicEngage City Council page embeds +a Municode Meetings publish page. The `cedar-park-council` scraper follows that +embed for the latest 12 months, uses deterministic HTML/PDF parsing, and writes +the provider-neutral local-government tables read by the `localGovernment` +tRPC router. No AI key is needed. Configuration lives in +`apps/scraper/src/scrapers/civicengage.config.ts`; a second jurisdiction using +the same embed supplies a new CivicEngage host/path, timezone, Municode +`cid`/`ppid`, and body matcher. + **VOTE411 / League of Women Voters** — nonpartisan voter guides, candidate questionnaires, measure explanations; no key (scraper, rate-limited + cached). ⚠️ ToS bars commercial use without **written consent** and prohibits automated queries — a negotiated partnership is the only compliant path (tracked in the Outreach Tracker). Also `cavotes.org/easy-voter-guide/` for CA. **CA SOS Official Voter Guide (scraper)** — official statewide proposition summaries (AG), pro/con arguments, full-text links; no key. Source `voterguide.sos.ca.gov`. Distinct from the (dead) results API — this carries the human-written official measure content and feeds the cross-validation engine (`measure-sources/ca-sos-voterguide.ts`). CA statewide only; local measures use the county pipelines. diff --git a/docs/data-sources-api.md b/docs/data-sources-api.md index fd61ad4e..2bb929b7 100644 --- a/docs/data-sources-api.md +++ b/docs/data-sources-api.md @@ -8,16 +8,16 @@ All live-API functions come from `@acme/api`. Scrapers run as CLI jobs in `apps/ ## Quick index -| Category | What you get | Import | -|---|---|---| -| [Ballot & elections](#ballot--elections) | Contests, candidates, polling places, ballot measures | `@acme/api` | -| [Ballot-measure enrichment](#ballot-measure-enrichment) | Summaries, fiscal impact, pro/con arguments, citations | `@acme/api` | -| [State legislation](#state-legislation) | CA bills, legislators, votes | `@acme/api` | -| [Local government](#local-government) | City/county meetings, ordinances, votes | `@acme/api` | -| [CA election results](#ca-election-results) | Statewide vote counts (live JSON feed) | `@acme/api` | -| [Address autocomplete](#address-autocomplete) | Street-address suggestions + place details | `@acme/api` | -| [Corpus content (DB)](#corpus-content-db) | Bills, court cases, government content articles | tRPC `content` router | -| [Scrapers (background jobs)](#scrapers-background-jobs) | Congress bills, SCOTUS, Federal Register, LAO, CA SOS | `apps/scraper` CLI | +| Category | What you get | Import | +| ------------------------------------------------------- | ------------------------------------------------------ | --------------------- | +| [Ballot & elections](#ballot--elections) | Contests, candidates, polling places, ballot measures | `@acme/api` | +| [Ballot-measure enrichment](#ballot-measure-enrichment) | Summaries, fiscal impact, pro/con arguments, citations | `@acme/api` | +| [State legislation](#state-legislation) | CA bills, legislators, votes | `@acme/api` | +| [Local government](#local-government) | City/county meetings, ordinances, votes | `@acme/api` | +| [CA election results](#ca-election-results) | Statewide vote counts (live JSON feed) | `@acme/api` | +| [Address autocomplete](#address-autocomplete) | Street-address suggestions + place details | `@acme/api` | +| [Corpus content (DB)](#corpus-content-db) | Bills, court cases, government content articles | tRPC `content` router | +| [Scrapers (background jobs)](#scrapers-background-jobs) | Congress bills, SCOTUS, Federal Register, LAO, CA SOS | `apps/scraper` CLI | --- @@ -32,26 +32,27 @@ All live-API functions come from `@acme/api`. Scrapers run as CLI jobs in `apps/ ```ts import { + getDistrictElectionResults, + getElectionResults, getElections, getVoterInfo, - getElectionResults, - getDistrictElectionResults, } from "@acme/api"; // Or, for direct tRPC consumption: // civic.getVoterInfo({ address, electionId }) ``` -| Function | Args | Returns | -|---|---|---| -| `getElections()` | — | `Election[]` — all elections Google knows about | -| `getVoterInfo(address, electionId)` | street address string, election ID | `VoterInfoResponse` — contests, polling places, drop boxes, enriched measures | -| `getElectionResults(stateFips, countyFips)` | FIPS codes | CA SOS results for the matching jurisdiction | -| `getDistrictElectionResults(districtRef)` | `DistrictRef` | narrowed by district | +| Function | Args | Returns | +| ------------------------------------------- | ---------------------------------- | ----------------------------------------------------------------------------- | +| `getElections()` | — | `Election[]` — all elections Google knows about | +| `getVoterInfo(address, electionId)` | street address string, election ID | `VoterInfoResponse` — contests, polling places, drop boxes, enriched measures | +| `getElectionResults(stateFips, countyFips)` | FIPS codes | CA SOS results for the matching jurisdiction | +| `getDistrictElectionResults(districtRef)` | `DistrictRef` | narrowed by district | **`getVoterInfo` runs the full enrichment pipeline** — ballot measures come back with summaries, fiscal impact, pro/con arguments, and per-field citations from the cross-validation engine. The raw Google Civic response alone omits nearly all content fields; this is what fills them. Key return types (all exported from `@acme/api`): + - `Contest` — a race or measure on the ballot (includes enriched `CanonicalMeasure` fields when present) - `Candidate` — name, party, photo, channels, incumbency, Vote Smart bio - `PollingLocation` — address, hours, type @@ -67,22 +68,21 @@ Key return types (all exported from `@acme/api`): **When to call directly:** when you need enriched measure data outside the `getVoterInfo` flow — e.g. article generation, batch enrichment scripts ```ts -import { - crossValidateMeasure, - type CanonicalMeasure, - type MeasureSourceData, - type MeasureCitation, - type SourceTier, - SOURCE_TIER_RANK, +import type { + CanonicalMeasure, + MeasureCitation, + MeasureSourceData, + SourceTier, } from "@acme/api"; +import type { CanonicalMeasure } from "@acme/api/lib/measure-sources/types"; +import { crossValidateMeasure, SOURCE_TIER_RANK } from "@acme/api"; // or via the named subpath: import { crossValidateMeasure } from "@acme/api/lib/measure-crossvalidate"; -import type { CanonicalMeasure } from "@acme/api/lib/measure-sources/types"; const result: CanonicalMeasure = await crossValidateMeasure( { title: "Proposition 1", - subtitle: "...", // optional — Google Civic's own text, used as last-resort + subtitle: "...", // optional — Google Civic's own text, used as last-resort text: "...", url: "...", }, @@ -96,32 +96,32 @@ const result: CanonicalMeasure = await crossValidateMeasure( `CanonicalMeasure` fields: -| Field | Type | Notes | -|---|---|---| -| `title` | `string` | Measure title (passed through) | -| `summary` | `string \| undefined` | Best available summary; cite `citations` for the source | -| `summaryShort` | `string \| undefined` | One-sentence card preview | -| `summaryLong` | `string \| undefined` | Fuller detail-screen paragraph | -| `summaryIsAiGenerated` | `boolean` | True when AI grounded on fetched text — UI should label it | -| `fiscalImpact` | `string \| undefined` | Official fiscal analysis (LAO for CA props, else Ballotpedia) | -| `fullText` | `string \| undefined` | Full measure text when available | -| `fullTextUrl` | `string \| undefined` | Link to the official text | -| `proArguments` | `MeasureArgument[]` | Attributed pro arguments | -| `conArguments` | `MeasureArgument[]` | Attributed con arguments | -| `citations` | `MeasureCitation[]` | One entry per populated field — `{field, sourceName, sourceUrl, tier, official}` | -| `discrepancies` | `string[] \| undefined` | Fields where top-2 sources disagreed; for human review | +| Field | Type | Notes | +| ---------------------- | ----------------------- | -------------------------------------------------------------------------------- | +| `title` | `string` | Measure title (passed through) | +| `summary` | `string \| undefined` | Best available summary; cite `citations` for the source | +| `summaryShort` | `string \| undefined` | One-sentence card preview | +| `summaryLong` | `string \| undefined` | Fuller detail-screen paragraph | +| `summaryIsAiGenerated` | `boolean` | True when AI grounded on fetched text — UI should label it | +| `fiscalImpact` | `string \| undefined` | Official fiscal analysis (LAO for CA props, else Ballotpedia) | +| `fullText` | `string \| undefined` | Full measure text when available | +| `fullTextUrl` | `string \| undefined` | Link to the official text | +| `proArguments` | `MeasureArgument[]` | Attributed pro arguments | +| `conArguments` | `MeasureArgument[]` | Attributed con arguments | +| `citations` | `MeasureCitation[]` | One entry per populated field — `{field, sourceName, sourceUrl, tier, official}` | +| `discrepancies` | `string[] \| undefined` | Fields where top-2 sources disagreed; for human review | **Sources wired into the engine** (in trust-tier order): -| Source | Tier | Scope | -|---|---|---| -| CA SOS Official Voter Guide | `state_sos` | CA statewide props — AG summary, official pro/con | -| CA LAO Fiscal Analyses | `state_sos` | CA statewide props — nonpartisan fiscal impact | -| LWV / CaVotes | `lwv` | CA statewide props — nonpartisan pros & cons | -| Ballotpedia | `ballotpedia` | Statewide + local lettered measures (all states) | -| Wikipedia | `wikipedia` | CA statewide props — encyclopedic overview | -| Vote Smart | `vote_smart` | State-level measures — summary + pro/con URLs | -| Google Civic | `google_civic` | The original input — lowest trust, always present | +| Source | Tier | Scope | +| --------------------------- | -------------- | ------------------------------------------------- | +| CA SOS Official Voter Guide | `state_sos` | CA statewide props — AG summary, official pro/con | +| CA LAO Fiscal Analyses | `state_sos` | CA statewide props — nonpartisan fiscal impact | +| LWV / CaVotes | `lwv` | CA statewide props — nonpartisan pros & cons | +| Ballotpedia | `ballotpedia` | Statewide + local lettered measures (all states) | +| Wikipedia | `wikipedia` | CA statewide props — encyclopedic overview | +| Vote Smart | `vote_smart` | State-level measures — summary + pro/con URLs | +| Google Civic | `google_civic` | The original input — lowest trust, always present | When no human source has a summary, the engine falls back to grounded AI (SPUR Bay Area voter guide as source text), flagged `summaryIsAiGenerated: true`. @@ -135,55 +135,80 @@ When no human source has a summary, the engine falls back to grounded AI (SPUR B **Scope:** California state bills and legislators (expandable to other states) ```ts +import type { + GetBillsOptions, + OpenStatesBill, + OpenStatesPerson, +} from "@acme/api"; import { - getBills, getBillDetails, - getLegislators, - getVotes, + getBills, + getBillsBySponsor, getCurrentSessions, getLegislatorById, - getBillsBySponsor, + getLegislators, + getVotes, openStatesClient, - type OpenStatesBill, - type OpenStatesPerson, - type GetBillsOptions, } from "@acme/api"; // or import { getBills } from "@acme/api/clients/open-states"; -const bills = await getBills({ state: "ca", session: "20232024", query: "housing" }); +const bills = await getBills({ + state: "ca", + session: "20232024", + query: "housing", +}); const detail = await getBillDetails("ocd-bill/..."); const legislators = await getLegislators({ state: "ca" }); ``` -| Function | What it returns | -|---|---| -| `getBills(opts)` | `OpenStatesBillSearchResult[]` — paginated bill search | -| `getBillDetails(billId)` | `OpenStatesBill` — full bill with actions, sponsors, versions | -| `getLegislators(opts)` | `OpenStatesPerson[]` — legislators matching the query | -| `getVotes(billId)` | `OpenStatesVote[]` — roll-call votes for a bill | -| `getCurrentSessions(state)` | Active legislative sessions | -| `getBillsBySponsor(personId)` | Bills sponsored by a legislator | -| `openStatesClient` | Raw client for custom queries | +| Function | What it returns | +| ----------------------------- | ------------------------------------------------------------- | +| `getBills(opts)` | `OpenStatesBillSearchResult[]` — paginated bill search | +| `getBillDetails(billId)` | `OpenStatesBill` — full bill with actions, sponsors, versions | +| `getLegislators(opts)` | `OpenStatesPerson[]` — legislators matching the query | +| `getVotes(billId)` | `OpenStatesVote[]` — roll-call votes for a bill | +| `getCurrentSessions(state)` | Active legislative sessions | +| `getBillsBySponsor(personId)` | Bills sponsored by a legislator | +| `openStatesClient` | Raw client for custom queries | --- ## Local government -**Source:** Legistar Web API -**Entry point:** `packages/api/src/integrations/legistar.ts` +**Sources:** persisted local-government records plus the Legistar Web API +**Entry points:** `packages/api/src/lib/local-government.ts`, `packages/api/src/integrations/legistar.ts` **Auth:** None (public API) -**Jurisdictions wired:** San Jose (`sanjose`), Santa Clara County (`santaclara`), Sunnyvale (`sunnyvale`) +**Persisted jurisdiction:** Cedar Park (`cedar-park-tx`); live Legistar jurisdictions: San Jose, Santa Clara County, Sunnyvale + +The product-facing reader uses normalized persisted records. Cedar Park's +scheduled scraper populates these from the city's official CivicEngage page and +embedded Municode Meetings publication: ```ts import { - legistar, - LegistarClient, - JURISDICTIONS, - type LegistarMeeting, - type LegistarMatter, - type LegistarVote, + getLocalGovernmentMeeting, + getLocalGovernmentMeetings, } from "@acme/api"; + +const meetings = await getLocalGovernmentMeetings({ + jurisdiction: "cedar-park-tx", + limit: 20, +}); +const detail = meetings[0] + ? await getLocalGovernmentMeeting(meetings[0].id) + : null; +``` + +The equivalent tRPC procedures are `localGovernment.meetings` and +`localGovernment.meeting`. Detail includes official/versioned documents, +agenda items, motions, outcomes, tally text, and named votes when published. + +Legistar remains available as a provider-specific live client: + +```ts +import type { LegistarMatter, LegistarMeeting, LegistarVote } from "@acme/api"; +import { JURISDICTIONS, legistar, LegistarClient } from "@acme/api"; // or import { legistar } from "@acme/api/integrations/legistar"; @@ -192,16 +217,34 @@ const matters = await legistar.getMatters("santaclara", { keywords: "budget" }); const votes = await legistar.getVotes("sanjose", matterId); ``` -| Method | Returns | -|---|---| -| `legistar.getMeetings(jurisdiction, opts?)` | `LegistarMeeting[]` — council meetings with agendas | -| `legistar.getMatters(jurisdiction, opts?)` | `LegistarMatter[]` — ordinances, resolutions, items | -| `legistar.getVotes(jurisdiction, matterId)` | `LegistarVote[]` — how each member voted | -| `legistar.getBodies(jurisdiction)` | `LegistarBody[]` — committees and boards | -| `legistar.getAttachments(jurisdiction, matterId)` | `LegistarAttachment[]` — PDFs, staff reports | +| Method | Returns | +| ------------------------------------------------- | --------------------------------------------------- | +| `legistar.getMeetings(jurisdiction, opts?)` | `LegistarMeeting[]` — council meetings with agendas | +| `legistar.getMatters(jurisdiction, opts?)` | `LegistarMatter[]` — ordinances, resolutions, items | +| `legistar.getVotes(jurisdiction, matterId)` | `LegistarVote[]` — how each member voted | +| `legistar.getBodies(jurisdiction)` | `LegistarBody[]` — committees and boards | +| `legistar.getAttachments(jurisdiction, matterId)` | `LegistarAttachment[]` — PDFs, staff reports | To add a city: add its `*.legistar.com` subdomain to `JURISDICTIONS` in `integrations/legistar.ts`. +### Provider-neutral local meetings + +- **Source:** [City of Durham OnBase Agenda Online](https://cityordinances.durhamnc.gov/OnBaseAgendaOnline/) +- **Reader:** `packages/api/src/lib/local-government.ts` +- **tRPC:** `localGovernment.meetings`, `localGovernment.meeting` + +The Durham and Cedar Park scrapers persist meetings, agenda or minutes item +outlines, supporting-document URLs, and official action/vote text. The reader +is provider-neutral and serves only cached database records; it does not +contact upstream meeting systems in the request path. + +```ts +const meetings = await getLocalGovernmentMeetings({ + jurisdiction: "durham-nc", +}); +const meeting = await getLocalGovernmentMeeting(meetings[0].id); +``` + --- ## CA election results @@ -211,11 +254,8 @@ To add a city: add its `*.legistar.com` subdomain to `JURISDICTIONS` in `integra **Auth:** None ```ts -import { - SOS_RESULTS_HOME, - type ElectionContestResult, - type ResultCandidate, -} from "@acme/api"; +import type { ElectionContestResult, ResultCandidate } from "@acme/api"; +import { SOS_RESULTS_HOME } from "@acme/api"; // or import { SOS_RESULTS_HOME } from "@acme/api/clients/ca-sos-results"; ``` @@ -231,11 +271,8 @@ import { SOS_RESULTS_HOME } from "@acme/api/clients/ca-sos-results"; **Auth:** `GOOGLE_PLACES_API_KEY` (falls back through `GOOGLE_API_KEY` → `GOOGLE_CIVIC_API_KEY`) ```ts -import { - getAddressSuggestions, - getPlaceDetails, - type AddressSuggestion, -} from "@acme/api"; +import type { AddressSuggestion } from "@acme/api"; +import { getAddressSuggestions, getPlaceDetails } from "@acme/api"; const suggestions = await getAddressSuggestions("123 Main St"); const details = await getPlaceDetails(suggestions[0].placeId); @@ -261,24 +298,24 @@ const typed = await trpc.content.getByType.query({ type: "court_case" }); `ContentCard` shape (returned by all three procedures): -| Field | Type | -|---|---| -| `id` | `string` | -| `title` | `string` | -| `description` | `string \| null` | -| `type` | `"bill" \| "court_case" \| "government_content"` | -| `isAIGenerated` | `boolean` | -| `thumbnailUrl` | `string \| null` | +| Field | Type | +| --------------- | ------------------------------------------------ | +| `id` | `string` | +| `title` | `string` | +| `description` | `string \| null` | +| `type` | `"bill" \| "court_case" \| "government_content"` | +| `isAIGenerated` | `boolean` | +| `thumbnailUrl` | `string \| null` | `getThumbnailForContent(id, type)` is also exported from `@acme/api` for cases where only the thumbnail is needed. **DB tables populated by scrapers:** -| Table | Populated by | Content | -|---|---|---| -| `Bill` | `congress.ts` scraper | Federal bills, actions, sponsor, status, full text | -| `GovernmentContent` | `federalregister.ts` scraper | EOs, proclamations, presidential memos | -| `CourtCase` | `scotus.ts` scraper | SCOTUS opinions via CourtListener | +| Table | Populated by | Content | +| ------------------- | ---------------------------- | -------------------------------------------------- | +| `Bill` | `congress.ts` scraper | Federal bills, actions, sponsor, status, full text | +| `GovernmentContent` | `federalregister.ts` scraper | EOs, proclamations, presidential memos | +| `CourtCase` | `scotus.ts` scraper | SCOTUS opinions via CourtListener | --- @@ -286,15 +323,15 @@ const typed = await trpc.content.getByType.query({ type: "court_case" }); These run as CLI jobs (`bun run scrape -- --scrapers `) in `apps/scraper` and are **not importable as functions at request time**. They populate `CivicApiCache` rows or DB tables that the live API reads. -| Scraper | Populates | Cadence | -|---|---|---| -| `congress` | `Bill` table | Periodic | -| `federalregister` | `GovernmentContent` table | Periodic | -| `scotus` | `CourtCase` table | Periodic | -| `vote411` | `CivicApiCache` (VOTE411 voter guides) | Pre-election | -| `sccCvig` | `CivicApiCache` (SCC county voter guide) | Pre-election | -| `caSosStatements` | `CivicApiCache` (CA SOS candidate statements) | Pre-election | -| `caLaoFiscal` | `CivicApiCache` (LAO fiscal analyses) | Pre-election (~90 days out) | +| Scraper | Populates | Cadence | +| ----------------- | --------------------------------------------- | --------------------------- | +| `congress` | `Bill` table | Periodic | +| `federalregister` | `GovernmentContent` table | Periodic | +| `scotus` | `CourtCase` table | Periodic | +| `vote411` | `CivicApiCache` (VOTE411 voter guides) | Pre-election | +| `sccCvig` | `CivicApiCache` (SCC county voter guide) | Pre-election | +| `caSosStatements` | `CivicApiCache` (CA SOS candidate statements) | Pre-election | +| `caLaoFiscal` | `CivicApiCache` (LAO fiscal analyses) | Pre-election (~90 days out) | **How the cache-warmer pattern works:** a scraper pre-fetches expensive HTML pages and stores structured JSON in `CivicApiCache`. When a user triggers `getVoterInfo`, the measure-source adapters read the cache row (sub-millisecond DB read) instead of doing a live HTML fetch inside the request. Cache TTL is 30 days for LAO; 24 hours for voter info responses. diff --git a/docs/measure-enrichment.md b/docs/measure-enrichment.md index 950a9700..56efd1db 100644 --- a/docs/measure-enrichment.md +++ b/docs/measure-enrichment.md @@ -14,6 +14,7 @@ The Google Civic Information API reliably returns ballot-measure **titles** but ``` CA SOS Official Voter Guide ─┐ +TX SOS facts + TLC analysis ─┤ LWV CaVotes (Pros & Cons) ─┤ Ballotpedia (statewide+local)─┤ Wikipedia (statewide props) ─┼──▶ Cross-Validation Engine ──▶ Canonical Measure ──▶ Cache (CivicApiCache) ──▶ App @@ -31,22 +32,23 @@ The engine fetches all sources concurrently (`Promise.all`) and merges them **fi When more than one source covers the same field, the higher tier wins (defined in `measure-sources/types.ts`, `SOURCE_TIER_RANK`): ``` -county_registrar > state_sos > lwv > ballotpedia > wikipedia > vote_smart > google_civic > ai_generated +county_registrar > state_sos = legislative_council > lwv > ballotpedia > wikipedia > vote_smart > google_civic > ai_generated ``` ## Source adapters (wired today) Each adapter fetches over a shared, defensive helper (`measure-sources/fetch.ts`) that uses a browser User-Agent and turns any failure into `null`. -| Source | Adapter | Tier | Scope / method | -| -------------------------------- | ---------------------- | -------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| CA SOS Official Voter Guide | `ca-sos-voterguide.ts` | `state_sos` | CA statewide props. Official AG summary, pro/con, full-text URL — **real official text, not AI.** Matches on the prop number parsed from the title. The guide is rebuilt each cycle and only serves the active election, so it yields nothing between prop cycles. | -| CA LAO Fiscal Analyses | `ca-lao-fiscal.ts` | `state_sos` | CA statewide props. Official nonpartisan fiscal impact analysis for each proposition. HTML scrape of `lao.ca.gov/BallotAnalysis/Proposition?number=N&year=YYYY` — no API exists. Pre-warmed by the `ca-lao-fiscal` scraper into `CivicApiCache`; adapter reads cache first, falls back to live fetch. Only fires for parsed proposition numbers (not local lettered measures). | -| League of Women Voters — CaVotes | `cavotes.ts` | `lwv` | CA statewide props. Nonpartisan "Pros & Cons" summary, fiscal effects, supporter/opponent arguments via the CaVotes WordPress REST API (`cavotes.org/wp-json/wp/v2/ballots`). Slugs are inconsistent across years, so it enumerates the list and matches by prop number + year. | -| Ballotpedia | `ballotpedia.ts` | `ballotpedia` | **Statewide _and_ local** lettered measures — the main source for local. Rendered article HTML (MediaWiki API disabled), resolved from a year/county index. Extracts ballot summary/question, fiscal impact / impartial analysis, arguments. Year-gated so a same-letter measure from another cycle isn't surfaced. | -| Wikipedia | `wikipedia.ts` | `wikipedia` | CA statewide props only — gated on a parsed prop number (local titles like "Measure Q" collide with unrelated articles). MediaWiki extract for ` California Proposition `; neutral encyclopedic overview. | -| Vote Smart | `votesmart.ts` | `vote_smart` | State-level measures. Fuzzy-matches the title to a Vote Smart measure → summary, full-text URL, pro/con URLs. Requires `VOTE_SMART_API_KEY`. | -| Google Civic | (the input itself) | `google_civic` | The measure as the API returned it — lowest-trust, so its subtitle/text still surface when nothing better exists. | +| Source | Adapter | Tier | Scope / method | +| -------------------------------- | ---------------------- | ----------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| CA SOS Official Voter Guide | `ca-sos-voterguide.ts` | `state_sos` | CA statewide props. Official AG summary, pro/con, full-text URL — **real official text, not AI.** Matches on the prop number parsed from the title. The guide is rebuilt each cycle and only serves the active election, so it yields nothing between prop cycles. | +| CA LAO Fiscal Analyses | `ca-lao-fiscal.ts` | `state_sos` | CA statewide props. Official nonpartisan fiscal impact analysis for each proposition. HTML scrape of `lao.ca.gov/BallotAnalysis/Proposition?number=N&year=YYYY` — no API exists. Pre-warmed by the `ca-lao-fiscal` scraper into `CivicApiCache`; adapter reads cache first, falls back to live fetch. Only fires for parsed proposition numbers (not local lettered measures). | +| Texas SOS + Legislative Council | `texas-official.ts` | `state_sos` + `legislative_council` | Current-cycle Texas statewide propositions. Matches Google Civic/Vote Smart-style records by election date/year plus proposition number (title similarity fallback), then supplies separately cited SOS result facts and TLC page-cited explanation. | +| League of Women Voters — CaVotes | `cavotes.ts` | `lwv` | CA statewide props. Nonpartisan "Pros & Cons" summary, fiscal effects, supporter/opponent arguments via the CaVotes WordPress REST API (`cavotes.org/wp-json/wp/v2/ballots`). Slugs are inconsistent across years, so it enumerates the list and matches by prop number + year. | +| Ballotpedia | `ballotpedia.ts` | `ballotpedia` | **Statewide _and_ local** lettered measures — the main source for local. Rendered article HTML (MediaWiki API disabled), resolved from a year/county index. Extracts ballot summary/question, fiscal impact / impartial analysis, arguments. Year-gated so a same-letter measure from another cycle isn't surfaced. | +| Wikipedia | `wikipedia.ts` | `wikipedia` | CA statewide props only — gated on a parsed prop number (local titles like "Measure Q" collide with unrelated articles). MediaWiki extract for ` California Proposition `; neutral encyclopedic overview. | +| Vote Smart | `votesmart.ts` | `vote_smart` | State-level measures. Fuzzy-matches the title to a Vote Smart measure → summary, full-text URL, pro/con URLs. Requires `VOTE_SMART_API_KEY`. | +| Google Civic | (the input itself) | `google_civic` | The measure as the API returned it — lowest-trust, so its subtitle/text still surface when nothing better exists. | **Local lettered measures:** the County Counsel / City Attorney **Impartial Analysis** (carried on Ballotpedia) is extracted on its own and wins the summary slot ahead of the bare ballot question — it's the authoritative neutral text, so it no longer gets buried in the fiscal field, and the advocacy/AI fallback only runs when it's absent. @@ -100,6 +102,7 @@ flowchart TD | `summaryIsAiGenerated` | true if `summary` came from AI | | `fiscalImpact` | official fiscal analysis | | `proArguments[]` / `conArguments[]` | attributed arguments (`{text, author?, sourceName, sourceUrl}`) | +| `result` | official SOS status, outcome, vote totals, choices, as-of time, and source link | | `citations[]` | which source supplied each field (`{field, sourceName, sourceUrl, tier, official}`) | | `sources[]` | one `Source` per citation, for attribution chips | diff --git a/docs/missouri-sos-election-data.md b/docs/missouri-sos-election-data.md new file mode 100644 index 00000000..a13f3555 --- /dev/null +++ b/docs/missouri-sos-election-data.md @@ -0,0 +1,55 @@ +# Missouri SOS current-cycle election data + +> **Status: disabled.** The implementation is retained under +> `apps/scraper/src/scrapers/disabled`, but it is not registered or runnable +> until its database and product integration are provisioned. + +`missouri-sos` ingests only the 2026 Missouri election cycle from the four +official Secretary of State entry points named in issue #186. It does not offer +historical browsing and never requests or persists voter-level data. + +## Discovery and source boundaries + +- The [2026 election calendar](https://www.sos.mo.gov/elections/calendar/2026cal) + determines the active statewide primary or general election and its final + certification date. No archival result URL is constructed. +- The [certified candidate system](https://s1.sos.mo.gov/candidatesonweb/) + supplies the current election code. The scraper follows its cumulative + certified-list and withdrawn/removed-list links, which avoids crawling every + office page. Only office, district, party, candidate name, ballot order, and + withdrawal status are parsed. Address, filing-contact, and other location + cells are deliberately ignored. +- The [2026 certified ballot-measure page](https://www.sos.mo.gov/petitions/2026BallotMeasures) + supplies the election date, official title/language, fair ballot language, + fiscal statement, and links to the full text and signed certificate. Only + sections explicitly labeled certified on that page are accepted. +- The [ShowMO Votes entry point](https://www.sos.mo.gov/elections/showmovotes) + is searched for a link matching the active 2026 election. Before publication, + the snapshot is still stored with an explicit `availability: "unavailable"` + diagnostic. A historical-looking results fixture exists only to lock the + table parser; no historical URL appears in production code. + +All network calls use the shared retry/backoff client and shared runner +concurrency. The candidate system is read through its single cumulative page, +keeping request volume low. `MISSOURI_SOS_MAX_ITEMS` defaults to `1000` and may +be lowered for development runs; a truncation diagnostic is persisted whenever +the cap is reached. + +## Persistence and API + +Missouri reuses PR #181's provider-neutral `election_source_snapshot` table: + +- jurisdiction: `MO` +- cycle: `2026` +- provider: `mo-sos` +- scope: `current-2026` + +The normalized snapshot is SHA-256 hashed without fetch timestamps. Its natural +key is upserted, so unchanged runs remain one row while official corrections or +result updates replace the payload and checksum. Each record carries an +official-source citation, and the reader validates the JSON contract before +returning it. + +Consumers call the input-free public tRPC query +`civic.getMissouriCurrentElection`. The API has no year, election ID, or history +parameter, so it cannot expose a historical cycle accidentally. diff --git a/docs/ncsbe-election-data.md b/docs/ncsbe-election-data.md new file mode 100644 index 00000000..287932b8 --- /dev/null +++ b/docs/ncsbe-election-data.md @@ -0,0 +1,49 @@ +# NCSBE current-cycle election data + +The `ncsbe` scraper discovers official public files from the NCSBE candidate-list +and election-results index pages. It does not hard-code election file URLs. At +runtime, discovery accepts only links for the current calendar-year election +cycle and prefers the structured candidate CSV and result ZIP/TSV files. The +official referendum PDF is used because NCSBE does not publish a structured +referendum download. + +```bash +pnpm --filter @acme/scraper run start ncsbe --max-items 4 +``` + +## Data boundaries + +- Candidate addresses, phone numbers, and email addresses are discarded by the + parser and have no database columns. +- Voter registration and voter-history products are never discovered or fetched. +- Historical links may be represented by small parser regression fixtures, but + runtime discovery, persistence, and API reads reject non-current-cycle dates. +- Durham County is the primary validation jurisdiction. Wake County fixtures + prevent county-specific assumptions. + +## Storage and idempotency + +`election_source` stores provider, source kind, exact URL, fetched time, SHA-256, +parser structure version, election date, and certification state. Provider-neutral +candidate, referendum, and result tables reference that row. A re-run upserts the +same source identity and replaces its child rows in one transaction, so interrupted +runs roll back and completed re-runs do not duplicate records. + +The result ZIP's official media file is marked `official_not_certified`. NCSBE's +separate certified-result PDFs are intentionally not used when structured data is +available; a future certified structured file can use `certified` without changing +the reader contract. + +## API reader and Civic matching + +`civic.getNcElectionData` accepts `county`, an ISO `electionDate`, optional Google +Civic contest references, and optional `includePrecincts`. It returns candidate, +referendum, county-total result, and (when requested) precinct records. Every row +includes its exact source URL, checksum, fetched time, structure version, and +certification state. + +Contest and candidate matching first compares normalized values exactly (case, +punctuation, `NC`/`North Carolina`, and primary-party suffixes are normalized). +The only fuzzy fallback is token Dice similarity of at least `0.82`, and it is +accepted only when the best result is at least `0.08` ahead of the runner-up. +Uncertain matches are omitted rather than guessed. diff --git a/docs/scraper.md b/docs/scraper.md index 19cecab9..82d156fb 100644 --- a/docs/scraper.md +++ b/docs/scraper.md @@ -15,21 +15,54 @@ process environment at runtime, not embedded during the build. ## Scrapers -| Scraper | Source | Content type | Method | -| ---------------------- | ------------------------------ | -------------------- | ------------------------------------------------------------------------------------------------------------------------------- | -| `congress.ts` | congress.gov REST API | `bill` | REST (`CONGRESS_API_KEY`), incremental by source `updateDate` — see [Incremental discovery](#incremental-discovery-congressgov) | -| `federalregister.ts` | federalregister.gov REST API | `government_content` | REST; HTML→Markdown via Turndown | -| `scotus.ts` | CourtListener REST API | `court_case` | REST (`COURTLISTENER_API_KEY`, optional) | -| `vote411.ts` | vote411.org | (cached locally) | cheerio HTML parse; does **not** write to the main DB | -| `scc-cvig.ts` | Santa Clara County voter guide | `civic_api_cache` | PDF extraction; optional Gemini fallback | -| `ca-sos-statements.ts` | CA Secretary of State guide | `civic_api_cache` | official candidate-statement pages | -| `ca-lao-fiscal.ts` | CA LAO ballot analyses | `civic_api_cache` | proposition fiscal analyses via HTML parse | -| `ca-vig-archive.ts` | CA SOS voter-guide archive | `civic_api_cache` | historical proposition guide pages via HTML parse | +| Scraper | Source | Content type | Method | +| --------------------------- | -------------------------------- | -------------------------- | ----------------------------------------------------------------------- | +| `congress.ts` | congress.gov REST API | `bill` | REST (`CONGRESS_API_KEY`), incremental by source `updateDate` | +| `federalregister.ts` | federalregister.gov REST API | `government_content` | REST; HTML→Markdown via Turndown | +| `scotus.ts` | CourtListener REST API | `court_case` | REST (`COURTLISTENER_API_KEY`, optional) | +| `vote411.ts` | vote411.org | (cached locally) | cheerio HTML parse; does **not** write to the main DB | +| `scc-cvig.ts` | Santa Clara County voter guide | `civic_api_cache` | PDF extraction; optional Gemini fallback | +| `ca-sos-statements.ts` | CA Secretary of State guide | `civic_api_cache` | official candidate-statement pages | +| `ca-lao-fiscal.ts` | CA LAO ballot analyses | `civic_api_cache` | proposition fiscal analyses via HTML parse | +| `ca-vig-archive.ts` | CA SOS voter-guide archive | `civic_api_cache` | historical proposition guide pages via HTML parse | +| `texas-current-election.ts` | Texas SOS + TLC | `election_source_snapshot` | current-cycle JSON + deterministic PDF text parsing | +| `texas-legislature.ts` | Texas Legislative Council FTP | `bill` | current-session XML + bulk HTML; no site mining | +| `civicengage.ts` | Cedar Park official council page | local-government tables | CivicEngage entry page + Municode embed; deterministic HTML/PDF parsing | +| `durham-onbase.ts` | Durham OnBase Agenda Online | local-government tables | current-cycle meetings, items, attachments, and official actions | +| `durham-bocc.ts` | Durham County Legistar API | local-government tables | current-cycle meetings, items, actions, votes, and documents | All HTTP goes through one `fetchWithRetry()` utility (`apps/scraper/src/utils/fetch.ts`): exponential backoff (1s/2s/4s…), `Retry-After` support (seconds or HTTP-date), 30s default timeout via `AbortController`, retriable on 429/5xx and `ECONNRESET`/`ECONNREFUSED`, plus a stateful **per-host backoff** that ramps on 429/5xx and relaxes on success. > Note: `whitehouse.gov` cheerio scraping was replaced by the structured **Federal Register** REST API. `vote411-ballot.ts` exists for address-based ballot lookup (needs Playwright) but isn't wired into the CLI. +The Texas scraper is intentionally current-cycle only. SOS election facts and +TLC explanatory text remain separate provider snapshots and separate citations; +see [Texas current-election data](./texas-current-election.md). + +The Cedar Park pilot is deliberately limited to City Council meetings from the +latest 12 months. It stores meetings, versioned documents, agenda items, +motions/outcomes, and named roll-call votes in provider-neutral tables. It does +not browse or backfill historical election cycles. + +### Durham OnBase + +Run `pnpm -F @acme/scraper start:dev -- durham-onbase`. The scraper reads the +official OnBase embedded meeting JSON, agenda/minutes HTML outlines, and agenda +item detail endpoints. It does not use AI or PDF vision. Requests are serialized +at a minimum 250 ms interval, use the shared retry/backoff client, and skip rows +refreshed in the last 24 hours by default. + +Only meetings in the current calendar-year council cycle are ingested. The +product does not expose historical election cycles or run an OnBase backfill. +`DURHAM_ONBASE_MAX_ITEMS` (default `100`) limits meetings and +`DURHAM_ONBASE_CACHE_TTL_HOURS` (default `24`) controls refresh frequency. + +### Durham County BOCC + +`durham-bocc` reads Durham County's official structured Legistar feed for BOCC body ID `138`. It bounds discovery to the current two-year election cycle, upserts stable provider IDs, and stores checksums/source row versions so replaced agendas update the existing meeting. Cancellations and explicit amendments remain visible; Spanish attachments are tagged separately. Agenda/minutes PDFs are retained as official links and are not sent through AI or OCR when structured item/action fields are available. + +Run it with `pnpm --filter @acme/scraper run start durham-bocc`. The default cap is 100 meetings and can be changed with `DURHAM_BOCC_MAX_ITEMS` or `--max-items`. + ## Incremental discovery (congress.gov) Each run walks the congress.gov bill feed forward from a stored cursor. Three @@ -90,40 +123,19 @@ giant bill cannot wedge the cursor. Section-aware storage is the real fix `apps/scraper/src/utils/db/operations.ts` centralizes writes behind a discriminated-union `upsertContent(type, data)` (`type` ∈ bill | government_content | court_case). Each run: 1. Compute a SHA-256 over the type-specific key fields (title, summary, full text, status…). -2. Look up the existing row by its natural key (`(billNumber, sourceWebsite)`, `url`, or `caseNumber`). +2. Look up the existing row by its natural key (`(billNumber, sourceWebsite, legislativeSession)`, `url`, or `caseNumber`). 3. **Unchanged hash** → skip AI entirely; backfill only missing AI assets. -4. **New bill without a source description** → generate the required description first; defer the insert only if there is no summary source at all. +4. **New bill without a source description** → generate the required description first; defer the insert if source text or every provider is unavailable. 5. **New or changed** → run the remaining AI pipeline, upsert via `onConflictDoUpdate`, append to `versions`. -### The new-item budget - -`SCRAPER_MAX_NEW_ITEMS_PER_RUN` (default 10) caps how many items pay for AI -generation in one run. Items past the cap are **still persisted with their raw -content** — they just skip the description, article, lens, brief, and video, -and are picked up later by the retroactive scripts. - -The cap counts **items that generate**, not new items, and each item draws at -most one slot however many assets it produces. A slot is claimed at the point -of generation, after each asset's own cache check, so a fully cached item costs -nothing and does not consume budget. Gating on "new" instead left the expensive -case uncapped: an existing bill whose content changed regenerated its brief and -its dual lens with no limit, which meant a backfill correcting stored text -ignored the budget almost entirely. - -Persisting them is not optional. The cursor advances past everything the run -fetched, so an item not written here is lost rather than deferred. Between -2026-07-21 and 2026-07-28 a regression skipped the insert instead, and new -bills per day fell from ~86 to under 10 while 1,742 upstream updates produced -17 rows. A raw row still renders because the content API coalesces -`description → summary`. - -Every derived asset must be gated on the budget for the cap to mean anything — -the dual lens in particular runs an agentic research loop. The article/summary/ -image block, the lens, the brief, and video generation all claim through the -same per-item function. - `SCRAPER_FORCE_AI_REGEN=1` overrides the cache. A `isUsableText()` gate refuses to feed AI any text under 200 chars or that's mostly blank/all-caps/single-word lines — keeps the model from "summarizing" garbage. +Texas is the provider-neutral state-bill extension to this flow. Its rows carry +an OCD jurisdiction, legislative session, subjects, sponsorships, documents, +votes, and an optional exact Open States ID. The scraper selects only the latest +FTP session and sets `skipEnrichment`; the API exposes only that newest session +through `content.texasBills`, with full supporting material in `content.getById`. + ## AI Pipeline Provider config lives in `apps/scraper/src/utils/ai/provider.ts`: text uses **OpenRouter** first, then an OpenAI-compatible local endpoint (`LOCAL_LLM_BASE_URL`, such as Ollama), with direct DeepSeek retained only as a deprecated last resort. PDF vision fallback uses **Gemini `gemini-2.5-flash`**. Images use hosted **Black Forest Labs FLUX.2 Klein 9B**, then `LOCAL_FLUX_BASE_URL` as a local fallback. Provider usage and hosted-image costs are tracked per run. @@ -132,13 +144,8 @@ Each new/changed item runs through: 1. **Summary** (`text-generation.ts`) — ≤100-char punchy summary, 8th-grade reading level. 2. **Article** (`text-generation.ts`) — structured 4-section markdown: _What This Means For You_, _Overview_, _Impact & Implications_, _The Debate_; balanced across perspectives. Stored in `ai_generated_article`. Throws a typed `AIRateLimitError` on 429. -3. **Brief** (`bill-brief.ts`, bills only) — the structured document that replaces the markdown wall of text in the app: hook, stat tiles, before/after changes, affected groups, unknowns, glossary, optional prose. Quotes are verified verbatim against the source and stripped if they don't match; loaded political phrasing in the model's own voice triggers one regeneration. The brief also receives the official CRS summary as authoritative, - explicitly non-quotable context, so provisions past its source window are still - known to it; quotes are verified against the official text alone. Cached in - `content_brief` by `contentHash`. See [Article generation](./article-generation.md). -4. **Dual lens** (`text-generation.ts`) — proponent/opponent arguments grounded in an agentic web-research loop, cached in `content_lens`. This is the most expensive step and the only one whose output is not reproducible: the same input can return different arguments, and the row is overwritten in place with no history. It is therefore cached on **its own inputs** (title + full text + article type + model version), not on the bill's overall `contentHash` — that hash also covers status and summary, so a routine action update ("Referred to committee" → "Received in the Senate") used to invalidate it and re-roll the dice. One such re-roll lost a finding that H.R. 7008 carried unrelated voter-ID provisions. -5. **Marketing copy** (`marketing-generation.ts`) — Zod-validated `{ title ≤25 chars, description ≤25 words, imagePrompt }` for the `video` feed card. -6. **Imagery** — multiple sources: +3. **Marketing copy** (`marketing-generation.ts`) — Zod-validated `{ title ≤25 chars, description ≤25 words, imagePrompt }` for the `video` feed card. +4. **Imagery** — multiple sources: - _Scraped thumbnail_ (preferred, free): source-provided image URL → `thumbnail_url`. - _Generated_: hosted FLUX.2 Klein 9B produces a 1024×1024 image, falling back to the configured local FLUX server at 768×768; `sharp` converts PNG→JPEG (q85); bytes land in the `image_data` `bytea` column. Hosted calls retry with backoff; moderation blocks return `null` silently. - _Stock-photo fallback_: `image-keywords.ts` → Google Custom Search (`GOOGLE_API_KEY` + `GOOGLE_SEARCH_ENGINE_ID`) can supply a thumbnail URL. diff --git a/docs/st-louis-aldermen.md b/docs/st-louis-aldermen.md new file mode 100644 index 00000000..b46dbe84 --- /dev/null +++ b/docs/st-louis-aldermen.md @@ -0,0 +1,57 @@ +# St. Louis Board of Aldermen Source Contract + +> **Status: disabled.** The implementation is retained under +> `apps/scraper/src/scrapers/disabled`, but it is not registered or runnable +> until its database and product integration are provisioned. + +## Scope and official sources + +The `st-louis-aldermen` scraper ingests full-board and committee meetings from +the City of St. Louis. It uses four official structured surfaces: + +- the Full Board Meeting Agendas page for the selected session, + `agendaViewID`, agenda/minutes links, Board Bills, resolutions, and sponsors; +- the Aldermanic Calendar and event pages for City event IDs, meeting times, + types, locations, published video links, and CivicClerk meeting IDs; +- Board Bill and resolution detail pages for stable legislative IDs, titles, + primary sponsors, latest activity, and published bill text; +- the City's keyless CivicClerk public API for committee agenda trees, + attachments, agendas, minutes, and packets. + +Production discovery fetches the agenda and calendar pages with no session +parameter and accepts only their selected `