Add North Carolina and Texas civic data scrapers

apps/expo/src/components/UpcomingMeetingsSection.tsx

@@ -7,16 +7,16 @@ import {
 import { FontAwesome } from "@expo/vector-icons";
 import { useQuery } from "@tanstack/react-query";
 
-import type { LegistarMeeting } from "@acme/api/integrations/legistar";
+import type { RouterOutputs } from "@acme/api";
 
 import { Text, View } from "~/components/Themed";
 import { fontBody, fontEditorial, fontSize, rd, sp, useTheme } from "~/styles";
 import { trpc } from "~/utils/api";
 
+type Meeting = RouterOutputs["localGovernment"]["listMeetings"][number];
+
 interface UpcomingMeetingsSectionProps {
-  onMeetingPress?: (
-    meeting: LegistarMeeting & { jurisdiction: string },
-  ) => void;
+  onMeetingPress?: (meeting: Meeting) => void;
 }
 
 function formatDate(iso: string): string {
@@ -34,7 +34,7 @@ export function UpcomingMeetingsSection({
   const { theme } = useTheme();
 
   const meetingsQuery = useQuery(
-    trpc.legistar.getMeetings.queryOptions({ daysAhead: 30 }),
+    trpc.localGovernment.listMeetings.queryOptions({ daysAhead: 90 }),
   );
 
   return (
@@ -47,30 +47,43 @@ export function UpcomingMeetingsSection({
 
       {meetingsQuery.data?.slice(0, 8).map((meeting, index) => (
         <TouchableOpacity
-          key={`${meeting.EventId}-${index}`}
+          key={`${meeting.source}-${meeting.externalId}-${index}`}
           style={[styles.card, { backgroundColor: theme.card }]}
-          onPress={() => onMeetingPress?.(meeting)}
+          onPress={() =>
+            onMeetingPress
+              ? onMeetingPress(meeting)
+              : void Linking.openURL(meeting.canonicalUrl)
+          }
           activeOpacity={0.8}
         >
           <View style={styles.cardAccent} />
           <View style={styles.cardContent}>
             <View style={styles.meta}>
               <Text style={styles.jurisdiction}>{meeting.jurisdiction}</Text>
-              <Text style={styles.date}>{formatDate(meeting.EventDate)}</Text>
+              <Text style={styles.date}>
+                {formatDate(meeting.startsAt.toString())}
+              </Text>
             </View>
             <Text style={styles.title} numberOfLines={2}>
-              {meeting.EventBodyName}
+              {meeting.isCancelled ? "Cancelled: " : ""}
+              {meeting.title}
             </Text>
-            {meeting.EventLocation && (
+            {meeting.location && (
               <Text style={styles.location} numberOfLines={1}>
-                {meeting.EventLocation}
+                {meeting.location}
               </Text>
             )}
             <View style={styles.icons}>
-              {meeting.EventAgendaFile && (
+              {meeting.documents.find(
+                (document) => document.type === "agenda",
+              ) && (
                 <TouchableOpacity
                   onPress={() =>
-                    void Linking.openURL(meeting.EventAgendaFile ?? "")
+                    void Linking.openURL(
+                      meeting.documents.find(
+                        (document) => document.type === "agenda",
+                      )?.url ?? "",
+                    )
                   }
                   hitSlop={8}
                 >
@@ -81,11 +94,9 @@ export function UpcomingMeetingsSection({
                   />
                 </TouchableOpacity>
               )}
-              {meeting.EventVideoPath && (
+              {meeting.videoUrl && (
                 <TouchableOpacity
-                  onPress={() =>
-                    void Linking.openURL(meeting.EventVideoPath ?? "")
-                  }
+                  onPress={() => void Linking.openURL(meeting.videoUrl ?? "")}
                   hitSlop={8}
                 >
                   <FontAwesome
@@ -95,10 +106,16 @@ export function UpcomingMeetingsSection({
                   />
                 </TouchableOpacity>
               )}
-              {meeting.EventMinutesFile && (
+              {meeting.documents.find(
+                (document) => document.type === "minutes",
+              ) && (
                 <TouchableOpacity
                   onPress={() =>
-                    void Linking.openURL(meeting.EventMinutesFile ?? "")
+                    void Linking.openURL(
+                      meeting.documents.find(
+                        (document) => document.type === "minutes",
+                      )?.url ?? "",
+                    )
                   }
                   hitSlop={8}
                 >

apps/scraper/README.md

@@ -4,15 +4,20 @@ Pulls in government content like bills, court cases, and White House content and
 
 ## Active data sources
 
-Only these five are registered and run by `all`:
-
-| CLI name            | Source and data fetched                                                                    | Stored/used as                                                                                        |
-| ------------------- | ------------------------------------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------- |
-| `federalregister`   | Federal Register API presidential documents, then each document's body HTML                | `government_content`; AI article/summary and feed-image enrichment                                    |
-| `congress`          | Congress.gov API bill list, detail, CRS summaries, formatted text, and legislative actions | `bill`; powers federal bill content and AI/feed enrichment                                            |
-| `scotus`            | CourtListener opinion clusters, dockets, and sub-opinion text for the Supreme Court        | `court_case`; powers court content and AI/feed enrichment                                             |
-| `scc-cvig`          | Hand-configured Santa Clara County voter-guide PDFs                                        | Candidate statements in `CivicApiCache`; the API matches statements to candidates                     |
-| `ca-sos-statements` | California SOS statewide-office candidate-statement pages                                  | Candidate statements in `CivicApiCache`; the API reads the cache and can fall back to the live source |
+These data sources are registered and run by `all`:
+
+| CLI name                 | Source and data fetched                                                                    | Stored/used as                                                                                        |
+| ------------------------ | ------------------------------------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------- |
+| `federalregister`        | Federal Register API presidential documents, then each document's body HTML                | `government_content`; AI article/summary and feed-image enrichment                                    |
+| `durham-bocc`            | Durham County's official Legistar API (current election cycle only)                        | Provider-neutral meetings, agenda items, actions, votes, and official document links; no AI required  |
+| `congress`               | Congress.gov API bill list, detail, CRS summaries, formatted text, and legislative actions | `bill`; powers federal bill content and AI/feed enrichment                                            |
+| `scotus`                 | CourtListener opinion clusters, dockets, and sub-opinion text for the Supreme Court        | `court_case`; powers court content and AI/feed enrichment                                             |
+| `scc-cvig`               | Hand-configured Santa Clara County voter-guide PDFs                                        | Candidate statements in `CivicApiCache`; the API matches statements to candidates                     |
+| `ca-sos-statements`      | California SOS statewide-office candidate-statement pages                                  | Candidate statements in `CivicApiCache`; the API reads the cache and can fall back to the live source |
+| `ncsbe`                  | Current-cycle NCSBE candidate CSV, referendum PDFs, and result ZIPs                        | Provider-neutral election tables; powers `civic.getNcElectionData` with exact file provenance         |
+| `texas-legislature`      | Texas Legislative Council anonymous FTP: current-session history XML and bulk documents    | State-aware `bill` rows; read through `content.texasBills` and `content.getById`                      |
+| `texas-current-election` | Texas SOS structured election feed and TLC amendment analyses                              | Current-cycle snapshots; powers `civic.getTexasCurrentElection` and measure enrichment                |
+| `cedar-park-council`     | Cedar Park's CivicEngage City Council page and its official Municode Meetings embed        | Provider-neutral local meetings, documents, agenda items, motions, outcomes, and votes                |
 
 `vote411`, `ca-lao-fiscal`, and `ca-vig-archive` remain under
 `src/scrapers/disabled/` and do not run. Their caches had no application
@@ -57,6 +62,7 @@ work:
 | `COURTLISTENER_API_KEY`                      | Optional                                | Higher CourtListener limits for `scotus`.                                                                          |
 | `GOOGLE_API_KEY` / `GOOGLE_SEARCH_ENGINE_ID` | Optional pair                           | Google Custom Search article thumbnails.                                                                           |
 | `GOOGLE_GENERATIVE_AI_API_KEY`               | Optional                                | Gemini vision fallback for `scc-cvig` PDF extraction.                                                              |
+| `OPEN_STATES_API_KEY`                        | Optional                                | Adds an exact Open States bill ID when Texas jurisdiction/session/identifier match.                                |
 
 See [the launch environment guide](../../docs/launch.md) for the complete
 per-scraper matrix, provider setup links, defaults, and production guidance.
@@ -128,6 +134,11 @@ CONGRESS_MAX_ITEMS=10 pnpm --filter @acme/scraper run start congress
 | `SCOTUS_MAX_ITEMS`              |      50 | CourtListener opinion clusters                      |
 | `SCC_CVIG_MAX_ITEMS`            |      10 | Voter-guide PDF documents                           |
 | `CA_SOS_MAX_ITEMS`              |       9 | Statewide-office candidate-statement pages          |
+| `NCSBE_MAX_ITEMS`               |       4 | Current-cycle candidate/referendum/result files     |
+| `TEXAS_LEGISLATURE_MAX_ITEMS`   |     100 | Bills from the latest Texas bulk session            |
+| `TX_SOS_MAX_ITEMS`              |      12 | Current-cycle Texas SOS election payloads           |
+| `CEDAR_PARK_COUNCIL_MAX_ITEMS`  |     100 | Council meetings (after the 12-month cutoff)        |
+| `DURHAM_BOCC_MAX_ITEMS`         |     100 | Current-cycle Durham County BOCC meetings           |
 | `SCRAPER_MAX_NEW_ITEMS_PER_RUN` |      10 | New records receiving expensive AI/image enrichment |
 
 These are per-run limits, not durable calendar-day quotas. Schedule one run per
@@ -137,6 +148,33 @@ invocation gets a fresh allowance. Source limits cap API/page work;
 bills that require a generated description are deferred before insertion;
 other content may still be stored raw for later backfill.
 
+The NCSBE integration is intentionally current-cycle only and excludes voter
+history plus candidate contact/address fields. See
+[`docs/ncsbe-election-data.md`](../../docs/ncsbe-election-data.md) for discovery,
+idempotency, provenance, API, and deterministic Civic-matching details.
+
+## Cedar Park City Council (`civicengage.ts`)
+
+Cedar Park's public site is CivicEngage, but the City Council records page now
+embeds a keyless Municode Meetings publish page. The adapter follows that
+official embed, keeps a 12-month Council-only window, downloads at most two
+documents concurrently, and deterministically parses PDF text. AI/vision is not
+used. Agendas, packets, minutes, and later document URLs are versioned beneath
+one provider-neutral meeting record; unchanged reruns produce the same natural
+keys and checksums.
+
+```bash
+pnpm --filter @acme/scraper run start cedar-park-council --max-items 1
+```
+
+To add a second CivicEngage jurisdiction using the same kind of official
+Municode embed, add a `CivicEngageJurisdictionConfig` beside
+`cedarParkCouncilSource` with its CivicEngage host/path, IANA timezone,
+Municode `cid`/`ppid`, and governing-body matcher, then instantiate the same
+discovery/parser pipeline. If its records page uses Agenda Center, Legistar, or
+another provider, implement that provider behind the same local-government
+persistence contract instead.
+
 ---
 
 ## Congress bills (`congress.ts`)
@@ -166,6 +204,34 @@ for why the cursor is source-based and why the order matters.
 
 ---
 
+## Texas Legislature Online (`texas-legislature.ts`)
+
+Uses only the Texas Legislative Council's anonymous FTP service at
+`ftp.legis.state.tx.us`; it does not crawl interactive TLO bill pages. The job:
+
+- discovers the newest session directory under `/bills` and rejects a stale
+  `TEXAS_LEGISLATURE_SESSION` assertion (official codes look like `89R` or
+  `892`);
+- parses bill-history XML for identity, caption, sponsors, subjects, actions,
+  structured votes when present, and document metadata;
+- downloads bill text, analyses, and fiscal notes from the matching FTP bulk
+  HTML paths and stores their extracted text alongside official HTML/PDF links;
+- optionally stores an exact Open States ID without using Open States as the
+  legislative data source.
+
+Run a small current-session import:
+
+```bash
+pnpm --filter @acme/scraper run start texas-legislature --max-items 10
+```
+
+Apply `packages/db/migrations/add_state_legislation_fields.sql` before the first
+run. The public `content.texasBills` procedure lists only the newest persisted
+Texas session; `content.getById` returns its documents, actions, and votes. This
+work deliberately does not provide historical-session browsing or backfills.
+
+---
+
 ## Court cases (`scotus.ts`)
 
 Uses the [CourtListener API](https://www.courtlistener.com/api/) — free, works without a key. Fetches recent opinions and pulls in the plain-text opinion content for AI article generation.
@@ -193,6 +259,9 @@ All scrapers call into `src/utils/db/operations.ts`. Each time a bill or case is
 - If the **content changed** → regenerates the article
 - If **nothing changed** → backfills any missing AI summary/article/thumbnail fields, otherwise skips AI generation
 
+The Texas importer passes `skipEnrichment` so official source data is persisted
+without AI summaries, generated imagery, or videos.
+
 Set `SCRAPER_FORCE_AI_REGEN=1` to force a full AI refresh even when the record already has AI content.
 
 For a new bill whose description must be generated, the summary is now created

apps/scraper/package.json

@@ -14,6 +14,7 @@
     "@ai-sdk/openai-compatible": "2.0.59",
     "@openrouter/ai-sdk-provider": "^2.10.0",
     "ai": "^6.0.141",
+    "basic-ftp": "^6.0.1",
     "cheerio": "^1.2.0",
     "consola": "^3.4.2",
     "drizzle-orm": "^0.45.2",

apps/scraper/src/fixtures/missouri-sos/calendar.html

@@ -0,0 +1,36 @@
+<main id="main">
+  <table class="cal">
+    <tr>
+      <th>Official Election Day</th>
+      <th>Style</th>
+      <th>Registration</th>
+      <th>Filing</th>
+      <th>Close</th>
+      <th>Final Certification</th>
+    </tr>
+    <tr>
+      <td>April 7, 2026</td>
+      <td>General Municipal Election Day</td>
+      <td></td>
+      <td></td>
+      <td></td>
+      <td>January 27, 2026</td>
+    </tr>
+    <tr>
+      <td>August 4, 2026</td>
+      <td>Primary Election</td>
+      <td></td>
+      <td></td>
+      <td></td>
+      <td>May 26, 2026</td>
+    </tr>
+    <tr>
+      <td>November 3, 2026</td>
+      <td>General Election</td>
+      <td></td>
+      <td></td>
+      <td></td>
+      <td>August 25, 2026</td>
+    </tr>
+  </table>
+</main>

apps/scraper/src/fixtures/missouri-sos/candidate-index.html

@@ -0,0 +1,11 @@
+<div id="main">
+  <h2>2026 Primary Election</h2>
+  <a
+    href="DisplayCandidatesPlacement.aspx?OfficeCode=20%20SE%2002&amp;ElectionCode=750006905"
+    >State Senator - District 2</a
+  >
+  <a
+    href="DisplayCandidatesPlacement.aspx?OfficeCode=20%20LE%20011&amp;ElectionCode=750006905"
+    >State Representative - District 11</a
+  >
+</div>

apps/scraper/src/fixtures/missouri-sos/candidates.html

@@ -0,0 +1,49 @@
+<div id="main">
+  <h1>Certified Candidate List</h1>
+  <h2>2026 Primary Election</h2>
+  <h3>State Senator - District 2</h3>
+  <table>
+    <caption>
+      Republican
+    </caption>
+    <tr>
+      <th class="NameCol">Name</th>
+      <th>Mailing Address</th>
+    </tr>
+    <tr>
+      <td class="NameCol">Alice First</td>
+      <td class="AddressCol">101 Secret Street</td>
+    </tr>
+    <tr>
+      <td class="NameCol">Bob Second</td>
+      <td class="AddressCol">202 Private Avenue</td>
+    </tr>
+  </table>
+  <table>
+    <caption>
+      Democratic
+    </caption>
+    <tr>
+      <th class="NameCol">Name</th>
+      <th>Mailing Address</th>
+    </tr>
+    <tr>
+      <td class="NameCol">Carol Third</td>
+      <td class="AddressCol">303 Hidden Road</td>
+    </tr>
+  </table>
+  <h3>State Representative - District 11</h3>
+  <table>
+    <caption>
+      Libertarian
+    </caption>
+    <tr>
+      <th class="NameCol">Name</th>
+      <th>Mailing Address</th>
+    </tr>
+    <tr>
+      <td class="NameCol">Dana Fourth</td>
+      <td class="AddressCol">404 Confidential Lane</td>
+    </tr>
+  </table>
+</div>

apps/scraper/src/fixtures/missouri-sos/historical-results-layout.html

@@ -0,0 +1,27 @@
+<!-- Historical-looking layout is a parser fixture only. Production never references this path or a historical URL. -->
+<div id="main">
+  <p>Last Updated: November 4, 2024 11:45 PM | Unofficial Results</p>
+  <table data-reporting-status="100% precincts reporting">
+    <caption>
+      Governor
+    </caption>
+    <tr>
+      <th>Candidate</th>
+      <th>Party</th>
+      <th>Total Votes</th>
+      <th>Percent</th>
+    </tr>
+    <tr>
+      <td>Example Candidate A</td>
+      <td>Republican</td>
+      <td>1,250</td>
+      <td>52.1%</td>
+    </tr>
+    <tr>
+      <td>Example Candidate B</td>
+      <td>Democratic</td>
+      <td>1,149</td>
+      <td>47.9%</td>
+    </tr>
+  </table>
+</div>