chore: import scorecards next to deps.dev data from openssf dataset (CM-1227)

.claude/skills/packages-worker-add-entrypoint/SKILL.md

@@ -19,7 +19,6 @@ each worker in its own `src/{worker}/` directory with its own entry point.
 services/apps/packages_worker/
   src/
     bin/
-      packages-worker.ts        ← parent stub
       github-repos-enricher.ts  ← existing worker
       <name>.ts                 ← entry point you will create
     github/                     ← existing worker logic

backend/src/osspckgs/migrations/V1781074345__add-scorecard-job-kinds.sql

@@ -0,0 +1,11 @@
+-- Extend osspckgs_ingest_jobs.job_kind CHECK constraint to include scorecard kinds.
+-- Required for ingestScorecard workflow (CM-1227).
+ALTER TABLE osspckgs_ingest_jobs
+  DROP CONSTRAINT osspckgs_ingest_jobs_job_kind_check,
+  ADD CONSTRAINT osspckgs_ingest_jobs_job_kind_check CHECK (job_kind IN (
+    'packages', 'versions', 'package_dependencies',
+    'repos', 'package_repos',
+    'advisories', 'advisory_packages',
+    'dependent_counts',
+    'scorecard_repos', 'scorecard_checks'
+  ));

docs/adr/0001-oss-packages-design-decisions.md

@@ -59,7 +59,6 @@ All packages_worker sub-workers live in a single npm package (`services/apps/pac
 services/apps/packages_worker/
   src/
     bin/
-      packages-worker.ts      ← parent / health-check stub
       github-repos-enricher.ts
     enricher/                 ← github-repos-enricher logic
     npm/                      ← npm worker (future)

scripts/builders/packages.env

@@ -1,4 +1,4 @@
 DOCKERFILE="./services/docker/Dockerfile.packages"
 CONTEXT="../"
 REPO="sjc.ocir.io/axbydjxa5zuh/packages"
-SERVICES="packages github-repos-enricher deps-dev-ingest npm-worker maven-worker osv-worker"
+SERVICES="github-repos-enricher bq-dataset-ingest npm-worker maven-worker osv-worker"

scripts/services/deps-dev-ingest.yaml → scripts/services/bq-dataset-ingest.yaml

@@ -3,18 +3,18 @@ version: '3.1'
 x-env-args: &env-args
   DOCKER_BUILDKIT: 1
   NODE_ENV: docker
-  SERVICE: deps-dev-ingest
-  CROWD_TEMPORAL_TASKQUEUE: deps-dev-ingest
+  SERVICE: bq-dataset-ingest
+  CROWD_TEMPORAL_TASKQUEUE: bq-dataset-ingest
   CROWD_TEMPORAL_NAMESPACE: ${CROWD_PACKAGES_TEMPORAL_NAMESPACE}
   SHELL: /bin/sh
   SUPPRESS_NO_CONFIG_WARNING: 'true'
 
 services:
-  deps-dev-ingest:
+  bq-dataset-ingest:
     build:
       context: ../../
       dockerfile: ./scripts/services/docker/Dockerfile.packages
-    command: 'pnpm run start:deps-dev-ingest'
+    command: 'pnpm run start:bq-dataset-ingest'
     working_dir: /usr/crowd/app/services/apps/packages_worker
     env_file:
       - ../../backend/.env.dist.local
@@ -27,11 +27,11 @@ services:
     networks:
       - crowd-bridge
 
-  deps-dev-ingest-dev:
+  bq-dataset-ingest-dev:
     build:
       context: ../../
       dockerfile: ./scripts/services/docker/Dockerfile.packages
-    command: 'pnpm run dev:deps-dev-ingest'
+    command: 'pnpm run dev:bq-dataset-ingest'
     working_dir: /usr/crowd/app/services/apps/packages_worker
     # user: '${USER_ID}:${GROUP_ID}'
     env_file:
@@ -41,7 +41,7 @@ services:
       - ../../backend/.env.override.composed
     environment:
       <<: *env-args
-    hostname: deps-dev-ingest
+    hostname: bq-dataset-ingest
     networks:
       - crowd-bridge
     volumes:

services/apps/packages_worker/package.json

@@ -2,9 +2,8 @@
   "name": "@crowd/packages-worker",
   "private": true,
   "scripts": {
-    "start:packages-worker": "CROWD_TEMPORAL_TASKQUEUE=packages-worker CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=packages-worker tsx src/bin/packages-worker.ts",
     "start:criticality-worker": "CROWD_TEMPORAL_TASKQUEUE=packages-worker CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=criticality-worker tsx src/bin/criticality-worker.ts",
-    "start:deps-dev-ingest": "CROWD_TEMPORAL_TASKQUEUE=deps-dev-ingest CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=deps-dev-ingest tsx src/bin/deps-dev-ingest.ts",
+    "start:bq-dataset-ingest": "CROWD_TEMPORAL_TASKQUEUE=bq-dataset-ingest CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=bq-dataset-ingest tsx src/bin/bq-dataset-ingest.ts",
     "start:npm-worker": "CROWD_TEMPORAL_TASKQUEUE=npm-worker CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=npm-worker tsx src/bin/npm-worker.ts",
     "start:osv-worker": "CROWD_TEMPORAL_TASKQUEUE=osv-worker CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=osv-worker tsx src/bin/osv-worker.ts",
     "start:github-repos-enricher": "SERVICE=github-repos-enricher tsx src/bin/github-repos-enricher.ts",
@@ -13,26 +12,24 @@
     "dev:pagerank": "tsx --expose-gc src/criticality/run-pagerank.ts",
     "start:pom-fetcher": "SERVICE=pom-fetcher tsx src/bin/pom-fetcher.ts",
     "backfill:maven": "SERVICE=maven tsx src/bin/maven-backfill.ts",
-    "dev:packages-worker": "CROWD_TEMPORAL_TASKQUEUE=packages-worker CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=packages-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9233 src/bin/packages-worker.ts",
     "dev:criticality-worker": "CROWD_TEMPORAL_TASKQUEUE=packages-worker CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=criticality-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9237 src/bin/criticality-worker.ts",
     "start:maven-worker": "CROWD_TEMPORAL_TASKQUEUE=packages-worker CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=maven-worker tsx src/bin/maven-worker.ts",
     "backfill:maven:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && SERVICE=maven LOG_LEVEL=info tsx src/bin/maven-backfill.ts",
     "dev:maven-worker": "CROWD_TEMPORAL_TASKQUEUE=packages-worker CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=maven-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9236 src/bin/maven-worker.ts",
-    "dev:deps-dev-ingest": "CROWD_TEMPORAL_TASKQUEUE=deps-dev-ingest CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=deps-dev-ingest nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9235 src/bin/deps-dev-ingest.ts",
+    "dev:bq-dataset-ingest": "CROWD_TEMPORAL_TASKQUEUE=bq-dataset-ingest CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=bq-dataset-ingest nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9235 src/bin/bq-dataset-ingest.ts",
     "dev:npm-worker": "CROWD_TEMPORAL_TASKQUEUE=npm-worker CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=npm-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9236 src/bin/npm-worker.ts",
     "dev:osv-worker": "CROWD_TEMPORAL_TASKQUEUE=osv-worker CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=osv-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9238 src/bin/osv-worker.ts",
     "dev:github-repos-enricher": "SERVICE=github-repos-enricher LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9234 src/bin/github-repos-enricher.ts",
-    "dev:packages-worker:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && CROWD_TEMPORAL_TASKQUEUE=packages-worker CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=packages-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9233 src/bin/packages-worker.ts",
     "dev:criticality-worker:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && CROWD_TEMPORAL_TASKQUEUE=packages-worker CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=criticality-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9237 src/bin/criticality-worker.ts",
     "dev:maven-worker:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && CROWD_TEMPORAL_TASKQUEUE=packages-worker CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=maven-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9236 src/bin/maven-worker.ts",
-    "dev:deps-dev-ingest:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && CROWD_TEMPORAL_TASKQUEUE=deps-dev-ingest CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=deps-dev-ingest nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9235 src/bin/deps-dev-ingest.ts",
+    "dev:bq-dataset-ingest:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && CROWD_TEMPORAL_TASKQUEUE=bq-dataset-ingest CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=bq-dataset-ingest nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9235 src/bin/bq-dataset-ingest.ts",
     "dev:npm-worker:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && CROWD_TEMPORAL_TASKQUEUE=npm-worker CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=npm-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9236 src/bin/npm-worker.ts",
     "dev:osv-worker:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && CROWD_TEMPORAL_TASKQUEUE=osv-worker CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=osv-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9238 src/bin/osv-worker.ts",
     "start:maven": "SERVICE=maven tsx src/bin/maven.ts",
     "dev:maven": "SERVICE=maven LOG_LEVEL=info nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9235 src/bin/maven.ts",
     "dev:github-repos-enricher:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && SERVICE=github-repos-enricher LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9234 src/bin/github-repos-enricher.ts",
-    "export-to-bucket": "SERVICE=deps-dev-ingest tsx src/scripts/exportToBucket.ts",
-    "export-to-bucket:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && SERVICE=deps-dev-ingest tsx src/scripts/exportToBucket.ts",
+    "export-to-bucket": "SERVICE=bq-dataset-ingest tsx src/scripts/exportToBucket.ts",
+    "export-to-bucket:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && SERVICE=bq-dataset-ingest tsx src/scripts/exportToBucket.ts",
     "monitor:osspckgs:local": "bash -c 'set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && node ../../../scripts/monitor-osspckgs.mjs'",
     "lint": "npx eslint --ext .ts src --max-warnings=0",
     "format": "npx prettier --write \"src/**/*.ts\"",

services/apps/packages_worker/src/bin/maven-worker.ts

@@ -1,10 +1,6 @@
 import { scheduleMavenCritical } from '../maven/schedule'
 import { svc } from '../service'
 
-// Maven-only worker: runs on the shared `packages-worker` taskqueue (so it picks up
-// the same bundled workflows/activities) but registers ONLY the maven-critical
-// schedule. Intended for local dev — lets you run Maven in isolation without also
-// firing the npm/osv schedules that bin/packages-worker.ts registers.
 setImmediate(async () => {
   await svc.init()
   await scheduleMavenCritical()

services/apps/packages_worker/src/deps-dev/config.ts

@@ -10,6 +10,7 @@ function requireEnv(name: string): string {
 export const GCP_PROJECT = requireEnv('OSSPCKGS_GCP_PROJECT')
 export const GCS_BUCKET = requireEnv('OSSPCKGS_GCS_BUCKET')
 export const DEPS_DEV_DATASET = 'bigquery-public-data.deps_dev_v1'
+export const SCORECARD_DATASET = 'openssf.scorecardcron'
 
 // ADR-0003: Option A = DependencyGraphEdgesLatest (prod default, has version_constraint).
 // Set OSSPCKGS_DEPS_TABLE=B locally to use DependenciesLatest (cheaper, no version_constraint).

services/apps/packages_worker/src/deps-dev/schedules/bootstrap.ts

@@ -11,7 +11,7 @@ export async function scheduleOsspckgsBootstrap(): Promise<void> {
     await temporal.schedule.create({
       scheduleId: 'osspckgs-bootstrap-weekly',
       spec: {
-        cronExpressions: ['0 2 * * 0'],
+        cronExpressions: ['0 2 * * 1'],
       },
       policies: {
         overlap: ScheduleOverlapPolicy.SKIP,
@@ -20,7 +20,7 @@ export async function scheduleOsspckgsBootstrap(): Promise<void> {
       action: {
         type: 'startWorkflow',
         workflowType: bootstrapOsspckgs,
-        taskQueue: 'deps-dev-ingest',
+        taskQueue: 'bq-dataset-ingest',
         workflowExecutionTimeout: '12 hours',
         retry: {
           initialInterval: '1 minute',

services/apps/packages_worker/src/deps-dev/workflows/bootstrapOsspckgs.ts

@@ -5,6 +5,7 @@ import {
   workflowInfo,
 } from '@temporalio/workflow'
 
+import { ingestScorecard } from '../../scorecard/workflows'
 import type * as depsDevActivities from '../activities'
 
 import { ingestAdvisories } from './ingestAdvisories'
@@ -224,4 +225,9 @@ export async function bootstrapOsspckgs(opts: {
       ],
     })
   }
+  if (runs('scorecard')) {
+    await executeChild(ingestScorecard, {
+      args: [{ runId, reuseExports: opts.reuseExports, exportName: opts.exportName }],
+    })
+  }
 }

services/apps/packages_worker/src/schedules/cleanup.ts

@@ -20,7 +20,7 @@ export async function scheduleOsspckgsCleanup(): Promise<void> {
       action: {
         type: 'startWorkflow',
         workflowType: cleanupOsspckgs,
-        taskQueue: 'deps-dev-ingest',
+        taskQueue: 'bq-dataset-ingest',
         workflowExecutionTimeout: '1 hour',
         retry: {
           initialInterval: '1 minute',

services/apps/packages_worker/src/scorecard/queries/scorecardSql.ts

@@ -0,0 +1,21 @@
+import { SCORECARD_DATASET } from '../../deps-dev/config'
+
+export const SCORECARD_REPOS_SQL = `
+SELECT
+  LOWER(CONCAT('https://', repo.name)) AS repo_url,
+  score,
+  date AS scanned_at
+FROM \`${SCORECARD_DATASET}.scorecard-v2_latest\`
+WHERE repo.name IS NOT NULL
+`
+
+export const SCORECARD_CHECKS_SQL = `
+SELECT
+  LOWER(CONCAT('https://', r.repo.name)) AS repo_url,
+  c.name   AS check_name,
+  c.score  AS check_score,
+  c.reason AS check_reason
+FROM \`${SCORECARD_DATASET}.scorecard-v2_latest\` r,
+UNNEST(r.checks) AS c
+WHERE r.repo.name IS NOT NULL
+`

services/apps/packages_worker/src/scorecard/workflows/index.ts

@@ -0,0 +1 @@
+export { ingestScorecard } from './ingestScorecard'