Stage 3: deploy FastAPI shell as Cloud Run candidate

.github/scripts/build_cloud_run_image.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .github/scripts/cloud_run_env.sh
+cloud_run_set_defaults
+
+if [[ "${CLOUD_RUN_DRY_RUN:-0}" == "1" ]]; then
+  cloud_run_run gcloud services enable run.googleapis.com artifactregistry.googleapis.com cloudbuild.googleapis.com --project "${CLOUD_RUN_PROJECT}"
+  cloud_run_run gcloud artifacts repositories describe "${CLOUD_RUN_ARTIFACT_REPOSITORY}" --project "${CLOUD_RUN_PROJECT}" --location "${CLOUD_RUN_REGION}"
+  cloud_run_run gcloud artifacts repositories create "${CLOUD_RUN_ARTIFACT_REPOSITORY}" --repository-format docker --location "${CLOUD_RUN_REGION}" --description "Docker repository for PolicyEngine API Cloud Run"
+  cloud_run_run gcloud auth configure-docker "${CLOUD_RUN_REGION}-docker.pkg.dev" --quiet
+  cloud_run_run docker build -f gcp/cloud_run/Dockerfile -t "${CLOUD_RUN_IMAGE_URI}" .
+  cloud_run_run docker push "${CLOUD_RUN_IMAGE_URI}"
+  exit 0
+fi
+
+gcloud services enable run.googleapis.com artifactregistry.googleapis.com cloudbuild.googleapis.com --project "${CLOUD_RUN_PROJECT}"
+
+if ! gcloud artifacts repositories describe "${CLOUD_RUN_ARTIFACT_REPOSITORY}" \
+  --project "${CLOUD_RUN_PROJECT}" \
+  --location "${CLOUD_RUN_REGION}" >/dev/null 2>&1; then
+  gcloud artifacts repositories create "${CLOUD_RUN_ARTIFACT_REPOSITORY}" \
+    --repository-format docker \
+    --location "${CLOUD_RUN_REGION}" \
+    --description "Docker repository for PolicyEngine API Cloud Run"
+fi
+
+gcloud auth configure-docker "${CLOUD_RUN_REGION}-docker.pkg.dev" --quiet
+docker build -f gcp/cloud_run/Dockerfile -t "${CLOUD_RUN_IMAGE_URI}" .
+docker push "${CLOUD_RUN_IMAGE_URI}"

.github/scripts/check_changelog_fragment.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+fragments="$(find changelog.d -type f ! -name '.gitkeep' | wc -l)"
+if [[ "${fragments}" -eq 0 ]]; then
+  echo "::error::No changelog fragment found in changelog.d/"
+  echo "Add one with: echo 'Description.' > changelog.d/\$(git branch --show-current).<type>.md"
+  echo "Types: added, changed, fixed, removed, breaking"
+  exit 1
+fi

.github/scripts/cloud_run_env.sh

@@ -0,0 +1,78 @@
+#!/usr/bin/env bash
+
+cloud_run_set_defaults() {
+  CLOUD_RUN_PROJECT="${CLOUD_RUN_PROJECT:-policyengine-api}"
+  CLOUD_RUN_REGION="${CLOUD_RUN_REGION:-us-central1}"
+  CLOUD_RUN_SERVICE="${CLOUD_RUN_SERVICE:-policyengine-api}"
+  CLOUD_RUN_ARTIFACT_REPOSITORY="${CLOUD_RUN_ARTIFACT_REPOSITORY:-policyengine-api}"
+  CLOUD_RUN_RUNTIME_SERVICE_ACCOUNT="${CLOUD_RUN_RUNTIME_SERVICE_ACCOUNT:-policyengine-api-cr-runtime@policyengine-api.iam.gserviceaccount.com}"
+  CLOUD_RUN_CLOUD_SQL_INSTANCE="${CLOUD_RUN_CLOUD_SQL_INSTANCE:-policyengine-api:us-central1:policyengine-api-data}"
+  CLOUD_RUN_CPU="${CLOUD_RUN_CPU:-4}"
+  CLOUD_RUN_MEMORY="${CLOUD_RUN_MEMORY:-16Gi}"
+  CLOUD_RUN_TIMEOUT="${CLOUD_RUN_TIMEOUT:-300}"
+  CLOUD_RUN_MIN_INSTANCES="${CLOUD_RUN_MIN_INSTANCES:-0}"
+  CLOUD_RUN_MAX_INSTANCES="${CLOUD_RUN_MAX_INSTANCES:-1}"
+  CLOUD_RUN_PORT="${CLOUD_RUN_PORT:-8080}"
+  CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET="${CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET:-policyengine-api-prod-db-password:latest}"
+  CLOUD_RUN_GITHUB_MICRODATA_TOKEN_SECRET="${CLOUD_RUN_GITHUB_MICRODATA_TOKEN_SECRET:-policyengine-api-prod-github-microdata-token:latest}"
+  CLOUD_RUN_ANTHROPIC_API_KEY_SECRET="${CLOUD_RUN_ANTHROPIC_API_KEY_SECRET:-policyengine-api-prod-anthropic-api-key:latest}"
+  CLOUD_RUN_OPENAI_API_KEY_SECRET="${CLOUD_RUN_OPENAI_API_KEY_SECRET:-policyengine-api-prod-openai-api-key:latest}"
+  CLOUD_RUN_HUGGING_FACE_TOKEN_SECRET="${CLOUD_RUN_HUGGING_FACE_TOKEN_SECRET:-policyengine-api-prod-hugging-face-token:latest}"
+
+  local sha
+  sha="${GITHUB_SHA:-local}"
+  CLOUD_RUN_IMAGE_TAG="${CLOUD_RUN_IMAGE_TAG:-${sha}}"
+  CLOUD_RUN_IMAGE_URI="${CLOUD_RUN_IMAGE_URI:-${CLOUD_RUN_REGION}-docker.pkg.dev/${CLOUD_RUN_PROJECT}/${CLOUD_RUN_ARTIFACT_REPOSITORY}/${CLOUD_RUN_SERVICE}:${CLOUD_RUN_IMAGE_TAG}}"
+
+  local short_sha
+  short_sha="${sha:0:7}"
+  CLOUD_RUN_TAG="${CLOUD_RUN_TAG:-stage3-${GITHUB_RUN_NUMBER:-local}-${short_sha}}"
+
+  export CLOUD_RUN_PROJECT
+  export CLOUD_RUN_REGION
+  export CLOUD_RUN_SERVICE
+  export CLOUD_RUN_ARTIFACT_REPOSITORY
+  export CLOUD_RUN_RUNTIME_SERVICE_ACCOUNT
+  export CLOUD_RUN_CLOUD_SQL_INSTANCE
+  export CLOUD_RUN_CPU
+  export CLOUD_RUN_MEMORY
+  export CLOUD_RUN_TIMEOUT
+  export CLOUD_RUN_MIN_INSTANCES
+  export CLOUD_RUN_MAX_INSTANCES
+  export CLOUD_RUN_PORT
+  export CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET
+  export CLOUD_RUN_GITHUB_MICRODATA_TOKEN_SECRET
+  export CLOUD_RUN_ANTHROPIC_API_KEY_SECRET
+  export CLOUD_RUN_OPENAI_API_KEY_SECRET
+  export CLOUD_RUN_HUGGING_FACE_TOKEN_SECRET
+  export CLOUD_RUN_IMAGE_TAG
+  export CLOUD_RUN_IMAGE_URI
+  export CLOUD_RUN_TAG
+}
+
+cloud_run_require_env() {
+  local missing=()
+  local name
+
+  for name in "$@"; do
+    if [[ -z "${!name:-}" ]]; then
+      missing+=("${name}")
+    fi
+  done
+
+  if (( ${#missing[@]} > 0 )); then
+    echo "Missing required Cloud Run deployment configuration: ${missing[*]}" >&2
+    return 1
+  fi
+}
+
+cloud_run_run() {
+  if [[ "${CLOUD_RUN_DRY_RUN:-0}" == "1" ]]; then
+    printf '+'
+    printf ' %q' "$@"
+    printf '\n'
+    return 0
+  fi
+
+  "$@"
+}

.github/scripts/deploy_cloud_run_candidate.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .github/scripts/cloud_run_env.sh
+cloud_run_set_defaults
+
+bash .github/scripts/validate_cloud_run_deploy_env.sh
+
+env_vars=(
+  "POLICYENGINE_DB_INSTANCE_CONNECTION_NAME=${CLOUD_RUN_CLOUD_SQL_INSTANCE}"
+  "POLICYENGINE_DB_USER=${POLICYENGINE_DB_USER:-policyengine}"
+  "POLICYENGINE_DB_NAME=${POLICYENGINE_DB_NAME:-policyengine}"
+  "SIMULATION_API_URL=${SIMULATION_API_URL}"
+  "GATEWAY_AUTH_REQUIRED=1"
+  "GATEWAY_AUTH_ISSUER=${GATEWAY_AUTH_ISSUER}"
+  "GATEWAY_AUTH_AUDIENCE=${GATEWAY_AUTH_AUDIENCE}"
+  "GATEWAY_AUTH_CLIENT_ID=${GATEWAY_AUTH_CLIENT_ID}"
+  "GATEWAY_AUTH_CLIENT_SECRET_RESOURCE=${GATEWAY_AUTH_CLIENT_SECRET_RESOURCE}"
+  "API_HOST_BACKEND=cloud_run"
+  "SIM_FRONT_DOOR=old_gateway_direct"
+  "SIM_COMPUTE_ECONOMY=old_gateway"
+  "CLOUD_RUN_REVISION_TAG=${CLOUD_RUN_TAG}"
+)
+
+secret_vars=(
+  "POLICYENGINE_DB_PASSWORD=${CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET}"
+  "POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN=${CLOUD_RUN_GITHUB_MICRODATA_TOKEN_SECRET}"
+  "ANTHROPIC_API_KEY=${CLOUD_RUN_ANTHROPIC_API_KEY_SECRET}"
+  "OPENAI_API_KEY=${CLOUD_RUN_OPENAI_API_KEY_SECRET}"
+  "HUGGING_FACE_TOKEN=${CLOUD_RUN_HUGGING_FACE_TOKEN_SECRET}"
+)
+
+set_env_vars="$(IFS='|'; echo "^|^${env_vars[*]}")"
+set_secret_vars="$(IFS='|'; echo "^|^${secret_vars[*]}")"
+
+cloud_run_run gcloud run deploy "${CLOUD_RUN_SERVICE}" \
+  --project "${CLOUD_RUN_PROJECT}" \
+  --region "${CLOUD_RUN_REGION}" \
+  --platform managed \
+  --image "${CLOUD_RUN_IMAGE_URI}" \
+  --tag "${CLOUD_RUN_TAG}" \
+  --no-traffic \
+  --allow-unauthenticated \
+  --execution-environment gen2 \
+  --service-account "${CLOUD_RUN_RUNTIME_SERVICE_ACCOUNT}" \
+  --add-cloudsql-instances "${CLOUD_RUN_CLOUD_SQL_INSTANCE}" \
+  --port "${CLOUD_RUN_PORT}" \
+  --cpu "${CLOUD_RUN_CPU}" \
+  --memory "${CLOUD_RUN_MEMORY}" \
+  --timeout "${CLOUD_RUN_TIMEOUT}" \
+  --min-instances "${CLOUD_RUN_MIN_INSTANCES}" \
+  --max-instances "${CLOUD_RUN_MAX_INSTANCES}" \
+  --set-env-vars "${set_env_vars}" \
+  --set-secrets "${set_secret_vars}"

.github/scripts/get_cloud_run_service_url.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .github/scripts/cloud_run_env.sh
+cloud_run_set_defaults
+
+if [[ "${CLOUD_RUN_DRY_RUN:-0}" == "1" ]]; then
+  echo "https://${CLOUD_RUN_SERVICE}-dry-run.a.run.app"
+  exit 0
+fi
+
+gcloud run services describe "${CLOUD_RUN_SERVICE}" \
+  --project "${CLOUD_RUN_PROJECT}" \
+  --region "${CLOUD_RUN_REGION}" \
+  --platform managed \
+  --format 'value(status.url)'

.github/scripts/get_cloud_run_tag_url.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .github/scripts/cloud_run_env.sh
+cloud_run_set_defaults
+
+if [[ "${CLOUD_RUN_DRY_RUN:-0}" == "1" ]]; then
+  echo "https://${CLOUD_RUN_TAG}---${CLOUD_RUN_SERVICE}-dry-run.a.run.app"
+  exit 0
+fi
+
+gcloud run services describe "${CLOUD_RUN_SERVICE}" \
+  --project "${CLOUD_RUN_PROJECT}" \
+  --region "${CLOUD_RUN_REGION}" \
+  --platform managed \
+  --format json | python -c '
+import json
+import os
+import sys
+
+service = json.load(sys.stdin)
+tag = os.environ["CLOUD_RUN_TAG"]
+for traffic_target in service.get("status", {}).get("traffic", []):
+    if traffic_target.get("tag") == tag and traffic_target.get("url"):
+        print(traffic_target["url"])
+        raise SystemExit(0)
+
+print(f"Failed to determine Cloud Run URL for tag {tag}", file=sys.stderr)
+raise SystemExit(1)
+'

.github/scripts/promote_cloud_run_tag.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .github/scripts/cloud_run_env.sh
+cloud_run_set_defaults
+
+cloud_run_require_env \
+  CLOUD_RUN_PROJECT \
+  CLOUD_RUN_REGION \
+  CLOUD_RUN_SERVICE \
+  CLOUD_RUN_TAG
+
+cloud_run_run gcloud run services update-traffic "${CLOUD_RUN_SERVICE}" \
+  --project "${CLOUD_RUN_PROJECT}" \
+  --region "${CLOUD_RUN_REGION}" \
+  --platform managed \
+  --to-tags "${CLOUD_RUN_TAG}=100"

.github/scripts/sync_cloud_run_secrets.sh

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+set +x
+
+CLOUD_RUN_PROJECT="${CLOUD_RUN_PROJECT:-policyengine-api}"
+
+require_env() {
+  local env_name="$1"
+  if [[ -z "${!env_name:-}" ]]; then
+    echo "::error::Missing required workflow environment ${env_name}."
+    exit 1
+  fi
+}
+
+sync_secret() {
+  local env_name="$1"
+  local secret_name="$2"
+  local secret_value="${!env_name:-}"
+
+  if [[ -z "${secret_value}" ]]; then
+    echo "::error::Missing required GitHub secret ${env_name}."
+    exit 1
+  fi
+
+  if ! gcloud secrets describe "${secret_name}" \
+    --project "${CLOUD_RUN_PROJECT}" >/dev/null 2>&1; then
+    gcloud secrets create "${secret_name}" \
+      --project "${CLOUD_RUN_PROJECT}" \
+      --replication-policy automatic
+  fi
+
+  printf '%s' "${secret_value}" | gcloud secrets versions add \
+    "${secret_name}" \
+    --project "${CLOUD_RUN_PROJECT}" \
+    --data-file=- >/dev/null
+
+  gcloud secrets add-iam-policy-binding "${secret_name}" \
+    --project "${CLOUD_RUN_PROJECT}" \
+    --member "serviceAccount:${CLOUD_RUN_RUNTIME_SERVICE_ACCOUNT}" \
+    --role roles/secretmanager.secretAccessor >/dev/null
+
+  echo "Synced ${env_name} to Secret Manager secret ${secret_name}."
+}
+
+require_env CLOUD_RUN_RUNTIME_SERVICE_ACCOUNT
+
+sync_secret POLICYENGINE_DB_PASSWORD policyengine-api-prod-db-password
+sync_secret POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN policyengine-api-prod-github-microdata-token
+sync_secret ANTHROPIC_API_KEY policyengine-api-prod-anthropic-api-key
+sync_secret OPENAI_API_KEY policyengine-api-prod-openai-api-key
+sync_secret HUGGING_FACE_TOKEN policyengine-api-prod-hugging-face-token

.github/scripts/validate_cloud_run_deploy_env.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .github/scripts/cloud_run_env.sh
+cloud_run_set_defaults
+
+cloud_run_require_env \
+  CLOUD_RUN_PROJECT \
+  CLOUD_RUN_REGION \
+  CLOUD_RUN_SERVICE \
+  CLOUD_RUN_ARTIFACT_REPOSITORY \
+  CLOUD_RUN_IMAGE_URI \
+  CLOUD_RUN_TAG \
+  CLOUD_RUN_RUNTIME_SERVICE_ACCOUNT \
+  CLOUD_RUN_CLOUD_SQL_INSTANCE \
+  CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET \
+  CLOUD_RUN_GITHUB_MICRODATA_TOKEN_SECRET \
+  CLOUD_RUN_ANTHROPIC_API_KEY_SECRET \
+  CLOUD_RUN_OPENAI_API_KEY_SECRET \
+  CLOUD_RUN_HUGGING_FACE_TOKEN_SECRET \
+  SIMULATION_API_URL \
+  GATEWAY_AUTH_ISSUER \
+  GATEWAY_AUTH_AUDIENCE \
+  GATEWAY_AUTH_CLIENT_ID \
+  GATEWAY_AUTH_CLIENT_SECRET_RESOURCE

.github/workflows/pr.yml

@@ -36,14 +36,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Check for changelog fragment
-        run: |
-          FRAGMENTS=$(find changelog.d -type f ! -name '.gitkeep' | wc -l)
-          if [ "$FRAGMENTS" -eq 0 ]; then
-            echo "::error::No changelog fragment found in changelog.d/"
-            echo "Add one with: echo 'Description.' > changelog.d/\$(git branch --show-current).<type>.md"
-            echo "Types: added, changed, fixed, removed, breaking"
-            exit 1
-          fi
+        run: bash .github/scripts/check_changelog_fragment.sh
   test_container_builds:
     name: Docker
     runs-on: ubuntu-latest
@@ -61,6 +54,16 @@ jobs:
           password: ${{ secrets.GITHUB_TOKEN }}
       - name: Build container
         run: docker build -t ghcr.io/policyengine/policyengine docker
+  test_cloud_run_container_builds:
+    name: Cloud Run container
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v4
+      - name: Build Cloud Run container
+        run: docker build -f gcp/cloud_run/Dockerfile -t policyengine-api-cloud-run:test .
   test_env_vars:
     name: Test environment variables
     runs-on: ubuntu-latest