v4.16.0 release to prod

.circleci/build-and-test/jobs.yml

@@ -77,7 +77,7 @@
           name: Setup cypress test data
           command: |
             cd tdrs-backend
-            docker-compose exec web python manage.py loaddata cypress/users cypress/data_files cypress/regions cypress/profile_editing_regions cypress/profile_editing_users
+            docker-compose exec web python manage.py loaddata cypress/users cypress/data_files cypress/regions cypress/profile_editing_regions cypress/profile_editing_users cypress/feature_flags
       - run:
           name: Run Cypress e2e tests
           command: cd tdrs-frontend; yarn test:e2e-ci

.github/ISSUE_TEMPLATE/bug-template.md

@@ -3,7 +3,7 @@ name: Bug Report template
 about: Template for bug reporting
 title: ''
 labels: bug, dev
-assignees: ''
+assignees: kennymcnett, reitermb, victoriaatraft, elipe17
 
 ---
 

.github/ISSUE_TEMPLATE/release-tracker-issue-template.md

@@ -39,6 +39,15 @@ https://github.com/raft-tech/TANF-app/releases/tag/vX.X.X
 ### 🧪 2. Staging Validation & QASP (ACF / Alex)
 *Tracking the status once ACF takes over deployment and testing.*
 
+### Before you Deploy
+- [ ] **Requires base image updates**: 
+  - [ ] Re-tag `ghcr.io/raft-tech/tdp-frontend-base:vX.X.X` for the HHS GHCR instance
+  - [ ] Re-tag `ghcr.io/raft-tech/tdp-backend-base:vX.X.X` for the HHS GHCR instance
+- [ ] **Requires HHS CircleCI config updates**: 
+  - [ ] 
+- [ ] **Requires PLG deployment**
+
+### Staging Deployment
 - [ ] **Staging Cleared:** Team notified that Staging is about to be updated/restarted.
 - [ ] **Deployed to Staging:** PR merged and deployed to the Staging environment.
 - [ ] **Feature Validation:** Testing instructions from the linked PRs have been executed and passed.

.gitignore

@@ -1,4 +1,5 @@
 # UI
+tdrs-frontend/.yarn/
 tdrs-frontend/node_modules/
 tdrs-frontend/build/
 tdrs-frontend/coverage/
@@ -125,4 +126,3 @@ cypress.env.json
 # DB seeds
 tdrs-backend/*.pg
 tdrs-backend/django.log
-

Taskfile.yml

@@ -294,7 +294,7 @@ tasks:
     cmds:
       - export CYPRESS_TOKEN=local-cypress-token
       - docker compose -f docker-compose.yml exec web python manage.py delete_cypress_users -usernames new-cypress@teamraft.com cypress-admin@teamraft.com cypress-data-analyst-dana@teamraft.com cypress-fra-data-analyst-derek@teamraft.com cypress-data-analyst-donna@teamraft.com cypress-fra-data-analyst-david@teamraft.com cypress-fra-ofa-regional-staff-rachel@acf.hhs.gov cypress-fra-ofa-regional-staff-robert@acf.hhs.gov cypress-fra-ofa-regional-staff-rita@acf.hhs.gov cypress-fra-ofa-regional-staff-ryan@acf.hhs.gov
-      - docker compose -f docker-compose.yml exec web python manage.py loaddata cypress/users cypress/data_files cypress/regions cypress/profile_editing_regions cypress/profile_editing_users
+      - docker compose -f docker-compose.yml exec web python manage.py loaddata cypress/users cypress/data_files cypress/regions cypress/profile_editing_regions cypress/profile_editing_users cypress/feature_flags
 
   frontend-e2e-local:
     desc: Run Cypress E2E tests locally (Cypress on host, app in docker)

docs/Technical-Documentation/tech-memos/submission-state-machine.md

@@ -0,0 +1,88 @@
+# Submission State Machine for File Processing
+
+## Purpose
+Define and enforce a clear lifecycle for uploaded files so parsing and triage share a consistent contract. This is a precursor to the parser refactor to avoid churn and make status handling predictable.
+
+## Why a state machine (and what it adds)
+- **Guardrails for future changes:** Even though end users cannot alter parsing, developers can. An explicit transition map prevents drift when we add steps (AV scan, retries, change requests) or touch the parser/reparser code paths. Instead of silently landing in an inconsistent state, we fail fast on illegal transitions.
+- **Durable, user-visible lifecycle:** `DataFileSummary` is per-parse and can be deleted/recreated during reparses. `DataFile.state` is a durable record of the submission lifecycle (upload -> scan -> parse) that survives reparses and exists even before a summary is created.
+- **Better triage and alerts:** Granular states (for example `virus_scan_started` vs `parse_started`) make it obvious where a file stalled without scraping logs. They enable targeted alerts (for example, "stuck in `parse_started` > 15m") and safer retries.
+
+## States
+`uploaded` -> `virus_scan_started` -> (`virus_scan_failed` | `virus_scan_successful`) -> `parse_started` -> (`parsed_with_errors` | `parsed_completed`) -> `completed`.
+
+Any active state can transition to `canceled`. A file that exceeds time thresholds in an active state is marked `stuck` (and may later be escalated to `failed` by policy).
+
+Note: parsing can write records in batches. This proposal does not model a separate ingest phase yet.
+
+## Allowed transitions (code sketch)
+```python
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Dict, Iterable
+
+
+class SubmissionState(str, Enum):
+    UPLOADED = "uploaded"
+    VIRUS_SCAN_STARTED = "virus_scan_started"
+    VIRUS_SCAN_FAILED = "virus_scan_failed"
+    VIRUS_SCAN_SUCCESSFUL = "virus_scan_successful"
+    PARSE_STARTED = "parse_started"
+    PARSE_COMPLETED = "parse_completed"
+    STUCK = "stuck"
+    COMPLETED = "completed"
+    CANCELED = "canceled"
+
+
+ALLOWED_TRANSITIONS: Dict[SubmissionState, Iterable[SubmissionState]] = {
+    SubmissionState.UPLOADED: {
+        SubmissionState.VIRUS_SCAN_STARTED,
+        SubmissionState.CANCELED,
+    },
+    SubmissionState.VIRUS_SCAN_STARTED: {
+        SubmissionState.VIRUS_SCAN_FAILED,
+        SubmissionState.VIRUS_SCAN_SUCCESSFUL,
+        SubmissionState.CANCELED,
+    },
+    SubmissionState.VIRUS_SCAN_FAILED: {
+        SubmissionState.CANCELED,
+    },
+    SubmissionState.VIRUS_SCAN_SUCCESSFUL: {
+        SubmissionState.PARSE_STARTED,
+        SubmissionState.CANCELED,
+    },
+    SubmissionState.PARSE_STARTED: {
+        SubmissionState.PARSED_WITH_ERRORS,
+        SubmissionState.PARSED_COMPLETED,
+        SubmissionState.CANCELED,
+    },
+    SubmissionState.PARSED_WITH_ERRORS: {
+        SubmissionState.COMPLETED,
+        SubmissionState.CANCELED,
+    },
+    SubmissionState.PARSED_COMPLETED: {
+        SubmissionState.COMPLETED,
+        SubmissionState.CANCELED,
+    },
+    SubmissionState.STUCK: {
+        SubmissionState.CANCELED,
+    },
+    SubmissionState.COMPLETED: set(),
+    SubmissionState.CANCELED: set(),
+}
+
+
+class InvalidTransition(Exception):
+    ...
+
+
+@dataclass
+class SubmissionLifecycle:
+    state: SubmissionState
+    history: list[str] = field(default_factory=list)
+
+    def transition(self, next_state: SubmissionState, note: str = "") -> None:
+        if next_state not in ALLOWED_TRANSITIONS[self.state]:
+            raise InvalidTransition(f"{self.state} -> {next_state} not allowed")
+        self.history.append(f"{self.state} -> {next_state}: {note}")
+        self.state = next_state

scripts/apply-database-config.sh

@@ -87,7 +87,7 @@ echo "Done."
 if [[ $app == "tdp-backend-develop" || $space == "tanf-dev" ]]; then
     echo "Applying e2e test data"
     python manage.py populate_stts
-    python manage.py loaddata cypress/users cypress/data_files cypress/regions cypress/profile_editing_regions cypress/profile_editing_users
+    python manage.py loaddata cypress/users cypress/data_files cypress/regions cypress/profile_editing_regions cypress/profile_editing_users cypress/feature_flags
     echo "Done."
 fi
 

tdrs-backend/plg/alertmanager/alertmanager.yml

@@ -39,19 +39,24 @@ route:
     - matchers:
         - alertname=~"UpTime"
       receiver: dev-team-emails
-      repeat_interval: 24h
+      repeat_interval: 48h
+      continue: true
     # Send all severity CRITICAL/ERROR alerts to OFA admin emails
     - matchers:
         - severity=~"ERROR|CRITICAL"
       receiver: admin-team-emails
       continue: true
     # Send all severity CRITICAL/ERROR/WARNING alerts to mattermost and dev team emails
     - matchers:
-        - severity=~"ERROR|CRITICAL|WARNING"
+        - severity=~"ERROR|CRITICAL"
       receiver: mattermost
       continue: true
     - matchers:
-        - severity=~"ERROR|CRITICAL|WARNING"
+        - severity=~"WARNING"
+      receiver: dev-mattermost
+      continue: true
+    - matchers:
+        - severity=~"ERROR|CRITICAL"
       receiver: dev-team-emails
       continue: true
 # Inhibition rules allow to mute a set of alerts given that another alert is
@@ -90,11 +95,29 @@ receivers:
           {{ if or (eq .Labels.severity "CRITICAL") (eq .Labels.severity "ERROR") }}
           @here
           {{ end }}
-          *Alert:* {{ .Annotations.title }}{{ if .Labels.severity }} - `{{ .Labels.severity }}`{{ end }}
+          *Alert:* {{ .Labels.alertname }}{{ if .Labels.severity }} - `{{ .Labels.severity }}`{{ end }}
+
+          *Description:* {{ .Annotations.description }}
+
+          *Details:*
+            • *Job:* `{{ .Labels.job }}`
+            • *Instance:* `{{ .Labels.instance }}`
+            • *Env:* `{{ .Labels.env }}`
+          {{ end }}
+  - name: 'dev-mattermost'
+    slack_configs:
+      - channel: 'tdp-dev-alerts'
+        username: 'alertmanager'
+        send_resolved: true
+        api_url: 'https://fake.mattermost.com'
+        text: |-
+          {{ range .Alerts -}}
+          *Alert:* {{ .Labels.alertname }}{{ if .Labels.severity }} - `{{ .Labels.severity }}`{{ end }}
 
           *Description:* {{ .Annotations.description }}
 
           *Details:*
-            {{ range .Labels.SortedPairs }} • *{{ .Name }}:* `{{ .Value }}`
-            {{ end }}
+            • *Job:* `{{ .Labels.job }}`
+            • *Instance:* `{{ .Labels.instance }}`
+            • *Env:* `{{ .Labels.env }}`
           {{ end }}

tdrs-backend/plg/deploy.sh

@@ -105,6 +105,7 @@ deploy_alertmanager() {
     yq eval -i ".global.slack_api_url = \"$MATTERMOST_WEBHOOK_URL\"" $CONFIG
     yq eval -i ".receivers[0].email_configs[0].to = \"${ADMIN_EMAILS}\"" $CONFIG
     yq eval -i ".receivers[1].email_configs[0].to = \"${DEV_EMAILS}\"" $CONFIG
+    yq eval -i ".receivers[3].slack_configs[0].api_url = \"${DEV_MATTERMOST_WEBHOOK_URL}\"" $CONFIG
     cf push --no-route -f manifest.yml -t 180  --strategy rolling
     cf map-route alertmanager apps.internal --hostname alertmanager
     rm $CONFIG