diff --git a/.circleci/build-and-test/jobs.yml b/.circleci/build-and-test/jobs.yml index 4b9e2c796..6105bc2ab 100644 --- a/.circleci/build-and-test/jobs.yml +++ b/.circleci/build-and-test/jobs.yml @@ -77,7 +77,7 @@ name: Setup cypress test data command: | cd tdrs-backend - docker-compose exec web python manage.py loaddata cypress/users cypress/data_files cypress/regions cypress/profile_editing_regions cypress/profile_editing_users + docker-compose exec web python manage.py loaddata cypress/users cypress/data_files cypress/regions cypress/profile_editing_regions cypress/profile_editing_users cypress/feature_flags - run: name: Run Cypress e2e tests command: cd tdrs-frontend; yarn test:e2e-ci diff --git a/.github/ISSUE_TEMPLATE/bug-template.md b/.github/ISSUE_TEMPLATE/bug-template.md index 0b371534d..019d7d52a 100644 --- a/.github/ISSUE_TEMPLATE/bug-template.md +++ b/.github/ISSUE_TEMPLATE/bug-template.md @@ -3,7 +3,7 @@ name: Bug Report template about: Template for bug reporting title: '' labels: bug, dev -assignees: '' +assignees: kennymcnett, reitermb, victoriaatraft, elipe17 --- diff --git a/.github/ISSUE_TEMPLATE/release-tracker-issue-template.md b/.github/ISSUE_TEMPLATE/release-tracker-issue-template.md index e912da957..8243289ed 100644 --- a/.github/ISSUE_TEMPLATE/release-tracker-issue-template.md +++ b/.github/ISSUE_TEMPLATE/release-tracker-issue-template.md @@ -39,6 +39,15 @@ https://github.com/raft-tech/TANF-app/releases/tag/vX.X.X ### 🧪 2. Staging Validation & QASP (ACF / Alex) *Tracking the status once ACF takes over deployment and testing.* +### Before you Deploy +- [ ] **Requires base image updates**: + - [ ] Re-tag `ghcr.io/raft-tech/tdp-frontend-base:vX.X.X` for the HHS GHCR instance + - [ ] Re-tag `ghcr.io/raft-tech/tdp-backend-base:vX.X.X` for the HHS GHCR instance +- [ ] **Requires HHS CircleCI config updates**: + - [ ] +- [ ] **Requires PLG deployment** + +### Staging Deployment - [ ] **Staging Cleared:** Team notified that Staging is about to be updated/restarted. - [ ] **Deployed to Staging:** PR merged and deployed to the Staging environment. - [ ] **Feature Validation:** Testing instructions from the linked PRs have been executed and passed. diff --git a/.gitignore b/.gitignore index 447283e14..d80201e44 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ # UI +tdrs-frontend/.yarn/ tdrs-frontend/node_modules/ tdrs-frontend/build/ tdrs-frontend/coverage/ @@ -125,4 +126,3 @@ cypress.env.json # DB seeds tdrs-backend/*.pg tdrs-backend/django.log - diff --git a/Taskfile.yml b/Taskfile.yml index 99c9a216c..16855ea9f 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -294,7 +294,7 @@ tasks: cmds: - export CYPRESS_TOKEN=local-cypress-token - docker compose -f docker-compose.yml exec web python manage.py delete_cypress_users -usernames new-cypress@teamraft.com cypress-admin@teamraft.com cypress-data-analyst-dana@teamraft.com cypress-fra-data-analyst-derek@teamraft.com cypress-data-analyst-donna@teamraft.com cypress-fra-data-analyst-david@teamraft.com cypress-fra-ofa-regional-staff-rachel@acf.hhs.gov cypress-fra-ofa-regional-staff-robert@acf.hhs.gov cypress-fra-ofa-regional-staff-rita@acf.hhs.gov cypress-fra-ofa-regional-staff-ryan@acf.hhs.gov - - docker compose -f docker-compose.yml exec web python manage.py loaddata cypress/users cypress/data_files cypress/regions cypress/profile_editing_regions cypress/profile_editing_users + - docker compose -f docker-compose.yml exec web python manage.py loaddata cypress/users cypress/data_files cypress/regions cypress/profile_editing_regions cypress/profile_editing_users cypress/feature_flags frontend-e2e-local: desc: Run Cypress E2E tests locally (Cypress on host, app in docker) diff --git a/docs/Technical-Documentation/tech-memos/parsing-refactor-plan.md b/docs/Technical-Documentation/tech-memos/parsing-refactor-plan.md new file mode 100644 index 000000000..1ef32846a --- /dev/null +++ b/docs/Technical-Documentation/tech-memos/parsing-refactor-plan.md @@ -0,0 +1,304 @@ +# Technical Memo: Refactoring Parsing and Reparsing in TANF Data Portal Backend + +## 1. Purpose + +This memo proposes a refactor of the **parsing** and **reparsing** pipelines in the TANF Data Portal backend (`tdrs-backend`). The goal is to: + +- Reduce duplicated orchestration logic between initial parsing and reparsing +- Make parsing behavior easier to reason about, test, and extend +- Improve performance and observability for large-scale reparsing operations +- Clearly separate “what to parse” (selection) from “how to parse” (pipeline) +- Establish a stable contract (service + state machine) so new file types and policy changes do not require touching Celery tasks or ad hoc utilities. + +### Why this refactor is beneficial +- **Single source of truth for parsing logic:** Today, behavior is split across parser classes, the Celery task, and reparse utilities. Moving to a ParsingService collapses side effects (status updates, summaries, error reports) into one place, reducing drift and regression risk. +- **Testability:** A service with clear inputs/outputs can be unit-tested without Celery, making it easier to cover both happy paths and failure modes. Reparsing can reuse the same entry point with explicit context. +- **Extensibility:** A factory-driven, class-based parser plus decoder abstraction makes it straightforward to add file types (for example, new CSV/XLSX variants) or program types without rewriting orchestration. SchemaManager and decoders become the main extension points. +- **Operational clarity:** With the submission state machine and centralized transitions, operators and users see consistent states (for example, uploaded, virus_scan_started, parse_started, parsed_with_errors, parsed_completed, completed) instead of implicit flags scattered across models. +- **Safer reparsing:** Consolidating reparse behavior (backups, deletions, status updates) through a dedicated reparse service and shared state transitions improves idempotency, makes resume/rollback safer, and keeps ReparseMeta/ReparseFileMeta in sync. +- **Observability:** Centralized logging and optional metrics around a single service boundary make it easier to trace a file journey, correlate errors, and measure performance (parse durations, error counts). +The recommendations are based on the current implementation in: + +- `tdpservice/parsers/…` +- `tdpservice/scheduling/parser_task.py` +- `tdpservice/search_indexes/reparse.py` +- `tdpservice/search_indexes/utils.py` +- `tdpservice/search_indexes/models/reparse_meta.py` +- `tdpservice/data_files/models.py` (especially `DataFile` and `ReparseFileMeta`) + + +## 2. Current Architecture (High-Level) + +### 2.1 Initial parsing flow + +**Entry point:** a TANF/SSP/TRIBAL/FRA data file is uploaded (`DataFile` instance created). + +**Core components:** + +- **Celery task:** `tdpservice/scheduling/parser_task.parse_data_file` + - Looks up the `DataFile` + - Uses `ParserFactory` to get the correct parser class for the file’s program type + - Calls parser methods (e.g., `parse_and_validate()`) + - Updates `DataFileSummary` / status flags + - Generates error reports via `ErrorReportFactory` + - Sends notification emails (`send_data_submitted_email`) + - If the parse is part of a reparse run, it also updates `ReparseFileMeta` + +- **Parser infrastructure:** + - `tdpservice/parsers/factory.ParserFactory` + - `tdpservice/parsers/parser_classes/base_parser.BaseParser` + - Concrete parsers: + - `TanfDataReportParser` + - `FRAParser` + - `ProgramAuditParser` + - `SchemaManager` (`schema_manager.py`) to manage program- and section-specific schema + - `ErrorGeneratorFactory` and `ParserError` to generate and persist row-level errors + - `DataFileSummary` to track high-level outcomes (record counts, error counts) + +**Characteristics:** +- “Initial parse” logic is **implicitly defined** by the behavior inside `parse_data_file` and the individual parser classes. +- The Celery task contains non-trivial orchestration logic: logging, error handling, and special cases for reparse runs. + + +### 2.2 Reparsing flow + +Reparsing is used to “clean and reprocess” existing data files, usually when schemas or validation logic change. + +**Core components:** + +- **Reparse orchestration:** `tdpservice/search_indexes/reparse.py` + - User / admin triggers some “clean and reparse” behavior (fiscal year, quarter, optional filters) + - Uses helpers from `tdpservice/search_indexes/utils.py` and `tdpservice/search_indexes/util.py`: + - `backup(...)` → creates a DB backup + - `delete_associated_models(...)` → deletes records associated with the selected files (ParserError, DataFileSummary, index rows, etc.) + - `calculate_timeout(...)` / `assert_sequential_execution(...)` → safety checks for long-running reparses + - `count_total_num_records(...)` / `count_all_records(...)` → record-count snapshots + +- **Meta tracking models:** + - `ReparseMeta` (`tdpservice/search_indexes/models/reparse_meta.py`) + - Represents a single “reparse run” (with fields like `timeout_at`, `finished`, `success`, `total_num_records_initial`, `total_num_records_after`, etc.) + - Aggregates related `ReparseFileMeta` records + - `ReparseFileMeta` (`tdpservice/data_files/models.py`) + - Represents the parse status of a single `DataFile` within a reparse run (finished, success, record counts, error counts) + +- **Scheduling reparses:** + - Once backup and deletion are done, `reparse.py` calls into `parser_task` for each `DataFile` to schedule Celery tasks for reparsing. + - The same `parse_data_file` task is used, but with additional `reparse_id` context that ties the parse to a `ReparseMeta` / `ReparseFileMeta` pair. + +**Characteristics:** +- Reparsing logic is spread across: + - `search_indexes/reparse.py` + - `search_indexes/utils.py` + - `parser_task.parse_data_file` + - The parsers and error-reporting logic +- `ReparseMeta` / `ReparseFileMeta` add an extra dimension of lifecycle and state, but much of the behavior to maintain them lives in the Celery task. + + +## 3. Pain Points & Risks + +From a maintainability and performance perspective, several issues stand out: + +### 3.1 Duplicated orchestration between “parse” and “reparse” + +- The **initial parsing path** and the **reparsing path** both: + - Determine which files to parse + - Manage database state (deleting old records, updating summaries) + - Schedule Celery tasks + - Generate logs and metrics +- However, the logic to do this is split across multiple modules, and the reparsing path has its own backup / cleanup orchestration. +- When schemas or error-handling rules change, engineers must remember to update both flows, which increases the risk of subtle inconsistencies. + +### 3.2 Tight coupling to Celery and logging + +- `parse_data_file` is both a Celery task and a “business service”: + - It contains domain logic (how we parse, validate, update summaries, etc.) + - It also contains infrastructure concerns (Celery wiring, logging, email notifications, file rotation). +- This makes unit testing harder and encourages direct calls to the Celery task instead of to a clear, reusable parsing service. + +### 3.3 Reparsing logic is scattered and not obviously idempotent + +- `search_indexes/reparse.py` performs several responsibilities: + - Safety checks (sequential execution, timeouts) + - Backup orchestration + - Bulk deletion of associated records + - Scheduling reparse tasks for each `DataFile` +- Much of this logic operates directly on the DB and logging functions, which makes it harder to reason about/retry safely. +- While `ReparseMeta` / `ReparseFileMeta` provide state tracking, the actual transitions are implemented in different modules, making it non-obvious how to safely resume or inspect a partially finished reparse run. + +### 3.4 Performance & operational concerns on large datasets + +- `count_total_num_records(...)`, `count_all_records(...)`, and bulk deletions may become expensive as datasets grow. +- Parsing and reparsing touch several related tables (DataFile, ParserError, DataFileSummary, index tables, etc.), so the order and batching of operations matters for performance and locking behavior. +- Today, these concerns are embedded in the reparse utilities and Celery task without a single place to tune or monitor the pipeline behavior. + + +## 4. Proposed Refactor: Single-File Parsing Service + Reparse Orchestration Service + +The core idea is to have a **single-file parsing service** that owns all parsing side effects for one `DataFile`, and a **reparse orchestration service** that manages reparse runs and delegates per-file work to the parsing service. This keeps SRP and makes the hierarchy explicit: + +- The Celery task (`parse_data_file`) only wires arguments/logging and calls the parsing service. +- The reparse service manages `ReparseMeta` / `ReparseFileMeta` lifecycle and calls the parsing service for each file in a reparse run. +- No other caller should reach directly into parser classes or touch reparse metadata. + + +### 4.1 Introduce a `ParsingService` (single file only) + +Create a new module, for example: + +- `tdpservice/parsing/service.py` or +- `tdpservice/parsers/service.py` + +with a class like: + +```python +class ParsingService: + def __init__(self, *, logger, now_fn=timezone.now): + self.logger = logger + self.now_fn = now_fn + + def parse_data_file(self, data_file_id: int) -> DataFileSummary: + """ + Orchestrate the full lifecycle for parsing a single DataFile. + + - Load DataFile and related metadata + - Select parser via ParserFactory + - Invoke parser.parse_and_validate() + - Update DataFileSummary / DataFile status + - Generate error report + - Return the refreshed DataFileSummary (no awareness of reparse metadata) + """ +``` + +This service should encapsulate **what it means** to fully process a `DataFile`, regardless of why it is being parsed (initial submit or reparse). + + +### 4.2 Make Celery task a thin wrapper around the service + +Refactor `tdpservice/scheduling/parser_task.parse_data_file` to: + +- Parse out its Celery-specific concerns (arguments, retries, logging context) +- Delegate the core work to `ParsingService.parse_data_file` + +Example (conceptually): + +```python +@shared_task(bind=True) +def parse_data_file(self, data_file_id: int, reparse_id: int | None = None): + logger = get_task_logger(__name__) + service = ParsingService(logger=logger) + service.parse_data_file(data_file_id) +``` + +This keeps Celery wiring and logging but moves domain logic into the service. + + +### 4.3 Introduce a `ReparseService` (orchestration + metadata) + +Refactor `tdpservice/search_indexes/reparse.py` and `search_indexes/utils.py` so that they: + +1. Determine **which DataFiles** should be reparsed (by fiscal year, quarter, program type, STT, etc.). +2. Perform backup operations (if needed). +3. Clean out associated records (ParserError, summaries, index rows) in a coherent, batched way. +4. For each file, create/update `ReparseFileMeta`, then invoke `ParsingService.parse_data_file(...)`. +5. Aggregate per-file outcomes back into `ReparseMeta` (finished, success, counts). + +`ReparseService` owns all `ReparseMeta` / `ReparseFileMeta` transitions; `ParsingService` stays focused on parsing a single file. + +This makes state transitions explicit and easier to test, and avoids scattering them between the Celery task and reparse utilities. + + +### 4.5 Improve testability & observability + +Once parsing and reparsing are routed through a single service class: +- **Unit tests** can exercise `ParsingService` directly using a small in-memory or test DB dataset. +- **Integration tests** can cover: + - Initial parse of a sample file + - Reparse run across a few files + - Recovery from a mid-run failure +- Logging can be standardized around a single logger/context, making it easier to trace parsing results in logs or APM tooling. + + +## 5. Suggested Implementation Phases + +To de-risk the refactor, implement in small, incremental steps: + + +### Phase 1 – Document & stabilize current behavior + +- Capture current parsing and reparsing flows in sequence diagrams or text: + - How `parse_data_file` is called + - Which tables are touched and in what order + - How ReparseMeta / ReparseFileMeta are created and updated +- Add any missing indexes or small performance improvements that are clearly safe (see separate performance tickets). + + +### Phase 2 – Introduce `ParsingService` without changing behavior + +- Extract the body of `parse_data_file` into `ParsingService.parse_data_file`, keeping behavior identical. +- The Celery task delegates to the service, but inputs/outputs remain unchanged. +- Add tests that assert the service produces the same side effects as the existing Celery task for a small fixture. + + +### Phase 3 – Wire reparsing through `ParsingService` + +- Refactor `search_indexes/reparse.py` and `search_indexes/utils.py` so that they: + - Only perform backup + selection + deletion + Celey scheduling + - Never run parsing logic directly +- Ensure that `ReparseMeta` / `ReparseFileMeta` updates are driven by `ReparseService` and not by ad-hoc logic outside the service. + + +### Phase 4 – Clean up and harden + +- Remove now-dead code paths or duplicated logic. +- Add better failure handling and idempotency guarantees for reparse runs. + +#### Phase 4 – Reparse hardening checklist (explicit requirements) + +- **Separate batch vs single-file logic** + - `ParsingService` owns exactly one `DataFile` parse end-to-end. + - `ReparseService` owns reparse orchestration for *N files* and is the only place that updates `ReparseMeta` / `ReparseFileMeta`. +- **Idempotent reparse runs** + - Define what happens if the same reparse is triggered twice (default: skip finished files; optional: force-restart with a new attempt id). + - Ensure a reparse run does not double-create or double-count `DataFileSummary`, `ParserError`, or error reports. +- **Explicit reparse attempt tracking** + - Record an `attempt` number (or `run_id`) per `DataFile` within a reparse so we can distinguish first-run vs retry outputs. + - Avoid overwriting debugging signals (timestamps, success/failure) without keeping attempt history. +- **Batch progress aggregation** + - Store or compute counts by child state: `pending`, `in_progress`, `succeeded`, `failed`, `stuck`, `canceled`. + - Provide a derived overall reparse status for admin visibility and logs (with explicit precedence rules). +- **Better failure handling** + - Decide policy for partial failures (recommended: continue processing remaining files; batch completes with failures). + - Persist failure context on file meta (stage, exception type/message) and surface a summarized view on the reparse meta. +- **Stuck detection + recovery** + - Track `started_at` and a “last progress” timestamp per file (e.g., `last_state_change_at` / `heartbeat_at`). + - If a file remains in an active stage past a threshold, mark it `stuck` and provide a clear operator action (retry, fail, or cancel). +- **Concurrency controls** + - Prevent two reparses from processing the same `DataFile` concurrently (DB guard/locking or explicit “in progress” ownership). + - Optional: chunk large reparses to avoid overwhelming workers and to improve progress reporting. +- **Make side effects configurable** + - Allow `ReparseService` to control whether to send submission emails and whether/when to regenerate error reports. + - Default behavior: do not send “data submitted” emails for reparses unless explicitly requested. +- **Transactional boundaries** + - Ensure per-file “start” and “finish” updates are atomic and durable even if a worker crashes mid-run. + - Prefer explicit transactions around metadata updates so reparse progress cannot end up partially written. +- Add regression tests for: + - Incremental reparsing (subset of files, STT-specific) + - Large batches (e.g., dozens/hundreds of files) + + +## 6. Expected Benefits + +1. **Reduced duplication** + Single source of truth for parsing logic, regardless of whether it is an initial parse or a reparse. + +2. **Easier reasoning and debugging** + Parsing behavior is concentrated in `ParsingService`; reparse orchestration just chooses “what” to parse. + +3. **Better testability** + Service-oriented design enables targeted unit tests instead of having to go through Celery + management commands for every behavior change. + +4. **Improved performance tuning** + Backup, deletion, and parsing steps are separated more clearly, making it easier to profile and optimize the heaviest operations. + +5. **Lower operational risk** + A more explicit lifecycle for ReparseMeta / ReparseFileMeta, with a single place where their states are updated, makes it easier to detect and recover from partial failures. diff --git a/docs/Technical-Documentation/tech-memos/submission-state-machine.md b/docs/Technical-Documentation/tech-memos/submission-state-machine.md new file mode 100644 index 000000000..74fc3ab1a --- /dev/null +++ b/docs/Technical-Documentation/tech-memos/submission-state-machine.md @@ -0,0 +1,88 @@ +# Submission State Machine for File Processing + +## Purpose +Define and enforce a clear lifecycle for uploaded files so parsing and triage share a consistent contract. This is a precursor to the parser refactor to avoid churn and make status handling predictable. + +## Why a state machine (and what it adds) +- **Guardrails for future changes:** Even though end users cannot alter parsing, developers can. An explicit transition map prevents drift when we add steps (AV scan, retries, change requests) or touch the parser/reparser code paths. Instead of silently landing in an inconsistent state, we fail fast on illegal transitions. +- **Durable, user-visible lifecycle:** `DataFileSummary` is per-parse and can be deleted/recreated during reparses. `DataFile.state` is a durable record of the submission lifecycle (upload -> scan -> parse) that survives reparses and exists even before a summary is created. +- **Better triage and alerts:** Granular states (for example `virus_scan_started` vs `parse_started`) make it obvious where a file stalled without scraping logs. They enable targeted alerts (for example, "stuck in `parse_started` > 15m") and safer retries. + +## States +`uploaded` -> `virus_scan_started` -> (`virus_scan_failed` | `virus_scan_successful`) -> `parse_started` -> (`parsed_with_errors` | `parsed_completed`) -> `completed`. + +Any active state can transition to `canceled`. A file that exceeds time thresholds in an active state is marked `stuck` (and may later be escalated to `failed` by policy). + +Note: parsing can write records in batches. This proposal does not model a separate ingest phase yet. + +## Allowed transitions (code sketch) +```python +from dataclasses import dataclass, field +from enum import Enum +from typing import Dict, Iterable + + +class SubmissionState(str, Enum): + UPLOADED = "uploaded" + VIRUS_SCAN_STARTED = "virus_scan_started" + VIRUS_SCAN_FAILED = "virus_scan_failed" + VIRUS_SCAN_SUCCESSFUL = "virus_scan_successful" + PARSE_STARTED = "parse_started" + PARSE_COMPLETED = "parse_completed" + STUCK = "stuck" + COMPLETED = "completed" + CANCELED = "canceled" + + +ALLOWED_TRANSITIONS: Dict[SubmissionState, Iterable[SubmissionState]] = { + SubmissionState.UPLOADED: { + SubmissionState.VIRUS_SCAN_STARTED, + SubmissionState.CANCELED, + }, + SubmissionState.VIRUS_SCAN_STARTED: { + SubmissionState.VIRUS_SCAN_FAILED, + SubmissionState.VIRUS_SCAN_SUCCESSFUL, + SubmissionState.CANCELED, + }, + SubmissionState.VIRUS_SCAN_FAILED: { + SubmissionState.CANCELED, + }, + SubmissionState.VIRUS_SCAN_SUCCESSFUL: { + SubmissionState.PARSE_STARTED, + SubmissionState.CANCELED, + }, + SubmissionState.PARSE_STARTED: { + SubmissionState.PARSED_WITH_ERRORS, + SubmissionState.PARSED_COMPLETED, + SubmissionState.CANCELED, + }, + SubmissionState.PARSED_WITH_ERRORS: { + SubmissionState.COMPLETED, + SubmissionState.CANCELED, + }, + SubmissionState.PARSED_COMPLETED: { + SubmissionState.COMPLETED, + SubmissionState.CANCELED, + }, + SubmissionState.STUCK: { + SubmissionState.CANCELED, + }, + SubmissionState.COMPLETED: set(), + SubmissionState.CANCELED: set(), +} + + +class InvalidTransition(Exception): + ... + + +@dataclass +class SubmissionLifecycle: + state: SubmissionState + history: list[str] = field(default_factory=list) + + def transition(self, next_state: SubmissionState, note: str = "") -> None: + if next_state not in ALLOWED_TRANSITIONS[self.state]: + raise InvalidTransition(f"{self.state} -> {next_state} not allowed") + self.history.append(f"{self.state} -> {next_state}: {note}") + self.state = next_state \ No newline at end of file diff --git a/scripts/apply-database-config.sh b/scripts/apply-database-config.sh index f5fe8da87..deadbccbc 100644 --- a/scripts/apply-database-config.sh +++ b/scripts/apply-database-config.sh @@ -87,7 +87,7 @@ echo "Done." if [[ $app == "tdp-backend-develop" || $space == "tanf-dev" ]]; then echo "Applying e2e test data" python manage.py populate_stts - python manage.py loaddata cypress/users cypress/data_files cypress/regions cypress/profile_editing_regions cypress/profile_editing_users + python manage.py loaddata cypress/users cypress/data_files cypress/regions cypress/profile_editing_regions cypress/profile_editing_users cypress/feature_flags echo "Done." fi diff --git a/tdrs-backend/plg/alertmanager/alertmanager.yml b/tdrs-backend/plg/alertmanager/alertmanager.yml index 888c8e27c..6b61725d9 100644 --- a/tdrs-backend/plg/alertmanager/alertmanager.yml +++ b/tdrs-backend/plg/alertmanager/alertmanager.yml @@ -39,7 +39,8 @@ route: - matchers: - alertname=~"UpTime" receiver: dev-team-emails - repeat_interval: 24h + repeat_interval: 48h + continue: true # Send all severity CRITICAL/ERROR alerts to OFA admin emails - matchers: - severity=~"ERROR|CRITICAL" @@ -47,11 +48,15 @@ route: continue: true # Send all severity CRITICAL/ERROR/WARNING alerts to mattermost and dev team emails - matchers: - - severity=~"ERROR|CRITICAL|WARNING" + - severity=~"ERROR|CRITICAL" receiver: mattermost continue: true - matchers: - - severity=~"ERROR|CRITICAL|WARNING" + - severity=~"WARNING" + receiver: dev-mattermost + continue: true + - matchers: + - severity=~"ERROR|CRITICAL" receiver: dev-team-emails continue: true # Inhibition rules allow to mute a set of alerts given that another alert is @@ -90,11 +95,29 @@ receivers: {{ if or (eq .Labels.severity "CRITICAL") (eq .Labels.severity "ERROR") }} @here {{ end }} - *Alert:* {{ .Annotations.title }}{{ if .Labels.severity }} - `{{ .Labels.severity }}`{{ end }} + *Alert:* {{ .Labels.alertname }}{{ if .Labels.severity }} - `{{ .Labels.severity }}`{{ end }} + + *Description:* {{ .Annotations.description }} + + *Details:* + • *Job:* `{{ .Labels.job }}` + • *Instance:* `{{ .Labels.instance }}` + • *Env:* `{{ .Labels.env }}` + {{ end }} + - name: 'dev-mattermost' + slack_configs: + - channel: 'tdp-dev-alerts' + username: 'alertmanager' + send_resolved: true + api_url: 'https://fake.mattermost.com' + text: |- + {{ range .Alerts -}} + *Alert:* {{ .Labels.alertname }}{{ if .Labels.severity }} - `{{ .Labels.severity }}`{{ end }} *Description:* {{ .Annotations.description }} *Details:* - {{ range .Labels.SortedPairs }} • *{{ .Name }}:* `{{ .Value }}` - {{ end }} + • *Job:* `{{ .Labels.job }}` + • *Instance:* `{{ .Labels.instance }}` + • *Env:* `{{ .Labels.env }}` {{ end }} diff --git a/tdrs-backend/plg/deploy.sh b/tdrs-backend/plg/deploy.sh index c6febf762..24631ea0a 100755 --- a/tdrs-backend/plg/deploy.sh +++ b/tdrs-backend/plg/deploy.sh @@ -105,6 +105,7 @@ deploy_alertmanager() { yq eval -i ".global.slack_api_url = \"$MATTERMOST_WEBHOOK_URL\"" $CONFIG yq eval -i ".receivers[0].email_configs[0].to = \"${ADMIN_EMAILS}\"" $CONFIG yq eval -i ".receivers[1].email_configs[0].to = \"${DEV_EMAILS}\"" $CONFIG + yq eval -i ".receivers[3].slack_configs[0].api_url = \"${DEV_MATTERMOST_WEBHOOK_URL}\"" $CONFIG cf push --no-route -f manifest.yml -t 180 --strategy rolling cf map-route alertmanager apps.internal --hostname alertmanager rm $CONFIG diff --git a/tdrs-backend/plg/grafana/dashboards/health.json b/tdrs-backend/plg/grafana/dashboards/health.json index 182e54487..c47c8a766 100644 --- a/tdrs-backend/plg/grafana/dashboards/health.json +++ b/tdrs-backend/plg/grafana/dashboards/health.json @@ -12,13 +12,28 @@ "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "enable": true, + "hide": true, + "expr": "pg_up{job=~\"postgres-$pg_env\"} == 0 or absent(pg_up{job=~\"postgres-$pg_env\"})", + "iconColor": "red", + "name": "Database Unreachable", + "step": "60s", + "tagKeys": "job", + "titleFormat": "Database Unreachable", + "textFormat": "The postgres exporter cannot reach the database (pg_up=0 or absent)" } ] }, "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 7, + "id": 5, "links": [], "panels": [ { @@ -47,7 +62,7 @@ }, "id": 1, "options": { - "alertInstanceLabelFilter": "", + "alertInstanceLabelFilter": "{job=~\"tdp-backend-$app_env\"}", "alertName": "backend", "dashboardAlerts": false, "datasource": "Prometheus", @@ -66,7 +81,7 @@ }, "viewMode": "list" }, - "pluginVersion": "12.0.1", + "pluginVersion": "12.0.2", "title": "Backend - Active Alerts", "type": "alertlist" }, @@ -131,7 +146,7 @@ "textMode": "value", "wideLayout": true }, - "pluginVersion": "12.0.1", + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -205,7 +220,7 @@ "textMode": "value", "wideLayout": true }, - "pluginVersion": "12.0.1", + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -285,7 +300,7 @@ "showThresholdMarkers": true, "sizing": "auto" }, - "pluginVersion": "12.0.1", + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -366,7 +381,7 @@ "sizing": "auto", "valueMode": "color" }, - "pluginVersion": "12.0.1", + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -388,63 +403,13 @@ "title": "Errors Rate (Backend)", "type": "bargauge" }, - { - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 10 - }, - "id": 29, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "# Add backend CPU here", - "mode": "markdown" - }, - "pluginVersion": "12.0.1", - "title": "Average CPU Usage (Backend)", - "type": "text" - }, - { - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 10 - }, - "id": 30, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "# Add backend Memory here", - "mode": "markdown" - }, - "pluginVersion": "12.0.1", - "title": "Average Memory Usage (Backend)", - "type": "text" - }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 18 + "y": 10 }, "id": 31, "panels": [], @@ -460,11 +425,11 @@ "h": 4, "w": 24, "x": 0, - "y": 19 + "y": 11 }, "id": 34, "options": { - "alertInstanceLabelFilter": "", + "alertInstanceLabelFilter": "{job=~\"tdp-celery-$app_env|tdp-celery-exporter-$app_env\"}", "alertName": "celery", "dashboardAlerts": false, "datasource": "Prometheus", @@ -483,7 +448,7 @@ }, "viewMode": "list" }, - "pluginVersion": "12.0.1", + "pluginVersion": "12.0.2", "title": "Celery - Active Alerts", "type": "alertlist" }, @@ -527,7 +492,7 @@ "h": 5, "w": 12, "x": 0, - "y": 23 + "y": 15 }, "id": 35, "options": { @@ -547,7 +512,7 @@ "textMode": "value", "wideLayout": true }, - "pluginVersion": "11.2.0", + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -601,7 +566,7 @@ "h": 5, "w": 12, "x": 12, - "y": 23 + "y": 15 }, "id": 24, "options": { @@ -621,7 +586,7 @@ "textMode": "value", "wideLayout": true }, - "pluginVersion": "11.2.0", + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -689,7 +654,7 @@ "h": 7, "w": 24, "x": 0, - "y": 28 + "y": 20 }, "id": 25, "options": { @@ -711,7 +676,7 @@ } ] }, - "pluginVersion": "11.2.0", + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -882,7 +847,7 @@ "h": 6, "w": 24, "x": 0, - "y": 35 + "y": 27 }, "id": 26, "options": { @@ -898,7 +863,7 @@ }, "showHeader": true }, - "pluginVersion": "11.2.0", + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -1000,7 +965,7 @@ "h": 7, "w": 24, "x": 0, - "y": 41 + "y": 33 }, "id": 28, "options": { @@ -1011,11 +976,12 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "desc" } }, - "pluginVersion": "11.2.0", + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -1039,7 +1005,7 @@ "h": 1, "w": 24, "x": 0, - "y": 48 + "y": 40 }, "id": 33, "panels": [], @@ -1055,27 +1021,30 @@ "h": 4, "w": 24, "x": 0, - "y": 49 + "y": 41 }, "id": 6, "options": { - "alertInstanceLabelFilter": "", + "alertInstanceLabelFilter": "{job=~\"postgres-$pg_env\"}", "alertName": "database", "dashboardAlerts": false, "datasource": "Prometheus", "groupBy": [], "groupMode": "default", "maxItems": 20, + "showInactiveAlerts": false, "sortOrder": 3, "stateFilter": { "error": true, "firing": true, "noData": true, "normal": true, - "pending": true + "pending": true, + "recovering": true }, "viewMode": "list" }, + "pluginVersion": "12.0.2", "title": "Database - Active Alerts", "type": "alertlist" }, @@ -1119,7 +1088,7 @@ "h": 5, "w": 12, "x": 0, - "y": 53 + "y": 45 }, "id": 18, "options": { @@ -1139,7 +1108,7 @@ "textMode": "value", "wideLayout": true }, - "pluginVersion": "11.2.0", + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -1193,7 +1162,7 @@ "h": 5, "w": 12, "x": 12, - "y": 53 + "y": 45 }, "id": 36, "options": { @@ -1213,7 +1182,7 @@ "textMode": "value", "wideLayout": true }, - "pluginVersion": "11.2.0", + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -1297,7 +1266,7 @@ "h": 8, "w": 12, "x": 0, - "y": 58 + "y": 50 }, "id": 20, "options": { @@ -1313,10 +1282,12 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -1401,7 +1372,7 @@ "h": 8, "w": 12, "x": 12, - "y": 58 + "y": 50 }, "id": 22, "options": { @@ -1417,10 +1388,12 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -1454,7 +1427,7 @@ } ], "preload": false, - "refresh": "5s", + "refresh": "30s", "schemaVersion": 41, "tags": [ "Backend", @@ -1552,27 +1525,6 @@ "query": "production, staging, dev, local", "type": "custom" }, - { - "current": { - "isNone": true, - "text": "None", - "value": "" - }, - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "definition": "", - "includeAll": false, - "label": "Namespace", - "name": "namespace", - "options": [], - "query": "label_values(celery_worker_up{}, namespace)", - "refresh": 2, - "regex": "", - "sort": 1, - "type": "query" - }, { "current": { "isNone": true, @@ -1604,5 +1556,5 @@ "timezone": "browser", "title": "Uptime/Health", "uid": "aeh7ymwdwpvk0e", - "version": 1 + "version": 4 } diff --git a/tdrs-backend/plg/grafana/dashboards/postgres_dashboard.json b/tdrs-backend/plg/grafana/dashboards/postgres_dashboard.json index 1a3a4109d..0ed41d6aa 100644 --- a/tdrs-backend/plg/grafana/dashboards/postgres_dashboard.json +++ b/tdrs-backend/plg/grafana/dashboards/postgres_dashboard.json @@ -49,6 +49,23 @@ "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "enable": true, + "expr": "pg_up{instance=~\"$instance\"} == 0 or absent(pg_up{instance=~\"$instance\"})", + "hide": false, + "iconColor": "red", + "name": "Database Unreachable", + "step": "60s", + "tagKeys": "instance", + "titleFormat": "Database Unreachable", + "textFormat": "The postgres exporter cannot reach the database (pg_up=0 or absent)", + "type": "tags", + "useValueForTime": false } ] }, @@ -235,12 +252,13 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "pg_static{release=\"$release\", instance=\"$instance\"}", + "expr": "pg_static{instance=\"$instance\"}", "format": "time_series", - "instant": true, + "instant": false, "intervalFactor": 1, "legendFormat": "{{short_version}}", - "refId": "A" + "refId": "A", + "range": true } ], "title": "Version", @@ -314,11 +332,12 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "pg_postmaster_start_time_seconds{release=\"$release\", instance=\"$instance\"} * 1000", + "expr": "pg_postmaster_start_time_seconds{instance=\"$instance\"} * 1000", "format": "time_series", "intervalFactor": 2, "legendFormat": "", - "refId": "A" + "refId": "A", + "range": true } ], "title": "Start Time", @@ -391,11 +410,12 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "SUM(pg_stat_database_tup_inserted{release=\"$release\", datname=~\"$datname\", instance=~\"$instance\"})", + "expr": "SUM(pg_stat_database_tup_inserted{datname=~\"$datname\", instance=~\"$instance\"})", "format": "time_series", "intervalFactor": 2, "refId": "A", - "step": 4 + "step": 4, + "range": true } ], "title": "Current insert data", @@ -472,7 +492,8 @@ "format": "time_series", "intervalFactor": 2, "refId": "A", - "step": 4 + "step": 4, + "range": true } ], "title": "Current fetch data", @@ -549,7 +570,8 @@ "format": "time_series", "intervalFactor": 2, "refId": "A", - "step": 4 + "step": 4, + "range": true } ], "title": "Current update data", @@ -622,10 +644,11 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "pg_settings_max_connections{release=\"$release\", instance=\"$instance\"}", + "expr": "pg_settings_max_connections{instance=\"$instance\"}", "format": "time_series", "intervalFactor": 1, - "refId": "A" + "refId": "A", + "range": true } ], "title": "Max Connections", @@ -723,7 +746,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "avg(rate(process_cpu_seconds_total{release=\"$release\", instance=\"$instance\"}[5m]) * 1000)", + "expr": "avg(rate(process_cpu_seconds_total{instance=\"$instance\"}[5m]) * 1000)", "format": "time_series", "intervalFactor": 2, "legendFormat": "CPU Time", @@ -825,7 +848,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "avg(rate(process_resident_memory_bytes{release=\"$release\", instance=\"$instance\"}[5m]))", + "expr": "avg(rate(process_resident_memory_bytes{instance=\"$instance\"}[5m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "Resident Mem", @@ -836,7 +859,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "avg(rate(process_virtual_memory_bytes{release=\"$release\", instance=\"$instance\"}[5m]))", + "expr": "avg(rate(process_virtual_memory_bytes{instance=\"$instance\"}[5m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "Virtual Mem", @@ -938,7 +961,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "process_open_fds{release=\"$release\", instance=\"$instance\"}", + "expr": "process_open_fds{instance=\"$instance\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "Open FD", @@ -1044,7 +1067,8 @@ "expr": "pg_settings_shared_buffers_bytes{instance=\"$instance\"}", "format": "time_series", "intervalFactor": 1, - "refId": "A" + "refId": "A", + "range": true } ], "title": "Shared Buffers", @@ -1120,7 +1144,8 @@ "expr": "pg_settings_effective_cache_size_bytes{instance=\"$instance\"}", "format": "time_series", "intervalFactor": 1, - "refId": "A" + "refId": "A", + "range": true } ], "title": "Effective Cache", @@ -1196,7 +1221,8 @@ "expr": "pg_settings_maintenance_work_mem_bytes{instance=\"$instance\"}", "format": "time_series", "intervalFactor": 1, - "refId": "A" + "refId": "A", + "range": true } ], "title": "Maintenance Work Mem", @@ -1273,7 +1299,8 @@ "format": "time_series", "intervalFactor": 1, "legendFormat": "", - "refId": "A" + "refId": "A", + "range": true } ], "title": "Work Mem", @@ -1350,7 +1377,8 @@ "expr": "pg_settings_max_wal_size_bytes{instance=\"$instance\"}", "format": "time_series", "intervalFactor": 1, - "refId": "A" + "refId": "A", + "range": true } ], "title": "Max WAL Size", @@ -1426,7 +1454,8 @@ "expr": "pg_settings_random_page_cost{instance=\"$instance\"}", "format": "time_series", "intervalFactor": 1, - "refId": "A" + "refId": "A", + "range": true } ], "title": "Random Page Cost", @@ -1502,7 +1531,8 @@ "expr": "pg_settings_seq_page_cost", "format": "time_series", "intervalFactor": 1, - "refId": "A" + "refId": "A", + "range": true } ], "title": "Seq Page Cost", @@ -1578,7 +1608,8 @@ "expr": "pg_settings_max_worker_processes{instance=\"$instance\"}", "format": "time_series", "intervalFactor": 1, - "refId": "A" + "refId": "A", + "range": true } ], "title": "Max Worker Processes", @@ -1654,7 +1685,8 @@ "expr": "pg_settings_max_parallel_workers{instance=\"$instance\"}", "format": "time_series", "intervalFactor": 1, - "refId": "A" + "refId": "A", + "range": true } ], "title": "Max Parallel Workers", @@ -3261,62 +3293,16 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "definition": "", - "hide": 0, - "includeAll": false, - "label": "Namespace", - "multi": false, - "name": "namespace", - "options": [], - "query": "query_result(pg_exporter_last_scrape_duration_seconds)", - "refresh": 2, - "regex": "/.*kubernetes_namespace=\"([^\"]+).*/", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "current": {}, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "definition": "", - "hide": 0, - "includeAll": false, - "label": "Release", - "multi": false, - "name": "release", - "options": [], - "query": "query_result(pg_exporter_last_scrape_duration_seconds{kubernetes_namespace=\"$namespace\"})", - "refresh": 2, - "regex": "/.*release=\"([^\"]+)/", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "current": {}, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "definition": "", + "definition": "label_values(pg_exporter_last_scrape_duration_seconds, instance)", "hide": 0, "includeAll": false, "label": "Instance", "multi": false, "name": "instance", "options": [], - "query": "query_result(pg_up{release=\"$release\"})", - "refresh": 1, - "regex": "/.*instance=\"([^\"]+).*/", + "query": "label_values(pg_exporter_last_scrape_duration_seconds, instance)", + "refresh": 2, + "regex": "", "skipUrlSync": false, "sort": 1, "tagValuesQuery": "", @@ -3330,15 +3316,15 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "definition": "", + "definition": "label_values(pg_stat_database_tup_fetched{instance=~\"$instance\"}, datname)", "hide": 0, "includeAll": true, "label": "Database", "multi": true, "name": "datname", "options": [], - "query": "label_values(datname)", - "refresh": 1, + "query": "label_values(pg_stat_database_tup_fetched{instance=~\"$instance\"}, datname)", + "refresh": 2, "regex": "", "skipUrlSync": false, "sort": 1, @@ -3353,7 +3339,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "definition": "", + "definition": "label_values({mode=~\"accessexclusivelock|accesssharelock|exclusivelock|rowexclusivelock|rowsharelock|sharelock|sharerowexclusivelock|shareupdateexclusivelock\"}, mode)", "hide": 0, "includeAll": true, "label": "Lock table", @@ -3361,7 +3347,7 @@ "name": "mode", "options": [], "query": "label_values({mode=~\"accessexclusivelock|accesssharelock|exclusivelock|rowexclusivelock|rowsharelock|sharelock|sharerowexclusivelock|shareupdateexclusivelock\"}, mode)", - "refresh": 1, + "refresh": 2, "regex": "", "skipUrlSync": false, "sort": 0, @@ -3406,4 +3392,4 @@ "uid": "000000039", "version": 8, "weekStart": "" -} \ No newline at end of file +} diff --git a/tdrs-backend/plg/prometheus/alerts.local.yml b/tdrs-backend/plg/prometheus/alerts.local.yml index e7ab2a770..3b5facc44 100644 --- a/tdrs-backend/plg/prometheus/alerts.local.yml +++ b/tdrs-backend/plg/prometheus/alerts.local.yml @@ -5,47 +5,46 @@ groups: expr: last_over_time(pg_up{job="postgres-local"}[1m]) == 0 or last_over_time(up{job="postgres-local"}[1m]) == 0 for: 1m labels: - severity: CRITICAL + severity: WARNING annotations: - summary: "The {{ $labels.service }} service is down." - description: "The {{ $labels.service }} service in the {{ $labels.env }} environment has been down for more than 1 minute." + summary: "The {{ $labels.job }} service is down." + description: "The {{ $labels.job }} service in the {{ $labels.env }} environment has been down for more than 1 minute." - name: backend.alerts rules: - alert: LocalBackendDown expr: last_over_time(up{job=~"tdp-backend-local"}[1m]) == 0 - for: 10m + for: 2m labels: - severity: ERROR + severity: WARNING annotations: - summary: "The {{ $labels.service }} service is down." + summary: "The {{ $labels.job }} service is down." description: "The {{ $labels.service }} service in the {{ $labels.env }} environment has been down for more than 10 minutes." - name: plg.alerts rules: - alert: LocalLokiDown expr: last_over_time(up{job="loki"}[1m]) == 0 labels: - severity: ERROR + severity: WARNING annotations: summary: "The {{ $labels.service }} service is down." description: "The {{ $labels.service }} service in the {{ $labels.env }} environment has been down for more than 1 minute." - name: app.alerts rules: - - alert: UpTime + - alert: LocalUpTime expr: avg_over_time(up[1m]) < 0.95 for: 30m labels: severity: WARNING annotations: - summary: "The {{ $labels.service }} service has a uptime warning." - description: "The {{ $labels.service }} service in the {{ $labels.env }} environment is not maintaining 95% uptime." + summary: "The {{ $labels.job }} service has a uptime warning." + description: "The {{ $labels.job }} service in the {{ $labels.env }} environment is not maintaining 95% uptime." - name: celery.alerts rules: - alert: LocalCeleryWorkerDown - expr: last_over_time(up{job=~"celery-.*|celery-exporter.*"}[1m]) == 0 - for: 10m + expr: last_over_time(up{job="tdp-celery-local"}[1m]) == 0 or last_over_time(celery_active_worker_count{job="tdp-celery-local"}[1m]) < 1 + for: 2m labels: - severity: ERROR + severity: WARNING annotations: - summary: "The {{ $labels.service }} service is down." - description: "The {{ $labels.service }} service in the {{ $labels.env }} environment has been down for more than 10 minutes." - + summary: "The {{ $labels.job }} service is down." + description: "The {{ $labels.job }} service in the {{ $labels.env }} environment has been down for more than 10 minutes." diff --git a/tdrs-backend/plg/prometheus/alerts.yml b/tdrs-backend/plg/prometheus/alerts.yml index 1a7493262..9c5049699 100644 --- a/tdrs-backend/plg/prometheus/alerts.yml +++ b/tdrs-backend/plg/prometheus/alerts.yml @@ -4,50 +4,50 @@ groups: - alert: DevDatabaseDown expr: last_over_time(pg_up{job="postgres-dev"}[1m]) == 0 or last_over_time(up{job="postgres-dev"}[1m]) == 0 labels: - severity: CRITICAL + severity: WARNING annotations: - summary: "The {{ $labels.service }} service is down." - description: "The {{ $labels.service }} service in the {{ $labels.env }} environment has been down for more than 1 minute." + summary: "The {{ $labels.job }} service is down." + description: "The {{ $labels.job }} service in the {{ $labels.env }} environment has been down for more than 1 minute." - alert: StagingDatabaseDown expr: last_over_time(pg_up{job="postgres-staging"}[1m]) == 0 or last_over_time(up{job="postgres-staging"}[1m]) == 0 labels: - severity: ERROR + severity: WARNING annotations: - summary: "The {{ $labels.service }} service is down." - description: "The {{ $labels.service }} service in the {{ $labels.env }} environment has been down for more than 1 minute." + summary: "The {{ $labels.job }} service is down." + description: "The {{ $labels.job }} service in the {{ $labels.env }} environment has been down for more than 1 minute." - alert: ProductionDatabaseDown expr: last_over_time(pg_up{job="postgres-production"}[1m]) == 0 or last_over_time(up{job="postgres-production"}[1m]) == 0 labels: severity: CRITICAL annotations: - summary: "The {{ $labels.service }} service is down." - description: "The {{ $labels.service }} service in the {{ $labels.env }} environment has been down for more than 1 minute." + summary: "The {{ $labels.job }} service is down." + description: "The {{ $labels.job }} service in the {{ $labels.env }} environment has been down for more than 1 minute." - name: backend.alerts rules: - alert: DevEnvironmentBackendDown expr: last_over_time(up{job=~"tdp-backend.*", job!~".*prod", job!~".*staging"}[1m]) == 0 for: 10m labels: - severity: ERROR + severity: WARNING annotations: - summary: "The {{ $labels.service }} service is down." - description: "The {{ $labels.service }} service in the {{ $labels.env }} environment has been down for more than 10 minutes." + summary: "The {{ $labels.job }} service is down." + description: "The {{ $labels.job }} service in the {{ $labels.env }} environment has been down for more than 10 minutes." - alert: StagingBackendDown expr: last_over_time(up{job=~"tdp-backend-staging"}[1m]) == 0 for: 10m labels: - severity: ERROR + severity: WARNING annotations: - summary: "The {{ $labels.service }} service is down." - description: "The {{ $labels.service }} service in the {{ $labels.env }} environment has been down for more than 10 minutes." + summary: "The {{ $labels.job }} service is down." + description: "The {{ $labels.job }} service in the {{ $labels.env }} environment has been down for more than 10 minutes." - alert: ProductionBackendDown expr: last_over_time(up{job=~"tdp-backend-prod"}[1m]) == 0 for: 10m labels: severity: CRITICAL annotations: - summary: "The {{ $labels.service }} service is down." - description: "The {{ $labels.service }} service in the {{ $labels.env }} environment has been down for more than 10 minutes." + summary: "The {{ $labels.job }} service is down." + description: "The {{ $labels.job }} service in the {{ $labels.env }} environment has been down for more than 10 minutes." - name: plg.alerts rules: - alert: LokiDown @@ -72,8 +72,8 @@ groups: labels: severity: WARNING annotations: - summary: "The {{ $labels.service }} service has a uptime warning." - description: "The {{ $labels.service }} service in the {{ $labels.env }} environment is not maintaining 95% uptime." + summary: "The {{ $labels.job }} service has a uptime warning." + description: "The {{ $labels.job }} service in the {{ $labels.env }} environment is not maintaining 95% uptime." - name: celery.alerts rules: - alert: CeleryTaskHighFailRate @@ -131,12 +131,51 @@ groups: for: 20m labels: severity: WARNING - - alert: CeleryWorkerDown - expr: last_over_time(up{job=~"celery-.*|celery-exporter.*"}[1m]) == 0 + - alert: A11yCeleryWorkerDown + expr: last_over_time(up{job="tdp-celery-a11y"}[1m]) == 0 or last_over_time(celery_active_worker_count{job="tdp-celery-a11y"}[1m]) < 1 + for: 10m + labels: + severity: WARNING + annotations: + summary: "The {{ $labels.service }} service is down." + description: "The {{ $labels.service }} service in the {{ $labels.env }} environment has been down for more than 10 minutes." + - alert: RaftCeleryWorkerDown + expr: last_over_time(up{job="tdp-celery-raft"}[1m]) == 0 or last_over_time(celery_active_worker_count{job="tdp-celery-raft"}[1m]) < 1 + for: 10m + labels: + severity: WARNING + annotations: + summary: "The {{ $labels.service }} service is down." + description: "The {{ $labels.service }} service in the {{ $labels.env }} environment has been down for more than 10 minutes." + - alert: QaspCeleryWorkerDown + expr: last_over_time(up{job="tdp-celery-qasp"}[1m]) == 0 or last_over_time(celery_active_worker_count{job="tdp-celery-qasp"}[1m]) < 1 + for: 10m + labels: + severity: WARNING + annotations: + summary: "The {{ $labels.service }} service is down." + description: "The {{ $labels.service }} service in the {{ $labels.env }} environment has been down for more than 10 minutes." + - alert: DevCeleryWorkerDown + expr: last_over_time(up{job="tdp-celery-develop"}[1m]) == 0 or last_over_time(celery_active_worker_count{job="tdp-celery-develop"}[1m]) < 1 + for: 10m + labels: + severity: WARNING + annotations: + summary: "The {{ $labels.service }} service is down." + description: "The {{ $labels.service }} service in the {{ $labels.env }} environment has been down for more than 10 minutes." + - alert: StagingCeleryWorkerDown + expr: last_over_time(up{job="tdp-celery-staging"}[1m]) == 0 or last_over_time(celery_active_worker_count{job="tdp-celery-staging"}[1m]) < 1 + for: 10m + labels: + severity: WARNING + annotations: + summary: "The {{ $labels.service }} service is down." + description: "The {{ $labels.service }} service in the {{ $labels.env }} environment has been down for more than 10 minutes." + - alert: ProdCeleryWorkerDown + expr: last_over_time(up{job="tdp-celery-prod"}[1m]) == 0 or last_over_time(celery_active_worker_count{job="tdp-celery-prod"}[1m]) < 1 for: 10m labels: severity: ERROR annotations: summary: "The {{ $labels.service }} service is down." description: "The {{ $labels.service }} service in the {{ $labels.env }} environment has been down for more than 10 minutes." - diff --git a/tdrs-backend/plg/prometheus/manifest.yml b/tdrs-backend/plg/prometheus/manifest.yml index d5c72d72f..c6f1fc680 100644 --- a/tdrs-backend/plg/prometheus/manifest.yml +++ b/tdrs-backend/plg/prometheus/manifest.yml @@ -11,6 +11,6 @@ applications: mv ./prometheus-2.54.1.linux-amd64/prometheus ./prometheus && mv ./prometheus-2.54.1.linux-amd64/promtool ./promtool && rm -rf ./prometheus-2.54.1.linux-amd64 && rm -rf prometheus-2.54.1.linux-amd64.tar.gz && - ./prometheus --config.file=/home/vcap/app/prometheus.yml --storage.tsdb.path=/home/vcap/app/prometheus-data --storage.tsdb.retention.time=30d --storage.tsdb.retention.size=6GB --web.listen-address="0.0.0.0:8080" --web.enable-lifecycle + ./prometheus --config.file=/home/vcap/app/prometheus.yml --storage.tsdb.path=/home/vcap/app/prometheus-data --storage.tsdb.retention.time=30d --storage.tsdb.retention.size=6GB --web.listen-address="0.0.0.0:8080" --web.enable-lifecycle --web.enable-remote-write-receiver --enable-feature=exemplar-storage --enable-feature=native-histograms buildpacks: - https://github.com/cloudfoundry/binary-buildpack diff --git a/tdrs-backend/scripts/create_grafana_postgres_role.py b/tdrs-backend/scripts/create_grafana_postgres_role.py index bbccdf3b9..d6682289d 100644 --- a/tdrs-backend/scripts/create_grafana_postgres_role.py +++ b/tdrs-backend/scripts/create_grafana_postgres_role.py @@ -13,7 +13,7 @@ $$; GRANT CONNECT ON DATABASE {db_name} TO {role}; GRANT USAGE ON SCHEMA public TO {role}; -{select_stmt} +{revoke_create}{select_stmt} """ SELECT_STATEMENT = "GRANT SELECT ON {tables} TO {role};" @@ -64,7 +64,9 @@ def run(*args): # noqa: C901 if remaining == ("all",): select_stmt = ADMIN_SELECT_STATEMENT.format(role=role) - sql = sql_tmpl.format(role=role, db_name=db_name, select_stmt=select_stmt) + sql = sql_tmpl.format( + role=role, db_name=db_name, revoke_create="", select_stmt=select_stmt + ) else: tables: list[str] = [] for arg in remaining: @@ -83,8 +85,15 @@ def run(*args): # noqa: C901 tables_str = ",".join(tables) select_stmt = SELECT_STATEMENT.format(tables=tables_str, role=role) + revoke_create = "REVOKE CREATE ON SCHEMA public FROM {role};\n".format( + role=role + ) sql = sql_tmpl.format( - role=role, tables=tables_str, db_name=db_name, select_stmt=select_stmt + role=role, + tables=tables_str, + db_name=db_name, + revoke_create=revoke_create, + select_stmt=select_stmt, ) with connection.cursor() as cursor: diff --git a/tdrs-backend/tdpservice/data_files/enums.py b/tdrs-backend/tdpservice/data_files/enums.py new file mode 100644 index 000000000..3525718f5 --- /dev/null +++ b/tdrs-backend/tdpservice/data_files/enums.py @@ -0,0 +1,19 @@ +"""Enums for the data_files app.""" + +from django.db import models + + +class SubmissionState(models.TextChoices): + """Lifecycle states for a submitted data file.""" + + UPLOADED = "uploaded", "Uploaded" + VIRUS_SCAN_STARTED = "virus_scan_started", "Virus scan started" + VIRUS_SCAN_FAILED = "virus_scan_failed", "Virus scan failed" + VIRUS_SCAN_COMPLETED = "virus_scan_completed", "Virus scan completed" + PARSE_STARTED = "parse_started", "Parse started" + PARSE_FAILED = "parse_failed", "Parse failed" + PARSED_WITH_ERRORS = "parsed_with_errors", "Parsed with errors" + PARSE_COMPLETED = "parse_completed", "Parse completed" + STUCK = "stuck", "Stuck" + COMPLETED = "completed", "Completed" + CANCELED = "canceled", "Canceled" diff --git a/tdrs-backend/tdpservice/data_files/migrations/0025_datafile_state.py b/tdrs-backend/tdpservice/data_files/migrations/0025_datafile_state.py new file mode 100644 index 000000000..fcfdd22e8 --- /dev/null +++ b/tdrs-backend/tdpservice/data_files/migrations/0025_datafile_state.py @@ -0,0 +1,34 @@ +# Generated by Django 3.2.15 on 2026-03-10 15:00 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("data_files", "0024_alter_datafile_file"), + ] + + operations = [ + migrations.AddField( + model_name="datafile", + name="state", + field=models.CharField( + choices=[ + ("uploaded", "Uploaded"), + ("virus_scan_started", "Virus scan started"), + ("virus_scan_failed", "Virus scan failed"), + ("virus_scan_completed", "Virus scan completed"), + ("parse_started", "Parse started"), + ("parse_failed", "Parse failed"), + ("parsed_with_errors", "Parsed with errors"), + ("parse_completed", "Parse completed"), + ("stuck", "Stuck"), + ("completed", "Completed"), + ("canceled", "Canceled"), + ], + default="uploaded", + max_length=32, + ), + preserve_default=True, + ), + ] diff --git a/tdrs-backend/tdpservice/data_files/models.py b/tdrs-backend/tdpservice/data_files/models.py index 6d0544a47..46bd3bf5d 100644 --- a/tdrs-backend/tdpservice/data_files/models.py +++ b/tdrs-backend/tdpservice/data_files/models.py @@ -17,6 +17,7 @@ from tdpservice.backends import DataFilesS3Storage from tdpservice.common.fields import S3VersionedFileField from tdpservice.common.models import FileRecord +from tdpservice.data_files.enums import SubmissionState from tdpservice.data_files.util import ( create_legacy_s3_log_file_path, create_s3_log_file_path, @@ -171,6 +172,13 @@ class Meta: is_program_audit = models.BooleanField(default=False) version = models.IntegerField() + state = models.CharField( + max_length=32, + blank=False, + null=False, + choices=SubmissionState.choices, + default=SubmissionState.UPLOADED, + ) user = models.ForeignKey( User, on_delete=models.CASCADE, related_name="user", blank=False, null=False diff --git a/tdrs-backend/tdpservice/data_files/submission_lifecycle.py b/tdrs-backend/tdpservice/data_files/submission_lifecycle.py new file mode 100644 index 000000000..9594a238b --- /dev/null +++ b/tdrs-backend/tdpservice/data_files/submission_lifecycle.py @@ -0,0 +1,126 @@ +"""Helpers for DataFile submission state transitions.""" + +import logging +from dataclasses import dataclass +from typing import Callable, Dict, Iterable + +from tdpservice.data_files.enums import SubmissionState + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class TransitionRecord: + """In-memory record of a single submission state transition.""" + + previous_state: SubmissionState + next_state: SubmissionState + note: str = "" + + +class InvalidTransition(ValueError): + """Raised when an invalid submission state transition is attempted.""" + + +ALLOWED_TRANSITIONS: Dict[SubmissionState, Iterable[SubmissionState]] = { + SubmissionState.UPLOADED: { + SubmissionState.VIRUS_SCAN_STARTED, + SubmissionState.CANCELED, + }, + SubmissionState.VIRUS_SCAN_STARTED: { + SubmissionState.VIRUS_SCAN_FAILED, + SubmissionState.VIRUS_SCAN_COMPLETED, + SubmissionState.CANCELED, + }, + SubmissionState.VIRUS_SCAN_FAILED: { + SubmissionState.CANCELED, + }, + SubmissionState.VIRUS_SCAN_COMPLETED: { + SubmissionState.PARSE_STARTED, + SubmissionState.CANCELED, + }, + SubmissionState.PARSE_STARTED: { + SubmissionState.PARSE_FAILED, + SubmissionState.PARSED_WITH_ERRORS, + SubmissionState.PARSE_COMPLETED, + SubmissionState.CANCELED, + }, + SubmissionState.PARSE_FAILED: { + SubmissionState.CANCELED, + }, + SubmissionState.PARSED_WITH_ERRORS: { + SubmissionState.COMPLETED, + SubmissionState.CANCELED, + }, + SubmissionState.PARSE_COMPLETED: { + SubmissionState.COMPLETED, + SubmissionState.CANCELED, + }, + SubmissionState.STUCK: { + SubmissionState.CANCELED, + }, + SubmissionState.COMPLETED: set(), + SubmissionState.CANCELED: set(), +} + + +def coerce_submission_state(state) -> SubmissionState: + """Normalize a state value into a SubmissionState enum.""" + if isinstance(state, SubmissionState): + return state + return SubmissionState(state) + + +def allowed_next_states(current_state) -> set[SubmissionState]: + """Return the allowed next states for the given current state.""" + normalized_current_state = coerce_submission_state(current_state) + return set(ALLOWED_TRANSITIONS[normalized_current_state]) + + +def validate_transition(current_state, next_state) -> TransitionRecord: + """Validate a transition request and return a transition record.""" + normalized_current_state = coerce_submission_state(current_state) + normalized_next_state = coerce_submission_state(next_state) + + if normalized_next_state not in allowed_next_states(normalized_current_state): + raise InvalidTransition( + f"Cannot transition submission from {normalized_current_state.value} " + + f"to {normalized_next_state.value}." + ) + + return TransitionRecord( + previous_state=normalized_current_state, + next_state=normalized_next_state, + ) + + +def transition_datafile( + data_file, + next_state, + note="", + logger_hook: Callable | None = None, +): + """Safely transition a DataFile.state value and persist the new state.""" + transition = validate_transition(data_file.state, next_state) + transition = TransitionRecord( + previous_state=transition.previous_state, + next_state=transition.next_state, + note=note, + ) + + data_file.state = transition.next_state + data_file.save(update_fields=["state"]) + + log_payload = { + "data_file_id": data_file.id, + "previous_state": transition.previous_state.value, + "next_state": transition.next_state.value, + "note": note, + } + + if logger_hook is not None: + logger_hook(log_payload) + else: + logger.info("DataFile submission state transition", extra=log_payload) + + return data_file diff --git a/tdrs-backend/tdpservice/data_files/test/factories.py b/tdrs-backend/tdpservice/data_files/test/factories.py index cce404d84..0dcd94b81 100644 --- a/tdrs-backend/tdpservice/data_files/test/factories.py +++ b/tdrs-backend/tdpservice/data_files/test/factories.py @@ -2,6 +2,7 @@ import factory +from tdpservice.data_files.enums import SubmissionState from tdpservice.stts.test.factories import STTFactory from tdpservice.users.test.factories import UserFactory @@ -22,6 +23,7 @@ class Meta: quarter = "Q1" year = 2020 version = 1 + state = SubmissionState.UPLOADED user = factory.SubFactory(UserFactory) stt = factory.SubFactory(STTFactory) file = factory.django.FileField(data=b"test", filename="my_data_file.txt") diff --git a/tdrs-backend/tdpservice/data_files/test/test_serializers.py b/tdrs-backend/tdpservice/data_files/test/test_serializers.py index c19c56afe..112a9d4c5 100644 --- a/tdrs-backend/tdpservice/data_files/test/test_serializers.py +++ b/tdrs-backend/tdpservice/data_files/test/test_serializers.py @@ -1,4 +1,5 @@ """Test data file serializers.""" + from django.core.exceptions import ValidationError import pytest @@ -10,6 +11,7 @@ validate_file_infection, ) from tdpservice.security.clients import ClamAVClient +from tdpservice.security.models import ClamAVFileScan @pytest.mark.django_db @@ -55,6 +57,14 @@ def test_immutability_of_data_file(data_file_instance): @pytest.mark.django_db def test_created_at(data_file_data, data_analyst): """If a serializer has valid data it will return a valid object.""" + ClamAVFileScan.objects.record_scan( + data_file_data["file"], + data_file_data["file"].name, + f"File scan marked as CLEAN for file: {data_file_data['file'].name}", + ClamAVFileScan.Result.CLEAN, + data_analyst, + ) + create_serializer = DataFileSerializer( context={"user": data_analyst}, data=data_file_data ) @@ -65,6 +75,14 @@ def test_created_at(data_file_data, data_analyst): assert data_file.av_scans.exists() +@pytest.mark.django_db +def test_state_not_exposed_by_serializer(data_file_instance): + """Test submission state remains schema-only for serializer output.""" + serialized = DataFileSerializer(data_file_instance).data + + assert "state" not in serialized + + @pytest.mark.django_db def test_data_file_still_created_if_av_scan_fails_to_create( data_file_data, mocker, data_analyst @@ -131,17 +149,19 @@ def test_rejects_invalid_file_extensions(file_name): @pytest.mark.django_db -def test_rejects_infected_file(infected_file, fake_file_name, user): +def test_rejects_infected_file(infected_file, fake_file_name, user, settings): """Test infected files are rejected by serializer validation.""" + settings.CLAMAV_NEEDED = True with pytest.raises(ValidationError): validate_file_infection(infected_file, fake_file_name, user) @pytest.mark.django_db def test_rejects_uploads_on_clamav_connection_error( - fake_file, fake_file_name, mocker, user + fake_file, fake_file_name, mocker, user, settings ): """Test that DataFiles cannot pass validation if ClamAV is down.""" + settings.CLAMAV_NEEDED = True mocker.patch( "tdpservice.security.clients.ClamAVClient.scan_file", side_effect=ClamAVClient.ServiceUnavailable(), diff --git a/tdrs-backend/tdpservice/data_files/test/test_submission_lifecycle.py b/tdrs-backend/tdpservice/data_files/test/test_submission_lifecycle.py new file mode 100644 index 000000000..bab6f3b36 --- /dev/null +++ b/tdrs-backend/tdpservice/data_files/test/test_submission_lifecycle.py @@ -0,0 +1,144 @@ +"""Tests for submission lifecycle helpers.""" + +import pytest + +from tdpservice.data_files.enums import SubmissionState +from tdpservice.data_files.submission_lifecycle import ( + InvalidTransition, + allowed_next_states, + transition_datafile, + validate_transition, +) +from tdpservice.data_files.test.factories import DataFileFactory + + +def test_valid_transitions_succeed(): + """Test allowed state transitions validate successfully.""" + first = validate_transition( + SubmissionState.UPLOADED, SubmissionState.VIRUS_SCAN_STARTED + ) + second = validate_transition( + SubmissionState.VIRUS_SCAN_STARTED, SubmissionState.VIRUS_SCAN_COMPLETED + ) + + assert first.previous_state == SubmissionState.UPLOADED + assert first.next_state == SubmissionState.VIRUS_SCAN_STARTED + assert second.previous_state == SubmissionState.VIRUS_SCAN_STARTED + assert second.next_state == SubmissionState.VIRUS_SCAN_COMPLETED + assert allowed_next_states(SubmissionState.UPLOADED) == { + SubmissionState.VIRUS_SCAN_STARTED, + SubmissionState.CANCELED, + } + + +def test_invalid_transition_raises(): + """Test invalid transitions raise InvalidTransition.""" + with pytest.raises(InvalidTransition, match="uploaded to parse_completed"): + validate_transition(SubmissionState.UPLOADED, SubmissionState.PARSE_COMPLETED) + + +@pytest.mark.parametrize( + "state", + [ + SubmissionState.COMPLETED, + SubmissionState.CANCELED, + ], +) +def test_terminal_states_cannot_transition(state): + """Test terminal states reject further transitions.""" + with pytest.raises(InvalidTransition, match=f"{state.value} to uploaded"): + validate_transition(state, SubmissionState.UPLOADED) + + +@pytest.mark.django_db +def test_transition_datafile_updates_state(): + """Test transition_datafile persists the expected state.""" + data_file = DataFileFactory(state=SubmissionState.UPLOADED) + + transition_datafile( + data_file, + SubmissionState.VIRUS_SCAN_STARTED, + note="Picked up by AV scan worker", + ) + data_file.refresh_from_db() + + assert data_file.state == SubmissionState.VIRUS_SCAN_STARTED + + +@pytest.mark.django_db +def test_transition_datafile_calls_logger_hook(): + """Test transition_datafile emits structured payloads to a logger hook.""" + data_file = DataFileFactory(state=SubmissionState.PARSE_STARTED) + payloads = [] + + transition_datafile( + data_file, + SubmissionState.PARSE_COMPLETED, + note="Parser completed successfully", + logger_hook=payloads.append, + ) + + assert payloads == [ + { + "data_file_id": data_file.id, + "previous_state": SubmissionState.PARSE_STARTED.value, + "next_state": SubmissionState.PARSE_COMPLETED.value, + "note": "Parser completed successfully", + } + ] + + +@pytest.mark.django_db +def test_transition_datafile_integration_persists_sequential_state_changes(): + """Test sequential persisted transitions on a real DataFile instance.""" + data_file = DataFileFactory(state=SubmissionState.UPLOADED) + payloads = [] + + transition_datafile( + data_file, + SubmissionState.VIRUS_SCAN_STARTED, + note="Virus scan worker picked up the file", + logger_hook=payloads.append, + ) + data_file.refresh_from_db() + + assert data_file.state == SubmissionState.VIRUS_SCAN_STARTED + + transition_datafile( + data_file, + SubmissionState.VIRUS_SCAN_COMPLETED, + note="Virus scan passed", + logger_hook=payloads.append, + ) + data_file.refresh_from_db() + + assert data_file.state == SubmissionState.VIRUS_SCAN_COMPLETED + assert payloads == [ + { + "data_file_id": data_file.id, + "previous_state": SubmissionState.UPLOADED.value, + "next_state": SubmissionState.VIRUS_SCAN_STARTED.value, + "note": "Virus scan worker picked up the file", + }, + { + "data_file_id": data_file.id, + "previous_state": SubmissionState.VIRUS_SCAN_STARTED.value, + "next_state": SubmissionState.VIRUS_SCAN_COMPLETED.value, + "note": "Virus scan passed", + }, + ] + + +@pytest.mark.django_db +def test_transition_datafile_supports_parse_failed_state(): + """Test transition_datafile persists parse failures caused by exceptions.""" + data_file = DataFileFactory(state=SubmissionState.PARSE_STARTED) + + transition_datafile( + data_file, + SubmissionState.PARSE_FAILED, + note="Parser raised an unexpected exception", + ) + data_file.refresh_from_db() + + assert data_file.state == SubmissionState.PARSE_FAILED diff --git a/tdrs-backend/tdpservice/email/helpers/data_file.py b/tdrs-backend/tdpservice/email/helpers/data_file.py index 3dc494bab..b7ea13037 100644 --- a/tdrs-backend/tdpservice/email/helpers/data_file.py +++ b/tdrs-backend/tdpservice/email/helpers/data_file.py @@ -1,5 +1,7 @@ """Helper functions for sending data file submission emails.""" +from zoneinfo import ZoneInfo + from django.conf import settings from tdpservice.data_files.models import DataFile @@ -79,6 +81,19 @@ def get_tanf_total_errors_context_count(datafile_summary): return {"total_errors": total_errors} +def get_pia_quarter_label(quarter): + """Return the human-readable quarter label for PIA submissions.""" + match quarter: + case DataFile.Quarter.Q1: + return "Quarter 1 (October - December)" + case DataFile.Quarter.Q2: + return "Quarter 2 (January - March)" + case DataFile.Quarter.Q3: + return "Quarter 3 (April - June)" + case DataFile.Quarter.Q4: + return "Quarter 4 (July - September)" + + def get_fra_aggregates_context_count(datafile_summary): """Return the relevant context data from case aggregates for FRA files.""" case_aggregates = datafile_summary.case_aggregates or {} @@ -113,10 +128,20 @@ def send_data_submitted_email( prog_type = datafile.program_type section_name = get_program_section_str(prog_type, datafile.section) + is_program_audit = datafile.is_program_audit - file_type = get_friendly_program_type(prog_type) + file_type = ( + "TANF Program Integrity Audit" + if is_program_audit + else get_friendly_program_type(prog_type) + ) stt_name = datafile.stt.name - submission_date = datafile.created_at + if datafile.created_at is not None: + stt_tz = ZoneInfo(datafile.stt.timezone or "UTC") + local_time = datafile.created_at.astimezone(stt_tz) + submission_date = local_time.strftime("%m/%d/%Y %I:%M %p %Z") + else: + submission_date = datafile.created_at fiscal_year = datafile.fiscal_year submitted_by = datafile.submitted_by @@ -135,51 +160,80 @@ def send_data_submitted_email( "status": datafile_summary.status, "has_errors": datafile_summary.status != DataFileSummary.Status.ACCEPTED, "is_aggregate": is_aggregate, + "is_program_audit": is_program_audit, "url": settings.FRONTEND_BASE_URL, } if datafile_summary.status == DataFileSummary.Status.PENDING: return - elif datafile_summary.status == DataFileSummary.Status.ACCEPTED: - subject = f"{section_name} Successfully Submitted Without Errors" - text_message = f"{file_type} has been submitted and processed without errors." - else: - subject = f"Action Required: {section_name} Contains Errors" - text_message = f"{file_type} has been submitted and processed with errors." - context.update({"subject": subject}) + text_message = ( + f"{file_type} has been submitted and processed without errors." + if datafile_summary.status == DataFileSummary.Status.ACCEPTED + else f"{file_type} has been submitted and processed with errors." + ) - match prog_type: - case ( - DataFile.ProgramType.TANF - | DataFile.ProgramType.SSP - | DataFile.ProgramType.TRIBAL - ): - if is_aggregate: - context.update(get_tanf_total_errors_context_count(datafile_summary)) - else: - context.update(get_tanf_aggregates_context_count(datafile_summary)) - - template_options = { - DataFileSummary.Status.ACCEPTED: TanfDataFileEmail.ACCEPTED.value, - DataFileSummary.Status.ACCEPTED_WITH_ERRORS: TanfDataFileEmail.ACCEPTED_WITH_ERRORS.value, - DataFileSummary.Status.PARTIALLY_ACCEPTED: TanfDataFileEmail.PARTIALLY_ACCEPTED.value, - DataFileSummary.Status.REJECTED: TanfDataFileEmail.REJECTED.value, - } - - template_path = template_options[datafile_summary.status] + if is_program_audit: + quarter_label = get_pia_quarter_label(datafile.quarter) + context.update({"quarter_label": quarter_label}) + context.update(get_tanf_aggregates_context_count(datafile_summary)) - case DataFile.ProgramType.FRA: - context.update(get_fra_aggregates_context_count(datafile_summary)) + if datafile_summary.status == DataFileSummary.Status.ACCEPTED: + subject = ( + f"{file_type}: {quarter_label} Successfully Submitted Without Errors" + ) + else: + subject = f"Action Required: {file_type}: {quarter_label} Contains Errors" - template_options = { - DataFileSummary.Status.ACCEPTED: FraDataFileEmail.ACCEPTED.value, - DataFileSummary.Status.ACCEPTED_WITH_ERRORS: FraDataFileEmail.ACCEPTED_WITH_ERRORS.value, - DataFileSummary.Status.PARTIALLY_ACCEPTED: FraDataFileEmail.PARTIALLY_ACCEPTED.value, - DataFileSummary.Status.REJECTED: FraDataFileEmail.REJECTED.value, - } + template_options = { + DataFileSummary.Status.ACCEPTED: TanfDataFileEmail.ACCEPTED.value, + DataFileSummary.Status.ACCEPTED_WITH_ERRORS: TanfDataFileEmail.ACCEPTED_WITH_ERRORS.value, + DataFileSummary.Status.PARTIALLY_ACCEPTED: TanfDataFileEmail.PARTIALLY_ACCEPTED.value, + DataFileSummary.Status.REJECTED: TanfDataFileEmail.REJECTED.value, + } - template_path = template_options[datafile_summary.status] + template_path = template_options[datafile_summary.status] + else: + if datafile_summary.status == DataFileSummary.Status.ACCEPTED: + subject = f"{section_name} Successfully Submitted Without Errors" + else: + subject = f"Action Required: {section_name} Contains Errors" + + match prog_type: + case ( + DataFile.ProgramType.TANF + | DataFile.ProgramType.SSP + | DataFile.ProgramType.TRIBAL + ): + if is_aggregate: + context.update( + get_tanf_total_errors_context_count(datafile_summary) + ) + else: + context.update(get_tanf_aggregates_context_count(datafile_summary)) + + template_options = { + DataFileSummary.Status.ACCEPTED: TanfDataFileEmail.ACCEPTED.value, + DataFileSummary.Status.ACCEPTED_WITH_ERRORS: TanfDataFileEmail.ACCEPTED_WITH_ERRORS.value, + DataFileSummary.Status.PARTIALLY_ACCEPTED: TanfDataFileEmail.PARTIALLY_ACCEPTED.value, + DataFileSummary.Status.REJECTED: TanfDataFileEmail.REJECTED.value, + } + + template_path = template_options[datafile_summary.status] + + case DataFile.ProgramType.FRA: + context.update(get_fra_aggregates_context_count(datafile_summary)) + + template_options = { + DataFileSummary.Status.ACCEPTED: FraDataFileEmail.ACCEPTED.value, + DataFileSummary.Status.ACCEPTED_WITH_ERRORS: FraDataFileEmail.ACCEPTED_WITH_ERRORS.value, + DataFileSummary.Status.PARTIALLY_ACCEPTED: FraDataFileEmail.PARTIALLY_ACCEPTED.value, + DataFileSummary.Status.REJECTED: FraDataFileEmail.REJECTED.value, + } + + template_path = template_options[datafile_summary.status] + + context.update({"subject": subject}) log( f"Data file submitted; emailing Data Analysts {list(recipients)}", diff --git a/tdrs-backend/tdpservice/email/helpers/feedback_report.py b/tdrs-backend/tdpservice/email/helpers/feedback_report.py index ce5a87943..cf12d5414 100644 --- a/tdrs-backend/tdpservice/email/helpers/feedback_report.py +++ b/tdrs-backend/tdpservice/email/helpers/feedback_report.py @@ -9,7 +9,7 @@ def send_feedback_report_available_email(report_file: ReportFile, recipients): """ - Send an email to Data Analysts when a feedback report is available. + Send an email when a feedback report is available. Parameters ---------- @@ -53,7 +53,7 @@ def send_feedback_report_available_email(report_file: ReportFile, recipients): } log( - f"Feedback report available; emailing Data Analysts {list(recipients)}", + f"Feedback report available; emailing recipients {list(recipients)}", logger_context=logger_context, ) diff --git a/tdrs-backend/tdpservice/email/helpers/test/test_data_file.py b/tdrs-backend/tdpservice/email/helpers/test/test_data_file.py index bf952d9d6..e799f072e 100644 --- a/tdrs-backend/tdpservice/email/helpers/test/test_data_file.py +++ b/tdrs-backend/tdpservice/email/helpers/test/test_data_file.py @@ -1,11 +1,14 @@ """Test functions for data_file email helper.""" +from datetime import datetime, timezone + from django.core import mail import pytest from tdpservice.data_files.models import DataFile from tdpservice.email.helpers.data_file import ( + get_pia_quarter_label, get_tanf_aggregates_context_count, get_tanf_total_errors_context_count, send_data_submitted_email, @@ -202,6 +205,76 @@ def test_send_data_submitted_email( assert mail.outbox[0].body == msg +_PIA_Q1_LABEL = "Quarter 1 (October - December)" +_PIA_FILE_TYPE = "TANF Program Integrity Audit" + + +@pytest.mark.django_db +@pytest.mark.parametrize( + "status,expected_subject,expected_text", + [ + ( + DataFileSummary.Status.ACCEPTED, + f"{_PIA_FILE_TYPE}: {_PIA_Q1_LABEL} Successfully Submitted Without Errors", + f"{_PIA_FILE_TYPE} has been submitted and processed without errors.", + ), + ( + DataFileSummary.Status.ACCEPTED_WITH_ERRORS, + f"Action Required: {_PIA_FILE_TYPE}: {_PIA_Q1_LABEL} Contains Errors", + f"{_PIA_FILE_TYPE} has been submitted and processed with errors.", + ), + ( + DataFileSummary.Status.PARTIALLY_ACCEPTED, + f"Action Required: {_PIA_FILE_TYPE}: {_PIA_Q1_LABEL} Contains Errors", + f"{_PIA_FILE_TYPE} has been submitted and processed with errors.", + ), + ( + DataFileSummary.Status.REJECTED, + f"Action Required: {_PIA_FILE_TYPE}: {_PIA_Q1_LABEL} Contains Errors", + f"{_PIA_FILE_TYPE} has been submitted and processed with errors.", + ), + ], +) +def test_send_data_submitted_email_pia( + user, stt, status, expected_subject, expected_text +): + """Test that PIA submissions use distinct subjects, text, and quarter-based templates.""" + df = DataFile( + user=user, + section=DataFile.Section.ACTIVE_CASE_DATA, + program_type=DataFile.ProgramType.TANF, + quarter=DataFile.Quarter.Q1, + year=2021, + stt=stt, + is_program_audit=True, + ) + + dfs = DataFileSummary(datafile=df, status=status) + + send_data_submitted_email(dfs, ["test@not-real.com"]) + + assert len(mail.outbox) == 1 + assert mail.outbox[0].subject == expected_subject + assert mail.outbox[0].body == expected_text + + +class TestGetPiaQuarterLabel: + """Tests for get_pia_quarter_label.""" + + @pytest.mark.parametrize( + "quarter,expected", + [ + (DataFile.Quarter.Q1, "Quarter 1 (October - December)"), + (DataFile.Quarter.Q2, "Quarter 2 (January - March)"), + (DataFile.Quarter.Q3, "Quarter 3 (April - June)"), + (DataFile.Quarter.Q4, "Quarter 4 (July - September)"), + ], + ) + def test_quarter_labels(self, quarter, expected): + """Test that all quarters map to correct human-readable labels.""" + assert get_pia_quarter_label(quarter) == expected + + class TestGetTanfAggregatesContextCount: """Tests for get_tanf_aggregates_context_count.""" @@ -348,3 +421,101 @@ def test_send_stuck_file_email(user, stt): == "List of submitted files with pending status after 1 hour" ) assert mail.outbox[0].body == "The system has detected stuck files." + + +@pytest.mark.django_db +def test_submission_date_formatted_in_stt_timezone(user, stt): + """Test that the email submission_date is formatted in the STT's timezone.""" + stt.timezone = "America/Chicago" + stt.save() + + df = DataFile.objects.create( + user=user, + section=DataFile.Section.ACTIVE_CASE_DATA, + program_type=DataFile.ProgramType.TANF, + quarter="Q1", + year=2021, + version=1, + stt=stt, + ) + # Override created_at to a known UTC time (2024-01-15 18:00 UTC = 12:00 PM CST) + DataFile.objects.filter(pk=df.pk).update( + created_at=datetime(2024, 1, 15, 18, 0, 0, tzinfo=timezone.utc) + ) + df.refresh_from_db() + + dfs = DataFileSummary.objects.create( + datafile=df, + status=DataFileSummary.Status.ACCEPTED, + ) + + send_data_submitted_email(dfs, ["test@not-real.com"]) + + assert len(mail.outbox) == 1 + body = mail.outbox[0].alternatives[0][0] # HTML body + assert "01/15/2024 12:00 PM CST" in body + + +@pytest.mark.django_db +def test_submission_date_formatted_in_eastern_timezone(user, stt): + """Test that Eastern timezone formatting includes EST/EDT label.""" + stt.timezone = "America/New_York" + stt.save() + + df = DataFile.objects.create( + user=user, + section=DataFile.Section.ACTIVE_CASE_DATA, + program_type=DataFile.ProgramType.TANF, + quarter="Q1", + year=2021, + version=1, + stt=stt, + ) + # 2024-01-15 18:00 UTC = 1:00 PM EST (January, so EST not EDT) + DataFile.objects.filter(pk=df.pk).update( + created_at=datetime(2024, 1, 15, 18, 0, 0, tzinfo=timezone.utc) + ) + df.refresh_from_db() + + dfs = DataFileSummary.objects.create( + datafile=df, + status=DataFileSummary.Status.ACCEPTED, + ) + + send_data_submitted_email(dfs, ["test@not-real.com"]) + + assert len(mail.outbox) == 1 + body = mail.outbox[0].alternatives[0][0] + assert "01/15/2024 01:00 PM EST" in body + + +@pytest.mark.django_db +def test_submission_date_utc_fallback_when_no_timezone(user, stt): + """Test that submission_date falls back to UTC when STT has no timezone.""" + stt.timezone = "" + stt.save() + + df = DataFile.objects.create( + user=user, + section=DataFile.Section.ACTIVE_CASE_DATA, + program_type=DataFile.ProgramType.TANF, + quarter="Q1", + year=2021, + version=1, + stt=stt, + ) + DataFile.objects.filter(pk=df.pk).update( + created_at=datetime(2024, 1, 15, 18, 0, 0, tzinfo=timezone.utc) + ) + df.refresh_from_db() + + dfs = DataFileSummary.objects.create( + datafile=df, + status=DataFileSummary.Status.ACCEPTED, + ) + + send_data_submitted_email(dfs, ["test@not-real.com"]) + + assert len(mail.outbox) == 1 + body = mail.outbox[0].alternatives[0][0] + assert "01/15/2024 06:00 PM UTC" in body diff --git a/tdrs-backend/tdpservice/email/templates/tanf/accepted.html b/tdrs-backend/tdpservice/email/templates/tanf/accepted.html index e883cad80..f28b50af1 100644 --- a/tdrs-backend/tdpservice/email/templates/tanf/accepted.html +++ b/tdrs-backend/tdpservice/email/templates/tanf/accepted.html @@ -1,7 +1,7 @@ {% extends '../datafile_base.html' %} {% block message %} -Your {{ file_type }} data files submitted for {{ stt_name }} on {{ submission_date }} in Fiscal Year {{ fiscal_year }} have been processed and accepted. +Your {{ file_type }} data file submitted for {{ stt_name }} on {{ submission_date }} in Fiscal Year {{ fiscal_year }} has been processed and accepted. {% endblock %} {% block table %} @@ -11,7 +11,7 @@ - Section + {% if is_program_audit %}Fiscal Quarter{% else %}Section{% endif %} Submitted By Status {% if is_aggregate %} @@ -23,7 +23,7 @@ - {{ section_name }} + {% if is_program_audit %}{{ quarter_label }}{% else %}{{ section_name }}{% endif %} {{ submitted_by }} {{ status }} {% if is_aggregate %} diff --git a/tdrs-backend/tdpservice/email/templates/tanf/accepted_with_errors.html b/tdrs-backend/tdpservice/email/templates/tanf/accepted_with_errors.html index 12cf48dbf..4cf371302 100644 --- a/tdrs-backend/tdpservice/email/templates/tanf/accepted_with_errors.html +++ b/tdrs-backend/tdpservice/email/templates/tanf/accepted_with_errors.html @@ -2,9 +2,9 @@ {% block message %} {% if is_aggregate %} -Your {{ file_type }} data files submitted for {{ stt_name }} on {{ submission_date }} in Fiscal Year {{ fiscal_year }} have been processed, but your data contain errors. Review the error report(s) for more details, and resubmit your data files once the errors have been corrected. +Your {{ file_type }} data file submitted for {{ stt_name }} on {{ submission_date }} in Fiscal Year {{ fiscal_year }} has been processed, but your data contains errors. Review the error report for more details, and resubmit your data file once the errors have been corrected. {% else %} -Your {{ file_type }} data files submitted for {{ stt_name }} on {{ submission_date }} in Fiscal Year {{ fiscal_year }} have been processed, but some cases within your data contain errors. Review the error report(s) for more details, and resubmit your data files once the errors have been corrected. +Your {{ file_type }} data file submitted for {{ stt_name }} on {{ submission_date }} in Fiscal Year {{ fiscal_year }} has been processed, but some cases within your data contain errors. Review the error report for more details, and resubmit your data file once the errors have been corrected. {% endif %} {% endblock %} @@ -15,7 +15,7 @@ - Section + {% if is_program_audit %}Fiscal Quarter{% else %}Section{% endif %} Submitted By Status {% if is_aggregate %} @@ -29,7 +29,7 @@ - {{ section_name }} + {% if is_program_audit %}{{ quarter_label }}{% else %}{{ section_name }}{% endif %} {{ submitted_by }} {{ status }} {% if is_aggregate %} diff --git a/tdrs-backend/tdpservice/email/templates/tanf/partially_accepted.html b/tdrs-backend/tdpservice/email/templates/tanf/partially_accepted.html index d30f72ccc..f2c368b5b 100644 --- a/tdrs-backend/tdpservice/email/templates/tanf/partially_accepted.html +++ b/tdrs-backend/tdpservice/email/templates/tanf/partially_accepted.html @@ -1,7 +1,7 @@ {% extends '../datafile_base.html' %} {% block message %} -Your {{ file_type }} data files submitted for {{ stt_name }} on {{ submission_date }} in Fiscal Year {{ fiscal_year }} were unable to be fully processed. Review the error report(s) for more details, and resubmit your data files once the errors have been corrected. +Your {{ file_type }} data file submitted for {{ stt_name }} on {{ submission_date }} in Fiscal Year {{ fiscal_year }} was unable to be fully processed. Review the error report for more details, and resubmit your data file once the errors have been corrected. {% endblock %} {% block table %} @@ -11,7 +11,7 @@ - Section + {% if is_program_audit %}Fiscal Quarter{% else %}Section{% endif %} Submitted By Status Cases Without Errors @@ -22,7 +22,7 @@ - {{ section_name }} + {% if is_program_audit %}{{ quarter_label }}{% else %}{{ section_name }}{% endif %} {{ submitted_by }} {{ status }} {{ cases_without_errors }} diff --git a/tdrs-backend/tdpservice/email/templates/tanf/rejected.html b/tdrs-backend/tdpservice/email/templates/tanf/rejected.html index 9812d5bc0..f155501d5 100644 --- a/tdrs-backend/tdpservice/email/templates/tanf/rejected.html +++ b/tdrs-backend/tdpservice/email/templates/tanf/rejected.html @@ -1,7 +1,7 @@ {% extends '../datafile_base.html' %} {% block message %} -Your {{ file_type }} data files submitted for {{ stt_name }} on {{ submission_date }} in Fiscal Year {{ fiscal_year }} were unable to be processed. Review the error report(s) for more details, and resubmit your data files once the errors have been corrected. +Your {{ file_type }} data file submitted for {{ stt_name }} on {{ submission_date }} in Fiscal Year {{ fiscal_year }} was unable to be processed. Review the error report for more details, and resubmit your data file once the errors have been corrected. {% endblock %} {% block table %} @@ -11,7 +11,7 @@ - Section + {% if is_program_audit %}Fiscal Quarter{% else %}Section{% endif %} Submitted By Status {% if not is_aggregate %} @@ -24,7 +24,7 @@ - {{ section_name }} + {% if is_program_audit %}{{ quarter_label }}{% else %}{{ section_name }}{% endif %} {{ submitted_by }} {{ status }} {% if not is_aggregate %} diff --git a/tdrs-backend/tdpservice/fixtures/cypress/data_files.json b/tdrs-backend/tdpservice/fixtures/cypress/data_files.json index df2c6708c..4b373c0d6 100644 --- a/tdrs-backend/tdpservice/fixtures/cypress/data_files.json +++ b/tdrs-backend/tdpservice/fixtures/cypress/data_files.json @@ -9,8 +9,9 @@ "created_at": "2025-02-24T13:48:55.022Z", "quarter": "Q1", "year": 2023, - "section": "Active Case Data", "program_type": "TAN", + "section": "Active Case Data", + "is_program_audit": false, "version": 4, "user": "bfcdc9bc-0b54-4b53-8f04-8aed0b0a9459", "stt": 14, @@ -18,12 +19,73 @@ "s3_versioning_id": "ad9b635b-6bf9-4e81-90c7-6f0250e5456e" } }, + { + "model": "data_files.datafile", + "pk": 10, + "fields": { + "original_filename": "fra.csv", + "slug": "3aac1f7e-4987-49af-b1d8-f091410feecd", + "extension": "txt", + "created_at": "2025-08-07T17:30:49.766Z", + "quarter": "Q2", + "year": 2024, + "program_type": "FRA", + "section": "Work Outcomes of TANF Exiters", + "is_program_audit": false, + "version": 1, + "user": "7b7c4778-17e4-49a1-83b8-0e6a962972f7", + "stt": 3, + "file": "data_files/2024/Q2/3/Work Outcomes of TANF Exiters/fra.csv", + "s3_versioning_id": "a9a751cb-a986-4e44-911f-40c42f8d4fab" + } + }, + { + "model": "data_files.datafile", + "pk": 11, + "fields": { + "original_filename": "small_ssp_section1.txt", + "slug": "d3273855-8d48-4d74-99d6-78451bff4c35", + "extension": "txt", + "created_at": "2025-08-07T20:27:56.073Z", + "quarter": "Q1", + "year": 2024, + "program_type": "SSP", + "section": "Active Case Data", + "is_program_audit": false, + "version": 1, + "user": "7b7c4778-17e4-49a1-83b8-0e6a96297888", + "stt": 26, + "file": "data_files/2024/Q1/26/SSP Active Case Data/small_ssp_section1.txt", + "s3_versioning_id": "c1cba040-5f25-4568-a831-80a5a34ee450" + } + }, + { + "model": "data_files.datafile", + "pk": 12, + "fields": { + "original_filename": "PI_Audit_space-fill.txt", + "slug": "cdd1bb40-ec99-4fa2-ab87-a5f8394145c4", + "extension": "txt", + "created_at": "2026-03-23T15:34:23.390Z", + "quarter": "Q1", + "year": 2024, + "program_type": "TAN", + "section": "Active Case Data", + "is_program_audit": true, + "version": 1, + "user": "bfcdc9bc-0b54-4b53-8f04-8aed0b0a9459", + "stt": 5, + "file": "data_files/2024/Q1/5/TAN/Active Case Data/PI_Audit_space-fill.txt", + "s3_versioning_id": "093fd4c0-3d67-418f-a4be-c4ec0574e263" + } + }, { "model": "parsers.datafilesummary", "pk": 1, "fields": { "status": "Accepted with Errors", "datafile": 9, + "error_report": "", "case_aggregates": { "months": [ { @@ -48,87 +110,78 @@ "total_number_of_records_created": 16 } }, - { - "model": "data_files.datafile", - "pk": 10, - "fields": { - "original_filename": "fra.csv", - "slug": "3aac1f7e-4987-49af-b1d8-f091410feecd", - "extension": "txt", - "created_at": "2025-08-07T17:30:49.766Z", - "quarter": "Q2", - "year": 2024, - "section": "Work Outcomes of TANF Exiters", - "program_type": "FRA", - "version": 1, - "user": "7b7c4778-17e4-49a1-83b8-0e6a962972f7", - "stt": 3, - "file": "data_files/2024/Q2/3/Work Outcomes of TANF Exiters/fra.csv", - "s3_versioning_id": "a9a751cb-a986-4e44-911f-40c42f8d4fab" - } - }, - { "model": "parsers.datafilesummary", "pk": 2, "fields": { "status": "Partially Accepted with Errors", "datafile": 10, - "case_aggregates": { - "total_errors": 8 - }, + "error_report": "", + "case_aggregates": { "total_errors": 8 }, "total_number_of_records_in_file": 11, "total_number_of_records_created": 5 } }, { - "model": "data_files.datafile", - "pk": 11, + "model": "parsers.datafilesummary", + "pk": 3, "fields": { - "original_filename": "small_ssp_section1.txt", - "slug": "d3273855-8d48-4d74-99d6-78451bff4c35", - "extension": "txt", - "created_at": "2025-08-07T20:27:56.073Z", - "quarter": "Q1", - "year": 2024, - "section": "Active Case Data", - "program_type": "SSP", - "version": 1, - "user": "7b7c4778-17e4-49a1-83b8-0e6a96297888", - "stt": 26, - "file": "data_files/2024/Q1/26/SSP Active Case Data/small_ssp_section1.txt", - "s3_versioning_id": "c1cba040-5f25-4568-a831-80a5a34ee450" + "status": "Partially Accepted with Errors", + "datafile": 11, + "error_report": "data_files/2024/Q1/26/SSP Active Case Data/small_ssp_section1.txt_error_report_c1cba040-5f25-4568-a831-80a5a34ee450.xlsx", + "case_aggregates": { + "months": [ + { + "month": "Oct", + "accepted_with_errors": 5, + "accepted_without_errors": 0 + }, + { + "month": "Nov", + "accepted_with_errors": 0, + "accepted_without_errors": 0 + }, + { + "month": "Dec", + "accepted_with_errors": 0, + "accepted_without_errors": 0 + } + ], + "rejected": 1 + }, + "total_number_of_records_in_file": 19, + "total_number_of_records_created": 19 } - }, - { + }, + { "model": "parsers.datafilesummary", - "pk": 3, + "pk": 4, "fields": { - "status": "Partially Accepted with Errors", - "datafile": 11, - "error_report": "data_files/2024/Q1/26/SSP Active Case Data/small_ssp_section1.txt_error_report_c1cba040-5f25-4568-a831-80a5a34ee450.xlsx", - "case_aggregates": { - "months": [ - { - "month": "Oct", - "accepted_with_errors": 5, - "accepted_without_errors": 0 - }, - { - "month": "Nov", - "accepted_with_errors": 0, - "accepted_without_errors": 0 - }, - { - "month": "Dec", - "accepted_with_errors": 0, - "accepted_without_errors": 0 - } - ], - "rejected": 1 - }, - "total_number_of_records_in_file": 19, - "total_number_of_records_created": 19 + "status": "Accepted with Errors", + "datafile": 12, + "error_report": "data_files/2024/Q1/5/TAN/Active Case Data/PI_Audit_space-fill.txt_error_report_093fd4c0-3d67-418f-a4be-c4ec0574e263.xlsx", + "case_aggregates": { + "months": [ + { + "month": "Oct", + "accepted_with_errors": 1, + "accepted_without_errors": 0 + }, + { + "month": "Nov", + "accepted_with_errors": 0, + "accepted_without_errors": 0 + }, + { + "month": "Dec", + "accepted_with_errors": 0, + "accepted_without_errors": 0 + } + ], + "rejected": 0 + }, + "total_number_of_records_in_file": 5, + "total_number_of_records_created": 5 } - } + } ] diff --git a/tdrs-backend/tdpservice/fixtures/cypress/feature_flags.json b/tdrs-backend/tdpservice/fixtures/cypress/feature_flags.json new file mode 100644 index 000000000..1cd4d3407 --- /dev/null +++ b/tdrs-backend/tdpservice/fixtures/cypress/feature_flags.json @@ -0,0 +1,14 @@ +[ + { + "model": "core.featureflag", + "pk": 1, + "fields": { + "feature_name": "program-integrity-audit", + "enabled": true, + "config": { "maxYear": 2025, "minYear": 2023 }, + "description": "", + "created_at": "2026-03-19T13:30:04.790Z", + "updated_at": "2026-03-19T13:30:04.790Z" + } + } +] diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t4.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t4.py index ed7d63306..b44ccc01b 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t4.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t4.py @@ -144,7 +144,7 @@ startIndex=34, endIndex=35, required=True, - validators=[category2.isBetween(1, 2, inclusive=True)], + validators=[category2.isBetween(0, 2, inclusive=True)], ), Field( item="13", @@ -154,7 +154,7 @@ startIndex=35, endIndex=36, required=True, - validators=[category2.isBetween(1, 3, inclusive=True)], + validators=[category2.isBetween(0, 3, inclusive=True)], ), Field( item="-1", diff --git a/tdrs-backend/tdpservice/reports/tasks.py b/tdrs-backend/tdpservice/reports/tasks.py index d18e2e6a7..f688be955 100644 --- a/tdrs-backend/tdpservice/reports/tasks.py +++ b/tdrs-backend/tdpservice/reports/tasks.py @@ -5,6 +5,7 @@ import zipfile from django.core.files.base import ContentFile +from django.db.models import Q from django.utils import timezone from celery import shared_task @@ -157,21 +158,29 @@ def _extract_and_validate_structure(source: ReportSource, zip_file: zipfile.ZipF def _send_report_file_notification(report_file: ReportFile): """ - Send email notification to all Data Analysts for the ReportFile's STT. + Send email notification to Data Analysts and Regional Staff for the ReportFile's STT. + + Data Analysts are notified if their assigned STT matches the report's STT. + Regional Staff are notified if their region includes the report's STT. Parameters ---------- report_file: The ReportFile that was just created """ - # Query all approved Data Analysts for this STT - data_analysts = User.objects.filter( - stt=report_file.stt, - account_approval_status=AccountApprovalStatusChoices.APPROVED, - groups__name="Data Analyst", - ).values_list("email", flat=True).distinct() - - if data_analysts: - send_feedback_report_available_email(report_file, list(data_analysts)) + # Data Analysts assigned to this STT + data_analyst_q = Q(stt=report_file.stt, groups__name="Data Analyst") + # Regional Staff whose region includes this STT + regional_staff_q = Q(regions=report_file.stt.region, groups__name="OFA Regional Staff") + + recipients = list( + User.objects.filter( + data_analyst_q | regional_staff_q, + account_approval_status=AccountApprovalStatusChoices.APPROVED, + ).values_list("email", flat=True).distinct() + ) + + if recipients: + send_feedback_report_available_email(report_file, recipients) def _process_stt_folder( diff --git a/tdrs-backend/tdpservice/reports/test/test_tasks.py b/tdrs-backend/tdpservice/reports/test/test_tasks.py index 59ce874bc..875e284cd 100644 --- a/tdrs-backend/tdpservice/reports/test/test_tasks.py +++ b/tdrs-backend/tdpservice/reports/test/test_tasks.py @@ -594,3 +594,87 @@ def test_only_sends_to_approved_data_analysts( assert "approved@example.com" in recipients assert "pending@example.com" not in recipients + + @patch("tdpservice.reports.tasks.send_feedback_report_available_email") + @patch("tdpservice.reports.tasks.timezone.now") + def test_sends_to_regional_staff_in_stt_region( + self, mock_now, mock_send_email, ofa_admin + ): + """Test that email is sent to Regional Staff whose region includes the STT.""" + mock_now.return_value = timezone.make_aware(datetime(2025, 2, 1)) + + from django.contrib.auth.models import Group + + from tdpservice.stts.models import STT, Region + from tdpservice.users.models import AccountApprovalStatusChoices, User + + # Create region and STT + region = Region.objects.create(id=9014, name="Test Region 14") + stt = STT.objects.create( + id=8014, + stt_code="01", + name="Test STT Regional Email", + region=region, + postal_code="TR", + type="STATE", + ) + + data_analyst_group, _ = Group.objects.get_or_create(name="Data Analyst") + regional_group, _ = Group.objects.get_or_create(name="OFA Regional Staff") + + # Create approved Data Analyst for this STT + analyst = User.objects.create( + username="analyst_regional_test", + email="analyst_regional@example.com", + stt=stt, + account_approval_status=AccountApprovalStatusChoices.APPROVED, + ) + analyst.groups.add(data_analyst_group) + + # Create approved Regional Staff in the same region + regional_user = User.objects.create( + username="regional_staff_test", + email="regional@example.com", + account_approval_status=AccountApprovalStatusChoices.APPROVED, + ) + regional_user.groups.add(regional_group) + regional_user.regions.add(region) + + # Create Regional Staff in a different region (should NOT receive email) + other_region = Region.objects.create(id=9015, name="Test Region 15") + other_regional = User.objects.create( + username="other_regional_test", + email="other_regional@example.com", + account_approval_status=AccountApprovalStatusChoices.APPROVED, + ) + other_regional.groups.add(regional_group) + other_regional.regions.add(other_region) + + structure = {"FY2025": {"RO1": {"F1": ["report.pdf"]}}} + zip_buffer = create_nested_zip(structure, "FY2025_test") + + from django.core.files.uploadedfile import SimpleUploadedFile + + uploaded_file = SimpleUploadedFile( + "report_source.zip", zip_buffer.read(), content_type="application/zip" + ) + + source = ReportSource.objects.create( + uploaded_by=ofa_admin, + original_filename="report_source.zip", + slug="report_source.zip", + file=uploaded_file, + year=2025, + date_extracted_on=date(2025, 1, 31), + ) + + process_report_source(source.id) + + # Verify both analyst and regional staff received email + mock_send_email.assert_called_once() + call_args = mock_send_email.call_args[0] + recipients = call_args[1] + + assert "analyst_regional@example.com" in recipients + assert "regional@example.com" in recipients + assert "other_regional@example.com" not in recipients diff --git a/tdrs-backend/tdpservice/scheduling/datafile_retention_tasks.py b/tdrs-backend/tdpservice/scheduling/datafile_retention_tasks.py index 620045981..097ac9b9a 100644 --- a/tdrs-backend/tdpservice/scheduling/datafile_retention_tasks.py +++ b/tdrs-backend/tdpservice/scheduling/datafile_retention_tasks.py @@ -2,7 +2,6 @@ from __future__ import absolute_import -import itertools import logging from datetime import datetime @@ -11,7 +10,6 @@ from tdpservice.core.utils import log from tdpservice.data_files.models import DataFile from tdpservice.search_indexes.utils import delete_records, get_log_context -from tdpservice.stts.models import STT from tdpservice.users.models import User logger = logging.getLogger(__name__) @@ -29,13 +27,8 @@ def remove_all_old_versions(): level="info", logger_context=log_context, ) - stts = STT.objects.all() min_year = 2019 # TDP didn't exist before this max_year = datetime.now().year - years = [year for year in range(min_year, max_year + 1)] - quarters = DataFile.Quarter - program_types = DataFile.ProgramType - sections = DataFile.Section num_exceptions = 0 num_out_of_range = DataFile.objects.exclude( @@ -49,26 +42,36 @@ def remove_all_old_versions(): logger_context=log_context, ) - for year, quarter, program_type, section, stt in itertools.product( - years, quarters, program_types, sections, stts - ): + # Query only the distinct file groupings that actually exist in the database, + # instead of iterating over the full Cartesian product of all possible combinations. + existing_groupings = ( + DataFile.objects.filter(year__range=(min_year, max_year)) + .values_list("year", "quarter", "program_type", "section", "stt") + .distinct() + ) + + # Collect all old-version file IDs across all groupings, then delete in one batch. + all_old_file_ids = [] + for year, quarter, program_type, section, stt_id in existing_groupings: + files = DataFile.objects.filter( + year=year, + quarter=quarter, + program_type=program_type, + section=section, + stt_id=stt_id, + ) + if files.count() <= 1: + continue + newest_file = files.latest("version") + old_ids = list(files.exclude(id=newest_file.id).values_list("id", flat=True)) + all_old_file_ids.extend(old_ids) + + if all_old_file_ids: try: - files = DataFile.objects.filter( - year=year, - quarter=quarter, - program_type=program_type, - section=section, - stt=stt, - ) - if files.count() == 0: - continue - newest_file = files.latest("version") - ids = files.exclude(id=newest_file.id).values_list("id", flat=True) - delete_records(ids, log_context) + delete_records(all_old_file_ids, log_context) except Exception as e: log( - f"Failed to delete old versions of file for: Year:{year}, Quarter:{quarter}, " - f"Program Type:{program_type}, Section:{section}, STT:{stt.name}", + f"Failed to delete old versions of {len(all_old_file_ids)} files.", level="error", logger_context=log_context, ) diff --git a/tdrs-backend/tdpservice/scheduling/test/test_datafile_retention_tasks.py b/tdrs-backend/tdpservice/scheduling/test/test_datafile_retention_tasks.py index 3c5c1fdfb..5ff263546 100644 --- a/tdrs-backend/tdpservice/scheduling/test/test_datafile_retention_tasks.py +++ b/tdrs-backend/tdpservice/scheduling/test/test_datafile_retention_tasks.py @@ -344,19 +344,15 @@ def test_logs_warning_for_future_year_files(self, mock_log, stt, user): assert "manual cleanup" in warning_message.lower() @patch("tdpservice.scheduling.datafile_retention_tasks.log") - def test_exception_handling_continues_processing( - self, mock_log, stt, second_stt, user - ): - """Test that exception handling allows processing to continue. + def test_exception_handling_logs_error(self, mock_log, stt, user): + """Test that exception handling logs the error gracefully. - Given: delete_records raises an exception for one file group + Given: delete_records raises an exception When: remove_all_old_versions is called - Then: Processing should continue for other file groups and log errors + Then: The error should be logged and the function should not raise """ current_year = datetime.now().year - stt_2 = second_stt - # Create files for two different STTs with multiple versions DataFileFactory.create( year=current_year, quarter="Q1", @@ -376,40 +372,9 @@ def test_exception_handling_continues_processing( version=2, ) - DataFileFactory.create( - year=current_year, - quarter="Q1", - program_type="TAN", - section="Active Case Data", - stt=stt_2, - user=user, - version=1, - ) - DataFileFactory.create( - year=current_year, - quarter="Q1", - program_type="TAN", - section="Active Case Data", - stt=stt_2, - user=user, - version=2, - ) - - # Patch delete_records to raise an exception on first call only - call_count = [0] - original_delete_records = __import__( - "tdpservice.search_indexes.utils", fromlist=["delete_records"] - ).delete_records - - def side_effect(file_ids, log_context): - call_count[0] += 1 - if call_count[0] == 1: - raise Exception("Test exception") - return original_delete_records(file_ids, log_context) - with patch( "tdpservice.scheduling.datafile_retention_tasks.delete_records", - side_effect=side_effect, + side_effect=Exception("Test exception"), ): # Should not raise an exception - the function should handle it remove_all_old_versions() diff --git a/tdrs-backend/tdpservice/stts/management/commands/data/states.csv b/tdrs-backend/tdpservice/stts/management/commands/data/states.csv index ef902005d..0f3982c72 100644 --- a/tdrs-backend/tdpservice/stts/management/commands/data/states.csv +++ b/tdrs-backend/tdpservice/stts/management/commands/data/states.csv @@ -1,52 +1,52 @@ -Code,Name,Region,STT_CODE,Sample,SSP,SSN_Encrypted,filenames -AL,Alabama,4,01,0,0,0,"{'Active Case Data': 'ADS.E2J.NDM1.TS01', 'Closed Case Data': 'ADS.E2J.NDM2.TS01', 'Aggregate Data': 'ADS.E2J.NDM3.TS01'}" -AK,Alaska,10,02,0,0,0,"{'Active Case Data': 'ADS.E2J.NDM1.TS02', 'Closed Case Data': 'ADS.E2J.NDM2.TS02', 'Aggregate Data': 'ADS.E2J.NDM3.TS02'}" -AZ,Arizona,9,04,0,0,0,"{'Active Case Data': 'ADS.E2J.FTP1.TS04', 'Closed Case Data': 'ADS.E2J.FTP2.TS04', 'Aggregate Data': 'ADS.E2J.FTP3.TS04'}" -AR,Arkansas,6,05,0,0,1,"{'Active Case Data': 'ADS.E2J.FTP1.TS05', 'Closed Case Data': 'ADS.E2J.FTP2.TS05', 'Aggregate Data': 'ADS.E2J.FTP3.TS05'}" -CA,California,9,06,1,1,0,"{'Active Case Data': 'ADS.E2J.FTP1.TS06', 'Closed Case Data': 'ADS.E2J.FTP2.TS06', 'Aggregate Data': 'ADS.E2J.FTP3.TS06', 'Stratum Data': 'ADS.E2J.FTP4.TS06', 'SSP Active Case Data': 'ADS.E2J.FTP1.MS06', 'SSP Closed Case Data': 'ADS.E2J.FTP2.MS06', 'SSP Aggregate Data': 'ADS.E2J.FTP3.MS06', 'SSP Stratum Data': 'ADS.E2J.FTP4.MS06'}" -CO,Colorado,8,08,1,0,0,"{'Active Case Data': 'ADS.E2J.FTP1.TS08', 'Closed Case Data': 'ADS.E2J.FTP2.TS08', 'Aggregate Data': 'ADS.E2J.FTP3.TS08', 'Stratum Data': 'ADS.E2J.FTP4.TS08'}" -CT,Connecticut,1,09,1,0,1,"{'Active Case Data': 'ADS.E2J.NDM1.TS09', 'Closed Case Data': 'ADS.E2J.NDM2.TS09', 'Aggregate Data': 'ADS.E2J.NDM3.TS09', 'Stratum Data': 'ADS.E2J.NDM4.TS09'}" -DE,Delaware,3,10,0,0,0,"{'Active Case Data': 'ADS.E2J.NDM1.TS10', 'Closed Case Data': 'ADS.E2J.NDM2.TS10', 'Aggregate Data': 'ADS.E2J.NDM3.TS10'}" -DC,District of Columbia,3,11,0,0,0,"{'Active Case Data': 'ADS.E2J.FTP1.TS11', 'Closed Case Data': 'ADS.E2J.FTP2.TS11', 'Aggregate Data': 'ADS.E2J.FTP3.TS11'}" -FL,Florida,4,12,1,0,0,"{'Active Case Data': 'ADS.E2J.FTP1.TS12', 'Closed Case Data': 'ADS.E2J.FTP2.TS12', 'Aggregate Data': 'ADS.E2J.FTP3.TS12', 'Stratum Data': 'ADS.E2J.FTP4.TS12'}" -GA,Georgia,4,13,0,0,0,"{'Active Case Data': 'ADS.E2J.NDM1.TS13', 'Closed Case Data': 'ADS.E2J.NDM2.TS13', 'Aggregate Data': 'ADS.E2J.NDM3.TS13'}" -HI,Hawaii,9,15,0,0,0,"{'Active Case Data': 'ADS.E2J.NDM1.TS15', 'Closed Case Data': 'ADS.E2J.NDM2.TS15', 'Aggregate Data': 'ADS.E2J.NDM3.TS15'}" -ID,Idaho,10,16,0,0,0,"{'Active Case Data': 'ADS.E2J.NDM1.TS16', 'Closed Case Data': 'ADS.E2J.NDM2.TS16', 'Aggregate Data': 'ADS.E2J.NDM3.TS16'}" -IL,Illinois,5,17,1,0,1,"{'Active Case Data': 'ADS.E2J.FTP1.TS17', 'Closed Case Data': 'ADS.E2J.FTP2.TS17', 'Aggregate Data': 'ADS.E2J.FTP3.TS17', 'Stratum Data': 'ADS.E2J.FTP4.TS17'}" -IN,Indiana,5,18,0,1,0,"{'Active Case Data': 'ADS.E2J.NDM1.TS18', 'Closed Case Data': 'ADS.E2J.NDM2.TS18', 'Aggregate Data': 'ADS.E2J.NDM3.TS18', 'SSP Active Case Data': 'ADS.E2J.NDM1.MS18', 'SSP Closed Case Data': 'ADS.E2J.NDM2.MS18', 'SSP Aggregate Data': 'ADS.E2J.NDM3.MS18'}" -IA,Iowa,7,19,0,1,0,"{'Active Case Data': 'ADS.E2J.NDM1.TS19', 'Closed Case Data': 'ADS.E2J.NDM2.TS19', 'Aggregate Data': 'ADS.E2J.NDM3.TS19', 'SSP Active Case Data': 'ADS.E2J.NDM1.MS19', 'SSP Closed Case Data': 'ADS.E2J.NDM2.MS19', 'SSP Aggregate Data': 'ADS.E2J.NDM3.MS19'}" -KS,Kansas,7,20,1,0,0,"{'Active Case Data': 'ADS.E2J.NDM1.TS20', 'Closed Case Data': 'ADS.E2J.NDM2.TS20', 'Aggregate Data': 'ADS.E2J.NDM3.TS20', 'Stratum Data': 'ADS.E2J.NDM4.TS20'}" -KY,Kentucky,4,21,0,0,1,"{'Active Case Data': 'ADS.E2J.FTP1.TS21', 'Closed Case Data': 'ADS.E2J.FTP2.TS21', 'Aggregate Data': 'ADS.E2J.FTP3.TS21'}" -LA,Louisiana,6,22,0,0,0,"{'Active Case Data': 'ADS.E2J.NDM1.TS22', 'Closed Case Data': 'ADS.E2J.NDM2.TS22', 'Aggregate Data': 'ADS.E2J.NDM3.TS22'}" -ME,Maine,1,23,0,1,0,"{'Active Case Data': 'ADS.E2J.NDM1.TS23', 'Closed Case Data': 'ADS.E2J.NDM2.TS23', 'Aggregate Data': 'ADS.E2J.NDM3.TS23', 'SSP Active Case Data': 'ADS.E2J.NDM1.MS23', 'SSP Closed Case Data': 'ADS.E2J.NDM2.MS23', 'SSP Aggregate Data': 'ADS.E2J.NDM3.MS23'}" -MD,Maryland,3,24,1,1,0,"{'Active Case Data': 'ADS.E2J.NDM1.TS24', 'Closed Case Data': 'ADS.E2J.NDM2.TS24', 'Aggregate Data': 'ADS.E2J.NDM3.TS24', 'Stratum Data': 'ADS.E2J.NDM4.TS24', 'SSP Active Case Data': 'ADS.E2J.NDM1.MS24', 'SSP Closed Case Data': 'ADS.E2J.NDM2.MS24', 'SSP Aggregate Data': 'ADS.E2J.NDM3.MS24', 'SSP Stratum Data': 'ADS.E2J.NDM4.MS24'}" -MA,Massachusetts,1,25,1,1,1,"{'Active Case Data': 'ADS.E2J.FTP1.TS25', 'Closed Case Data': 'ADS.E2J.FTP2.TS25', 'Aggregate Data': 'ADS.E2J.FTP3.TS25', 'Stratum Data': 'ADS.E2J.FTP4.TS25', 'SSP Active Case Data': 'ADS.E2J.FTP1.MS25', 'SSP Closed Case Data': 'ADS.E2J.FTP2.MS25', 'SSP Aggregate Data': 'ADS.E2J.FTP3.MS25', 'SSP Stratum Data': 'ADS.E2J.FTP4.MS25'}" -MI,Michigan,5,26,1,0,0,"{'Active Case Data': 'ADS.E2J.NDM1.TS26', 'Closed Case Data': 'ADS.E2J.NDM2.TS26', 'Aggregate Data': 'ADS.E2J.NDM3.TS26', 'Stratum Data': 'ADS.E2J.NDM4.TS26'}" -MN,Minnesota,5,27,0,0,0,"{'Active Case Data': 'ADS.E2J.NDM1.TS27', 'Closed Case Data': 'ADS.E2J.NDM2.TS27', 'Aggregate Data': 'ADS.E2J.NDM3.TS27'}" -MS,Mississippi,4,28,1,0,1,"{'Active Case Data': 'ADS.E2J.NDM1.TS28', 'Closed Case Data': 'ADS.E2J.NDM2.TS28', 'Aggregate Data': 'ADS.E2J.NDM3.TS28', 'Stratum Data': 'ADS.E2J.NDM4.TS28'}" -MO,Missouri,7,29,0,1,0,"{'Active Case Data': 'ADS.E2J.NDM1.TS29', 'Closed Case Data': 'ADS.E2J.NDM2.TS29', 'Aggregate Data': 'ADS.E2J.NDM3.TS29', 'SSP Active Case Data': 'ADS.E2J.NDM1.MS29', 'SSP Closed Case Data': 'ADS.E2J.NDM2.MS29', 'SSP Aggregate Data': 'ADS.E2J.NDM3.MS29'}" -MT,Montana,8,30,0,0,0,"{'Active Case Data': 'ADS.E2J.NDM1.TS30', 'Closed Case Data': 'ADS.E2J.NDM2.TS30', 'Aggregate Data': 'ADS.E2J.NDM3.TS30'}" -NE,Nebraska,7,31,0,1,0,"{'Active Case Data': 'ADS.E2J.NDM1.TS31', 'Closed Case Data': 'ADS.E2J.NDM2.TS31', 'Aggregate Data': 'ADS.E2J.NDM3.TS31', 'SSP Active Case Data': 'ADS.E2J.NDM1.MS31', 'SSP Closed Case Data': 'ADS.E2J.NDM2.MS31', 'SSP Aggregate Data': 'ADS.E2J.NDM3.MS31'}" -NV,Nevada,9,32,1,1,0,"{'Active Case Data': 'ADS.E2J.NDM1.TS32', 'Closed Case Data': 'ADS.E2J.NDM2.TS32', 'Aggregate Data': 'ADS.E2J.NDM3.TS32', 'Stratum Data': 'ADS.E2J.NDM4.TS32','SSP Active Case Data': 'ADS.E2J.NDM1.MS32', 'SSP Closed Case Data': 'ADS.E2J.NDM2.MS32', 'SSP Aggregate Data': 'ADS.E2J.NDM3.MS32', 'SSP Stratum Data': 'ADS.E2J.NDM4.MS32'}" -NH,New Hampshire,1,33,0,1,1,"{'Active Case Data': 'ADS.E2J.FTP1.TS33', 'Closed Case Data': 'ADS.E2J.FTP2.TS33', 'Aggregate Data': 'ADS.E2J.FTP3.TS33', 'SSP Active Case Data': 'ADS.E2J.FTP1.MS33', 'SSP Closed Case Data': 'ADS.E2J.FTP2.MS33', 'SSP Aggregate Data': 'ADS.E2J.FTP3.MS33'}" -NJ,New Jersey,2,34,0,0,0,"{'Active Case Data': 'ADS.E2J.NDM1.TS34', 'Closed Case Data': 'ADS.E2J.NDM2.TS34', 'Aggregate Data': 'ADS.E2J.NDM3.TS34'}" -NM,New Mexico,6,35,1,0,1,"{'Active Case Data': 'ADS.E2J.FTP1.TS35', 'Closed Case Data': 'ADS.E2J.FTP2.TS35', 'Aggregate Data': 'ADS.E2J.FTP3.TS35', 'Stratum Data': 'ADS.E2J.FTP4.TS35'}" -NY,New York,2,36,1,1,0,"{'Active Case Data': 'ADS.E2J.FTP1.TS36', 'Closed Case Data': 'ADS.E2J.FTP2.TS36', 'Aggregate Data': 'ADS.E2J.FTP3.TS36', 'Stratum Data': 'ADS.E2J.FTP4.TS36', 'SSP Active Case Data': 'ADS.E2J.FTP1.MS36', 'SSP Closed Case Data': 'ADS.E2J.FTP2.MS36', 'SSP Aggregate Data': 'ADS.E2J.FTP3.MS36', 'SSP Stratum Data': 'ADS.E2J.FTP4.MS36'}" -NC,North Carolina,4,37,0,0,0,"{'Active Case Data': 'ADS.E2J.FTP1.TS37', 'Closed Case Data': 'ADS.E2J.FTP2.TS37', 'Aggregate Data': 'ADS.E2J.FTP3.TS37'}" -ND,North Dakota,8,38,0,0,0,"{'Active Case Data': 'ADS.E2J.NDM1.TS38', 'Closed Case Data': 'ADS.E2J.NDM2.TS38', 'Aggregate Data': 'ADS.E2J.NDM3.TS38'}" -OH,Ohio,5,39,1,0,1,"{'Active Case Data': 'ADS.E2J.NDM1.TS39', 'Closed Case Data': 'ADS.E2J.NDM2.TS39', 'Aggregate Data': 'ADS.E2J.NDM3.TS39', 'Stratum Data': 'ADS.E2J.NDM4.TS39'}" -OK,Oklahoma,6,40,0,0,0,"{'Active Case Data': 'ADS.E2J.NDM1.TS40', 'Closed Case Data': 'ADS.E2J.NDM2.TS40', 'Aggregate Data': 'ADS.E2J.NDM3.TS40'}" -OR,Oregon,10,41,0,1,0,"{'Active Case Data': 'ADS.E2J.NDM1.TS41', 'Closed Case Data': 'ADS.E2J.NDM2.TS41', 'Aggregate Data': 'ADS.E2J.NDM3.TS41', 'SSP Active Case Data': 'ADS.E2J.NDM1.MS41', 'SSP Closed Case Data': 'ADS.E2J.NDM2.MS41', 'SSP Aggregate Data': 'ADS.E2J.NDM3.MS41'}" -PA,Pennsylvania,3,42,1,0,0,"{'Active Case Data': 'ADS.E2J.FTP1.TS42', 'Closed Case Data': 'ADS.E2J.FTP2.TS42', 'Aggregate Data': 'ADS.E2J.FTP3.TS42', 'Stratum Data': 'ADS.E2J.FTP4.TS42'}" -RI,Rhode Island,1,44,0,0,0,"{'Active Case Data': 'ADS.E2J.NDM1.TS44', 'Closed Case Data': 'ADS.E2J.NDM2.TS44', 'Aggregate Data': 'ADS.E2J.NDM3.TS44'}" -SC,South Carolina,4,45,1,0,1,"{'Active Case Data': 'ADS.E2J.FTP1.TS45', 'Closed Case Data': 'ADS.E2J.FTP2.TS45', 'Aggregate Data': 'ADS.E2J.FTP3.TS45', 'Stratum Data': 'ADS.E2J.FTP4.TS45'}" -SD,South Dakota,8,46,1,0,0,"{'Active Case Data': 'ADS.E2J.NDM1.TS46', 'Closed Case Data': 'ADS.E2J.NDM2.TS46', 'Aggregate Data': 'ADS.E2J.NDM3.TS46', 'Stratum Data': 'ADS.E2J.NDM4.TS46'}" -TN,Tennessee,4,47,1,0,0,"{'Active Case Data': 'ADS.E2J.NDM1.TS47', 'Closed Case Data': 'ADS.E2J.NDM2.TS47', 'Aggregate Data': 'ADS.E2J.NDM3.TS47', 'Stratum Data': 'ADS.E2J.NDM4.TS47'}" -TX,Texas,6,48,1,0,1,"{'Active Case Data': 'ADS.E2J.FTP1.TS48', 'Closed Case Data': 'ADS.E2J.FTP2.TS48', 'Aggregate Data': 'ADS.E2J.FTP3.TS48', 'Stratum Data': 'ADS.E2J.FTP4.TS48'}" -UT,Utah,8,49,0,1,0,"{'Active Case Data': 'ADS.E2J.NDM1.TS49', 'Closed Case Data': 'ADS.E2J.NDM2.TS49', 'Aggregate Data': 'ADS.E2J.NDM3.TS49','SSP Active Case Data': 'ADS.E2J.NDM1.MS49', 'SSP Closed Case Data': 'ADS.E2J.NDM2.MS49', 'SSP Aggregate Data': 'ADS.E2J.NDM3.MS49'}" -VT,Vermont,1,50,0,1,0,"{'Active Case Data': 'ADS.E2J.FTP1.TS50', 'Closed Case Data': 'ADS.E2J.FTP2.TS50', 'Aggregate Data': 'ADS.E2J.FTP3.TS50', 'SSP Active Case Data': 'ADS.E2J.FTP1.MS50', 'SSP Closed Case Data': 'ADS.E2J.FTP2.MS50', 'SSP Aggregate Data': 'ADS.E2J.FTP3.MS50'}" -VA,Virginia,3,51,0,1,0,"{'Active Case Data': 'ADS.E2J.FTP1.TS51', 'Closed Case Data': 'ADS.E2J.FTP2.TS51', 'Aggregate Data': 'ADS.E2J.FTP3.TS51', 'SSP Active Case Data': 'ADS.E2J.FTP1.MS51', 'SSP Closed Case Data': 'ADS.E2J.FTP2.MS51', 'SSP Aggregate Data': 'ADS.E2J.FTP3.MS51'}" -WA,Washington,10,53,0,1,0,"{'Active Case Data': 'ADS.E2J.NDM1.TS53', 'Closed Case Data': 'ADS.E2J.NDM2.TS53', 'Aggregate Data': 'ADS.E2J.NDM3.TS53', 'SSP Active Case Data': 'ADS.E2J.NDM1.MS53', 'SSP Closed Case Data': 'ADS.E2J.NDM2.MS53', 'SSP Aggregate Data': 'ADS.E2J.NDM3.MS53'}" -WV,West Virginia,3,54,1,0,0,"{'Active Case Data': 'ADS.E2J.NDM1.TS54', 'Closed Case Data': 'ADS.E2J.NDM2.TS54', 'Aggregate Data': 'ADS.E2J.NDM3.TS54', 'Stratum Data': 'ADS.E2J.NDM4.TS54'}" -WI,Wisconsin,5,55,0,1,0,"{'Active Case Data': 'ADS.E2J.NDM1.TS55', 'Closed Case Data': 'ADS.E2J.NDM2.TS55', 'Aggregate Data': 'ADS.E2J.NDM3.TS55', 'SSP Active Case Data': 'ADS.E2J.NDM1.MS55', 'SSP Closed Case Data': 'ADS.E2J.NDM2.MS55', 'SSP Aggregate Data': 'ADS.E2J.NDM3.MS55'}" -WY,Wyoming,8,56,0,0,1,"{'Active Case Data': 'ADS.E2J.NDM1.TS56', 'Closed Case Data': 'ADS.E2J.NDM2.TS56', 'Aggregate Data': 'ADS.E2J.NDM3.TS56'}" +Code,Name,Region,STT_CODE,Sample,SSP,SSN_Encrypted,filenames,Timezone +AL,Alabama,4,01,0,0,0,"{'Active Case Data': 'ADS.E2J.NDM1.TS01', 'Closed Case Data': 'ADS.E2J.NDM2.TS01', 'Aggregate Data': 'ADS.E2J.NDM3.TS01'}",America/Chicago +AK,Alaska,10,02,0,0,0,"{'Active Case Data': 'ADS.E2J.NDM1.TS02', 'Closed Case Data': 'ADS.E2J.NDM2.TS02', 'Aggregate Data': 'ADS.E2J.NDM3.TS02'}",America/Anchorage +AZ,Arizona,9,04,0,0,0,"{'Active Case Data': 'ADS.E2J.FTP1.TS04', 'Closed Case Data': 'ADS.E2J.FTP2.TS04', 'Aggregate Data': 'ADS.E2J.FTP3.TS04'}",America/Phoenix +AR,Arkansas,6,05,0,0,1,"{'Active Case Data': 'ADS.E2J.FTP1.TS05', 'Closed Case Data': 'ADS.E2J.FTP2.TS05', 'Aggregate Data': 'ADS.E2J.FTP3.TS05'}",America/Chicago +CA,California,9,06,1,1,0,"{'Active Case Data': 'ADS.E2J.FTP1.TS06', 'Closed Case Data': 'ADS.E2J.FTP2.TS06', 'Aggregate Data': 'ADS.E2J.FTP3.TS06', 'Stratum Data': 'ADS.E2J.FTP4.TS06', 'SSP Active Case Data': 'ADS.E2J.FTP1.MS06', 'SSP Closed Case Data': 'ADS.E2J.FTP2.MS06', 'SSP Aggregate Data': 'ADS.E2J.FTP3.MS06', 'SSP Stratum Data': 'ADS.E2J.FTP4.MS06'}",America/Los_Angeles +CO,Colorado,8,08,1,0,0,"{'Active Case Data': 'ADS.E2J.FTP1.TS08', 'Closed Case Data': 'ADS.E2J.FTP2.TS08', 'Aggregate Data': 'ADS.E2J.FTP3.TS08', 'Stratum Data': 'ADS.E2J.FTP4.TS08'}",America/Denver +CT,Connecticut,1,09,1,0,1,"{'Active Case Data': 'ADS.E2J.NDM1.TS09', 'Closed Case Data': 'ADS.E2J.NDM2.TS09', 'Aggregate Data': 'ADS.E2J.NDM3.TS09', 'Stratum Data': 'ADS.E2J.NDM4.TS09'}",America/New_York +DE,Delaware,3,10,0,0,0,"{'Active Case Data': 'ADS.E2J.NDM1.TS10', 'Closed Case Data': 'ADS.E2J.NDM2.TS10', 'Aggregate Data': 'ADS.E2J.NDM3.TS10'}",America/New_York +DC,District of Columbia,3,11,0,0,0,"{'Active Case Data': 'ADS.E2J.FTP1.TS11', 'Closed Case Data': 'ADS.E2J.FTP2.TS11', 'Aggregate Data': 'ADS.E2J.FTP3.TS11'}",America/New_York +FL,Florida,4,12,1,0,0,"{'Active Case Data': 'ADS.E2J.FTP1.TS12', 'Closed Case Data': 'ADS.E2J.FTP2.TS12', 'Aggregate Data': 'ADS.E2J.FTP3.TS12', 'Stratum Data': 'ADS.E2J.FTP4.TS12'}",America/New_York +GA,Georgia,4,13,0,0,0,"{'Active Case Data': 'ADS.E2J.NDM1.TS13', 'Closed Case Data': 'ADS.E2J.NDM2.TS13', 'Aggregate Data': 'ADS.E2J.NDM3.TS13'}",America/New_York +HI,Hawaii,9,15,0,0,0,"{'Active Case Data': 'ADS.E2J.NDM1.TS15', 'Closed Case Data': 'ADS.E2J.NDM2.TS15', 'Aggregate Data': 'ADS.E2J.NDM3.TS15'}",Pacific/Honolulu +ID,Idaho,10,16,0,0,0,"{'Active Case Data': 'ADS.E2J.NDM1.TS16', 'Closed Case Data': 'ADS.E2J.NDM2.TS16', 'Aggregate Data': 'ADS.E2J.NDM3.TS16'}",America/Boise +IL,Illinois,5,17,1,0,1,"{'Active Case Data': 'ADS.E2J.FTP1.TS17', 'Closed Case Data': 'ADS.E2J.FTP2.TS17', 'Aggregate Data': 'ADS.E2J.FTP3.TS17', 'Stratum Data': 'ADS.E2J.FTP4.TS17'}",America/Chicago +IN,Indiana,5,18,0,1,0,"{'Active Case Data': 'ADS.E2J.NDM1.TS18', 'Closed Case Data': 'ADS.E2J.NDM2.TS18', 'Aggregate Data': 'ADS.E2J.NDM3.TS18', 'SSP Active Case Data': 'ADS.E2J.NDM1.MS18', 'SSP Closed Case Data': 'ADS.E2J.NDM2.MS18', 'SSP Aggregate Data': 'ADS.E2J.NDM3.MS18'}",America/Indiana/Indianapolis +IA,Iowa,7,19,0,1,0,"{'Active Case Data': 'ADS.E2J.NDM1.TS19', 'Closed Case Data': 'ADS.E2J.NDM2.TS19', 'Aggregate Data': 'ADS.E2J.NDM3.TS19', 'SSP Active Case Data': 'ADS.E2J.NDM1.MS19', 'SSP Closed Case Data': 'ADS.E2J.NDM2.MS19', 'SSP Aggregate Data': 'ADS.E2J.NDM3.MS19'}",America/Chicago +KS,Kansas,7,20,1,0,0,"{'Active Case Data': 'ADS.E2J.NDM1.TS20', 'Closed Case Data': 'ADS.E2J.NDM2.TS20', 'Aggregate Data': 'ADS.E2J.NDM3.TS20', 'Stratum Data': 'ADS.E2J.NDM4.TS20'}",America/Chicago +KY,Kentucky,4,21,0,0,1,"{'Active Case Data': 'ADS.E2J.FTP1.TS21', 'Closed Case Data': 'ADS.E2J.FTP2.TS21', 'Aggregate Data': 'ADS.E2J.FTP3.TS21'}",America/New_York +LA,Louisiana,6,22,0,0,0,"{'Active Case Data': 'ADS.E2J.NDM1.TS22', 'Closed Case Data': 'ADS.E2J.NDM2.TS22', 'Aggregate Data': 'ADS.E2J.NDM3.TS22'}",America/Chicago +ME,Maine,1,23,0,1,0,"{'Active Case Data': 'ADS.E2J.NDM1.TS23', 'Closed Case Data': 'ADS.E2J.NDM2.TS23', 'Aggregate Data': 'ADS.E2J.NDM3.TS23', 'SSP Active Case Data': 'ADS.E2J.NDM1.MS23', 'SSP Closed Case Data': 'ADS.E2J.NDM2.MS23', 'SSP Aggregate Data': 'ADS.E2J.NDM3.MS23'}",America/New_York +MD,Maryland,3,24,1,1,0,"{'Active Case Data': 'ADS.E2J.NDM1.TS24', 'Closed Case Data': 'ADS.E2J.NDM2.TS24', 'Aggregate Data': 'ADS.E2J.NDM3.TS24', 'Stratum Data': 'ADS.E2J.NDM4.TS24', 'SSP Active Case Data': 'ADS.E2J.NDM1.MS24', 'SSP Closed Case Data': 'ADS.E2J.NDM2.MS24', 'SSP Aggregate Data': 'ADS.E2J.NDM3.MS24', 'SSP Stratum Data': 'ADS.E2J.NDM4.MS24'}",America/New_York +MA,Massachusetts,1,25,1,1,1,"{'Active Case Data': 'ADS.E2J.FTP1.TS25', 'Closed Case Data': 'ADS.E2J.FTP2.TS25', 'Aggregate Data': 'ADS.E2J.FTP3.TS25', 'Stratum Data': 'ADS.E2J.FTP4.TS25', 'SSP Active Case Data': 'ADS.E2J.FTP1.MS25', 'SSP Closed Case Data': 'ADS.E2J.FTP2.MS25', 'SSP Aggregate Data': 'ADS.E2J.FTP3.MS25', 'SSP Stratum Data': 'ADS.E2J.FTP4.MS25'}",America/New_York +MI,Michigan,5,26,1,0,0,"{'Active Case Data': 'ADS.E2J.NDM1.TS26', 'Closed Case Data': 'ADS.E2J.NDM2.TS26', 'Aggregate Data': 'ADS.E2J.NDM3.TS26', 'Stratum Data': 'ADS.E2J.NDM4.TS26'}",America/Detroit +MN,Minnesota,5,27,0,0,0,"{'Active Case Data': 'ADS.E2J.NDM1.TS27', 'Closed Case Data': 'ADS.E2J.NDM2.TS27', 'Aggregate Data': 'ADS.E2J.NDM3.TS27'}",America/Chicago +MS,Mississippi,4,28,1,0,1,"{'Active Case Data': 'ADS.E2J.NDM1.TS28', 'Closed Case Data': 'ADS.E2J.NDM2.TS28', 'Aggregate Data': 'ADS.E2J.NDM3.TS28', 'Stratum Data': 'ADS.E2J.NDM4.TS28'}",America/Chicago +MO,Missouri,7,29,0,1,0,"{'Active Case Data': 'ADS.E2J.NDM1.TS29', 'Closed Case Data': 'ADS.E2J.NDM2.TS29', 'Aggregate Data': 'ADS.E2J.NDM3.TS29', 'SSP Active Case Data': 'ADS.E2J.NDM1.MS29', 'SSP Closed Case Data': 'ADS.E2J.NDM2.MS29', 'SSP Aggregate Data': 'ADS.E2J.NDM3.MS29'}",America/Chicago +MT,Montana,8,30,0,0,0,"{'Active Case Data': 'ADS.E2J.NDM1.TS30', 'Closed Case Data': 'ADS.E2J.NDM2.TS30', 'Aggregate Data': 'ADS.E2J.NDM3.TS30'}",America/Denver +NE,Nebraska,7,31,0,1,0,"{'Active Case Data': 'ADS.E2J.NDM1.TS31', 'Closed Case Data': 'ADS.E2J.NDM2.TS31', 'Aggregate Data': 'ADS.E2J.NDM3.TS31', 'SSP Active Case Data': 'ADS.E2J.NDM1.MS31', 'SSP Closed Case Data': 'ADS.E2J.NDM2.MS31', 'SSP Aggregate Data': 'ADS.E2J.NDM3.MS31'}",America/Chicago +NV,Nevada,9,32,1,1,0,"{'Active Case Data': 'ADS.E2J.NDM1.TS32', 'Closed Case Data': 'ADS.E2J.NDM2.TS32', 'Aggregate Data': 'ADS.E2J.NDM3.TS32', 'Stratum Data': 'ADS.E2J.NDM4.TS32','SSP Active Case Data': 'ADS.E2J.NDM1.MS32', 'SSP Closed Case Data': 'ADS.E2J.NDM2.MS32', 'SSP Aggregate Data': 'ADS.E2J.NDM3.MS32', 'SSP Stratum Data': 'ADS.E2J.NDM4.MS32'}",America/Los_Angeles +NH,New Hampshire,1,33,0,1,1,"{'Active Case Data': 'ADS.E2J.FTP1.TS33', 'Closed Case Data': 'ADS.E2J.FTP2.TS33', 'Aggregate Data': 'ADS.E2J.FTP3.TS33', 'SSP Active Case Data': 'ADS.E2J.FTP1.MS33', 'SSP Closed Case Data': 'ADS.E2J.FTP2.MS33', 'SSP Aggregate Data': 'ADS.E2J.FTP3.MS33'}",America/New_York +NJ,New Jersey,2,34,0,0,0,"{'Active Case Data': 'ADS.E2J.NDM1.TS34', 'Closed Case Data': 'ADS.E2J.NDM2.TS34', 'Aggregate Data': 'ADS.E2J.NDM3.TS34'}",America/New_York +NM,New Mexico,6,35,1,0,1,"{'Active Case Data': 'ADS.E2J.FTP1.TS35', 'Closed Case Data': 'ADS.E2J.FTP2.TS35', 'Aggregate Data': 'ADS.E2J.FTP3.TS35', 'Stratum Data': 'ADS.E2J.FTP4.TS35'}",America/Denver +NY,New York,2,36,1,1,0,"{'Active Case Data': 'ADS.E2J.FTP1.TS36', 'Closed Case Data': 'ADS.E2J.FTP2.TS36', 'Aggregate Data': 'ADS.E2J.FTP3.TS36', 'Stratum Data': 'ADS.E2J.FTP4.TS36', 'SSP Active Case Data': 'ADS.E2J.FTP1.MS36', 'SSP Closed Case Data': 'ADS.E2J.FTP2.MS36', 'SSP Aggregate Data': 'ADS.E2J.FTP3.MS36', 'SSP Stratum Data': 'ADS.E2J.FTP4.MS36'}",America/New_York +NC,North Carolina,4,37,0,0,0,"{'Active Case Data': 'ADS.E2J.FTP1.TS37', 'Closed Case Data': 'ADS.E2J.FTP2.TS37', 'Aggregate Data': 'ADS.E2J.FTP3.TS37'}",America/New_York +ND,North Dakota,8,38,0,0,0,"{'Active Case Data': 'ADS.E2J.NDM1.TS38', 'Closed Case Data': 'ADS.E2J.NDM2.TS38', 'Aggregate Data': 'ADS.E2J.NDM3.TS38'}",America/Chicago +OH,Ohio,5,39,1,0,1,"{'Active Case Data': 'ADS.E2J.NDM1.TS39', 'Closed Case Data': 'ADS.E2J.NDM2.TS39', 'Aggregate Data': 'ADS.E2J.NDM3.TS39', 'Stratum Data': 'ADS.E2J.NDM4.TS39'}",America/New_York +OK,Oklahoma,6,40,0,0,0,"{'Active Case Data': 'ADS.E2J.NDM1.TS40', 'Closed Case Data': 'ADS.E2J.NDM2.TS40', 'Aggregate Data': 'ADS.E2J.NDM3.TS40'}",America/Chicago +OR,Oregon,10,41,0,1,0,"{'Active Case Data': 'ADS.E2J.NDM1.TS41', 'Closed Case Data': 'ADS.E2J.NDM2.TS41', 'Aggregate Data': 'ADS.E2J.NDM3.TS41', 'SSP Active Case Data': 'ADS.E2J.NDM1.MS41', 'SSP Closed Case Data': 'ADS.E2J.NDM2.MS41', 'SSP Aggregate Data': 'ADS.E2J.NDM3.MS41'}",America/Los_Angeles +PA,Pennsylvania,3,42,1,0,0,"{'Active Case Data': 'ADS.E2J.FTP1.TS42', 'Closed Case Data': 'ADS.E2J.FTP2.TS42', 'Aggregate Data': 'ADS.E2J.FTP3.TS42', 'Stratum Data': 'ADS.E2J.FTP4.TS42'}",America/New_York +RI,Rhode Island,1,44,0,0,0,"{'Active Case Data': 'ADS.E2J.NDM1.TS44', 'Closed Case Data': 'ADS.E2J.NDM2.TS44', 'Aggregate Data': 'ADS.E2J.NDM3.TS44'}",America/New_York +SC,South Carolina,4,45,1,0,1,"{'Active Case Data': 'ADS.E2J.FTP1.TS45', 'Closed Case Data': 'ADS.E2J.FTP2.TS45', 'Aggregate Data': 'ADS.E2J.FTP3.TS45', 'Stratum Data': 'ADS.E2J.FTP4.TS45'}",America/New_York +SD,South Dakota,8,46,1,0,0,"{'Active Case Data': 'ADS.E2J.NDM1.TS46', 'Closed Case Data': 'ADS.E2J.NDM2.TS46', 'Aggregate Data': 'ADS.E2J.NDM3.TS46', 'Stratum Data': 'ADS.E2J.NDM4.TS46'}",America/Chicago +TN,Tennessee,4,47,1,0,0,"{'Active Case Data': 'ADS.E2J.NDM1.TS47', 'Closed Case Data': 'ADS.E2J.NDM2.TS47', 'Aggregate Data': 'ADS.E2J.NDM3.TS47', 'Stratum Data': 'ADS.E2J.NDM4.TS47'}",America/Chicago +TX,Texas,6,48,1,0,1,"{'Active Case Data': 'ADS.E2J.FTP1.TS48', 'Closed Case Data': 'ADS.E2J.FTP2.TS48', 'Aggregate Data': 'ADS.E2J.FTP3.TS48', 'Stratum Data': 'ADS.E2J.FTP4.TS48'}",America/Chicago +UT,Utah,8,49,0,1,0,"{'Active Case Data': 'ADS.E2J.NDM1.TS49', 'Closed Case Data': 'ADS.E2J.NDM2.TS49', 'Aggregate Data': 'ADS.E2J.NDM3.TS49','SSP Active Case Data': 'ADS.E2J.NDM1.MS49', 'SSP Closed Case Data': 'ADS.E2J.NDM2.MS49', 'SSP Aggregate Data': 'ADS.E2J.NDM3.MS49'}",America/Denver +VT,Vermont,1,50,0,1,0,"{'Active Case Data': 'ADS.E2J.FTP1.TS50', 'Closed Case Data': 'ADS.E2J.FTP2.TS50', 'Aggregate Data': 'ADS.E2J.FTP3.TS50', 'SSP Active Case Data': 'ADS.E2J.FTP1.MS50', 'SSP Closed Case Data': 'ADS.E2J.FTP2.MS50', 'SSP Aggregate Data': 'ADS.E2J.FTP3.MS50'}",America/New_York +VA,Virginia,3,51,0,1,0,"{'Active Case Data': 'ADS.E2J.FTP1.TS51', 'Closed Case Data': 'ADS.E2J.FTP2.TS51', 'Aggregate Data': 'ADS.E2J.FTP3.TS51', 'SSP Active Case Data': 'ADS.E2J.FTP1.MS51', 'SSP Closed Case Data': 'ADS.E2J.FTP2.MS51', 'SSP Aggregate Data': 'ADS.E2J.FTP3.MS51'}",America/New_York +WA,Washington,10,53,0,1,0,"{'Active Case Data': 'ADS.E2J.NDM1.TS53', 'Closed Case Data': 'ADS.E2J.NDM2.TS53', 'Aggregate Data': 'ADS.E2J.NDM3.TS53', 'SSP Active Case Data': 'ADS.E2J.NDM1.MS53', 'SSP Closed Case Data': 'ADS.E2J.NDM2.MS53', 'SSP Aggregate Data': 'ADS.E2J.NDM3.MS53'}",America/Los_Angeles +WV,West Virginia,3,54,1,0,0,"{'Active Case Data': 'ADS.E2J.NDM1.TS54', 'Closed Case Data': 'ADS.E2J.NDM2.TS54', 'Aggregate Data': 'ADS.E2J.NDM3.TS54', 'Stratum Data': 'ADS.E2J.NDM4.TS54'}",America/New_York +WI,Wisconsin,5,55,0,1,0,"{'Active Case Data': 'ADS.E2J.NDM1.TS55', 'Closed Case Data': 'ADS.E2J.NDM2.TS55', 'Aggregate Data': 'ADS.E2J.NDM3.TS55', 'SSP Active Case Data': 'ADS.E2J.NDM1.MS55', 'SSP Closed Case Data': 'ADS.E2J.NDM2.MS55', 'SSP Aggregate Data': 'ADS.E2J.NDM3.MS55'}",America/Chicago +WY,Wyoming,8,56,0,0,1,"{'Active Case Data': 'ADS.E2J.NDM1.TS56', 'Closed Case Data': 'ADS.E2J.NDM2.TS56', 'Aggregate Data': 'ADS.E2J.NDM3.TS56'}",America/Denver diff --git a/tdrs-backend/tdpservice/stts/management/commands/data/territories.csv b/tdrs-backend/tdpservice/stts/management/commands/data/territories.csv index f5068c4c1..0ff2e22ca 100644 --- a/tdrs-backend/tdpservice/stts/management/commands/data/territories.csv +++ b/tdrs-backend/tdpservice/stts/management/commands/data/territories.csv @@ -1,4 +1,4 @@ -Code,Name,Region,STT_CODE,Sample,SSP,SSN_Encrypted,filenames -GU,Guam,9,66,0,0,0,"{'Active Case Data': 'ADS.E2J.FTP1.TS66', 'Closed Case Data': 'ADS.E2J.FTP2.TS66', 'Aggregate Data': 'ADS.E2J.FTP3.TS66'}" -PR,Puerto Rico,2,72,1,0,1,"{'Active Case Data': 'ADS.E2J.FTP1.TS72', 'Closed Case Data': 'ADS.E2J.FTP2.TS72', 'Aggregate Data': 'ADS.E2J.FTP3.TS72', 'Stratum Data': 'ADS.E2J.FTP4.TS72'}" -VI,Virgin Islands,2,78,0,0,2,"{'Active Case Data': 'ADS.E2J.NDM1.TS78', 'Closed Case Data': 'ADS.E2J.NDM2.TS78', 'Aggregate Data': 'ADS.E2J.NDM3.TS78'}" +Code,Name,Region,STT_CODE,Sample,SSP,SSN_Encrypted,filenames,Timezone +GU,Guam,9,66,0,0,0,"{'Active Case Data': 'ADS.E2J.FTP1.TS66', 'Closed Case Data': 'ADS.E2J.FTP2.TS66', 'Aggregate Data': 'ADS.E2J.FTP3.TS66'}",Pacific/Guam +PR,Puerto Rico,2,72,1,0,1,"{'Active Case Data': 'ADS.E2J.FTP1.TS72', 'Closed Case Data': 'ADS.E2J.FTP2.TS72', 'Aggregate Data': 'ADS.E2J.FTP3.TS72', 'Stratum Data': 'ADS.E2J.FTP4.TS72'}",America/Puerto_Rico +VI,Virgin Islands,2,78,0,0,2,"{'Active Case Data': 'ADS.E2J.NDM1.TS78', 'Closed Case Data': 'ADS.E2J.NDM2.TS78', 'Aggregate Data': 'ADS.E2J.NDM3.TS78'}",America/Virgin diff --git a/tdrs-backend/tdpservice/stts/management/commands/data/tribes.csv b/tdrs-backend/tdpservice/stts/management/commands/data/tribes.csv index 34dd1bca2..a60982fb5 100644 --- a/tdrs-backend/tdpservice/stts/management/commands/data/tribes.csv +++ b/tdrs-backend/tdpservice/stts/management/commands/data/tribes.csv @@ -1,78 +1,78 @@ -Code,Name,Region,STT_CODE,Sample,SSP,SSN_Encrypted,filenames -AK,Association of Village Council Presidents,10,805,0,0,0,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS805', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS805', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS805'}" -WI,Bad River Band of Lake Superior Tribe of Chippewa Indians ,5,012,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS012', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS012', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS012'}" -MT,Blackfeet Nation ,8,020,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS020', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS020', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS020'}" -AK,Bristol Bay Native Association,10,808,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS808', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS808', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS808'}" -AK,Central Council of Tlingit and Haida Indian Tribes of Alaska ,10,811,0,0,0,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS811', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS811', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS811'}" -MT,Chippewa-Cree Indians of the Rocky Boy's Reservation,8,043,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS043', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS043', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS043'}" -ID,Coeur d'Alene Tribe ,10,050,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS050', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS050', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS050'}" -MT,Confederated Salish & Kootenai Tribes of the Flathead Reservation,8,054,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS054', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS054', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS054'}" -WA,Confederated Tribes of the Colville Reservation,10,056,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS056', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS056', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS056'}" -OR,Confederated Tribes of Siletz Indians ,10,060,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS060', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS060', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS060'}" -AK,"Cook Inlet Tribal Council, Inc.",10,807,0,0,0,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS807', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS807', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS807'}" -NC,Eastern Band of Cherokee Indians,4,078,0,0,0,"{'Tribal Active Case Data': 'ADS.E2J.NDM1.TS078', 'Tribal Closed Case Data': 'ADS.E2J.NDM2.TS078', 'Tribal Aggregate Data': 'ADS.E2J.NDM3.TS078'}" -WY,Eastern Shoshone Tribe of the Wind River Reservation,8,272,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS272', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS272', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS272'}" -WI,Forest County Potawatomi Community,5,085,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS085', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS085', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS085'}" -MT,Fort Belknap Indian Community Council,8,086,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS086', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS086', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS086'}" -CA,Federated Indians of Graton Rancheria ,9,517,0,0,0,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS517', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS517', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS517'}" -CA,Hoopa Valley Tribe ,9,102,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS102', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS102', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS102'}" -AZ,Hopi Tribe,9,103,0,0,0,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS103', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS103', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS103'}" -CA,Karuk Tribe,9,120,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS120', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS120', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS120'}" -OR,Klamath Tribes,10,129,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS129', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS129', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS129'}" -AK,Kodiak Area Native Assoc.,10,812,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS812', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS812', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS812'}" -WI,Lac Courte Oreilles Band of Lake Superior Chippewa Indians,5,133,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS133', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS133', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS133'}" -WI,Lac du Flambeau Band of Lake Superior Chippewa Indians,5,134,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS134', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS134', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS134'}" -WA,Lower Elwha Klallam Tribe,10,142,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS142', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS142', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS142'}" -WA,Lummi Nation,10,144,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS144', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS144', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS144'}" -AK,Maniilaq Association,10,804,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS804', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS804', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS804'}" -WI,Menominee Indian Tribe of Wisconsin,5,151,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS151', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS151', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS151'}" -MN,Mille Lacs Band of Ojibwe,5,405,0,0,0,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS405', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS405', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS405'}" -CA,Morongo Band of Mission Indians,9,163,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS163', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS163', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS163'}" -OK,Muscogee Creek Nation,6,165,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS165', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS165', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS165'}" -AZ,Navajo Nation,9,167,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS167', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS167', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS167'}" -ID,Nez Perce Tribe,10,168,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS168', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS168', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS168'}" -WA,Nooksack Indian Tribe,10,170,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS170', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS170', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS170'}" -CA,Northfork Rancheria of Mono Indians of California,9,172,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS172', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS172', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS172'}" -WY,Northern Arapaho Tribe of the Wind River Indian Reservation,8,008,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS008', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS008', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS008'}" -NE,Omaha Tribe of Nebraska,7,175,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS175', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS175', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS175'}" -WI,Oneida Nation of Wisconsin,5,177,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS177', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS177', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS177'}" -OK,Osage Nation of Oklahoma,6,179,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS179', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS179', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS179'}" -CA,Owens Valley Career Development Center,9,514,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS514', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS514', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS514'}" -AZ,Pascua Yaqui Tribe of Arizona,9,188,0,0,0,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS188', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS188', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS188'}" -CA,Pechanga Band of Luiseno Mission Indians,9,193,0,0,0,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS193', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS193', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS193'}" -WA,Port Gamble S'Klallam Tribe,10,203,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS203', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS203', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS203'}" -KS,Prairie Band Potawatomi Nation,7,205,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS205', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS205', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS205'}" -NM,Pueblo of Zuni,6,334,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS334', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS334', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS334'}" -WA,Quileute Tribe,10,230,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS230', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS230', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS230'}" -WA,Quinault Indian Nation,10,231,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS231', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS231', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS231'}" -WI,Red Cliff Band of Lake Superior Chippewa Indians,5,233,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS233', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS233', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS233'}" -MN,Red Lake Band of Chippewa Indians,5,234,0,0,0,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS234', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS234', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS234'}" -CA,Robinson Rancheria,9,520,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS520', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS520', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS520'}" -CA,Round Valley Indian Tribes,9,241,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS241', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS241', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS241'}" -AZ,Salt River - Pima Maricopa Indian Community,9,248,0,0,0,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS248', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS248', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS248'}" -AZ,San Carlos Apache Tribe,9,250,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS250', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS250', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS250'}" -NM,Santo Domingo Pueblo,6,221,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS221', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS221', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS221'}" -CA,Scotts Valley Band of Pomo Indians,9,262,0,0,0,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS262', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS262', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS262'}" -CA,Shingle Springs Band of Miwok Indians,9,270,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS270', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS270', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS270'}" -ID,Shoshone-Bannock Tribes,10,273,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS273', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS273', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS273'}" -SD,Sisseton-Wahpeton Oyate of the Lake Traverse Reservation,8,275,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS275', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS275', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS275'}" -CA,Soboba Band of Luiseno Indians,9,279,0,0,0,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS279', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS279', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS279'}" -WI,Sokaogon Chippewa Community,5,280,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS280', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS280', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS280'}" -WA,South Puget Inter-Tribal Planning Agency,10,504,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS504', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS504', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS504'}" -CA,"Southern California Tribal Chairmen's Association, Inc.",9,501,0,0,0,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS501', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS501', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS501'}" -WA,Spokane Tribe of Indians,10,282,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS282', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS282', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS282'}" -WI,Stockbridge-Munsee Community Band of Mohican Indians,5,287,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS287', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS287', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS287'}" -AK,Tanana Chiefs Conference,10,806,0,0,0,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS806', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS806', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS806'}" -CA,Torres Martinez Desert Cahuilla Indians,9,513,0,0,0,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS513', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS513', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS513'}" -WA,Tulalip Tribes,10,305,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS305', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS305', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS305'}" -CA,Tuolumne Band of Me-Wuk Indians,9,307,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS307', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS307', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS307'}" -WA,Upper Skagit Indian Tribe,10,315,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS315', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS315', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS315'}" -NV,Washoe Tribe of Nevada and California,9,321,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS321', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS321', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS321'}" -AZ,White Mountain Apache Tribe,9,322,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS322', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS322', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS322'}" -NE,Winnebago Tribe of Nebraska,7,324,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS324', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS324', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS324'}" -CA,Yurok Tribe,9,333,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS333', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS333', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS333'}" -OK,Cherokee Nation,6,038,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS038', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS038', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS038'}" -NE,Santee Sioux Nation,7,259,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS259', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS259', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS259'}" -CA,Tolowa Dee-ni' Nation,9,278,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS278', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS278', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS278'}" -MS,Mississippi Band of Choctaw Indians,4,158,0,0,0,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS158', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS158', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS158'}" -WA,"Suquamish Indian Tribe of the Port Madison Reservation, Washington",10,290,0,0,0,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS290', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS290', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS290'}" \ No newline at end of file +Code,Name,Region,STT_CODE,Sample,SSP,SSN_Encrypted,filenames,Timezone +AK,Association of Village Council Presidents,10,805,0,0,0,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS805', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS805', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS805'}",America/Anchorage +WI,Bad River Band of Lake Superior Tribe of Chippewa Indians ,5,012,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS012', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS012', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS012'}",America/Chicago +MT,Blackfeet Nation ,8,020,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS020', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS020', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS020'}",America/Denver +AK,Bristol Bay Native Association,10,808,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS808', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS808', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS808'}",America/Anchorage +AK,Central Council of Tlingit and Haida Indian Tribes of Alaska ,10,811,0,0,0,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS811', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS811', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS811'}",America/Anchorage +MT,Chippewa-Cree Indians of the Rocky Boy's Reservation,8,043,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS043', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS043', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS043'}",America/Denver +ID,Coeur d'Alene Tribe ,10,050,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS050', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS050', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS050'}",America/Boise +MT,Confederated Salish & Kootenai Tribes of the Flathead Reservation,8,054,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS054', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS054', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS054'}",America/Denver +WA,Confederated Tribes of the Colville Reservation,10,056,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS056', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS056', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS056'}",America/Los_Angeles +OR,Confederated Tribes of Siletz Indians ,10,060,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS060', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS060', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS060'}",America/Los_Angeles +AK,"Cook Inlet Tribal Council, Inc.",10,807,0,0,0,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS807', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS807', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS807'}",America/Anchorage +NC,Eastern Band of Cherokee Indians,4,078,0,0,0,"{'Tribal Active Case Data': 'ADS.E2J.NDM1.TS078', 'Tribal Closed Case Data': 'ADS.E2J.NDM2.TS078', 'Tribal Aggregate Data': 'ADS.E2J.NDM3.TS078'}",America/New_York +WY,Eastern Shoshone Tribe of the Wind River Reservation,8,272,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS272', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS272', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS272'}",America/Denver +WI,Forest County Potawatomi Community,5,085,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS085', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS085', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS085'}",America/Chicago +MT,Fort Belknap Indian Community Council,8,086,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS086', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS086', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS086'}",America/Denver +CA,Federated Indians of Graton Rancheria ,9,517,0,0,0,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS517', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS517', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS517'}",America/Los_Angeles +CA,Hoopa Valley Tribe ,9,102,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS102', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS102', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS102'}",America/Los_Angeles +AZ,Hopi Tribe,9,103,0,0,0,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS103', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS103', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS103'}",America/Phoenix +CA,Karuk Tribe,9,120,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS120', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS120', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS120'}",America/Los_Angeles +OR,Klamath Tribes,10,129,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS129', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS129', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS129'}",America/Los_Angeles +AK,Kodiak Area Native Assoc.,10,812,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS812', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS812', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS812'}",America/Anchorage +WI,Lac Courte Oreilles Band of Lake Superior Chippewa Indians,5,133,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS133', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS133', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS133'}",America/Chicago +WI,Lac du Flambeau Band of Lake Superior Chippewa Indians,5,134,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS134', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS134', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS134'}",America/Chicago +WA,Lower Elwha Klallam Tribe,10,142,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS142', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS142', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS142'}",America/Los_Angeles +WA,Lummi Nation,10,144,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS144', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS144', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS144'}",America/Los_Angeles +AK,Maniilaq Association,10,804,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS804', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS804', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS804'}",America/Anchorage +WI,Menominee Indian Tribe of Wisconsin,5,151,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS151', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS151', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS151'}",America/Chicago +MN,Mille Lacs Band of Ojibwe,5,405,0,0,0,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS405', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS405', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS405'}",America/Chicago +CA,Morongo Band of Mission Indians,9,163,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS163', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS163', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS163'}",America/Los_Angeles +OK,Muscogee Creek Nation,6,165,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS165', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS165', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS165'}",America/Chicago +AZ,Navajo Nation,9,167,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS167', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS167', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS167'}",America/Denver +ID,Nez Perce Tribe,10,168,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS168', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS168', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS168'}",America/Boise +WA,Nooksack Indian Tribe,10,170,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS170', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS170', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS170'}",America/Los_Angeles +CA,Northfork Rancheria of Mono Indians of California,9,172,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS172', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS172', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS172'}",America/Los_Angeles +WY,Northern Arapaho Tribe of the Wind River Indian Reservation,8,008,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS008', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS008', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS008'}",America/Denver +NE,Omaha Tribe of Nebraska,7,175,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS175', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS175', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS175'}",America/Chicago +WI,Oneida Nation of Wisconsin,5,177,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS177', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS177', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS177'}",America/Chicago +OK,Osage Nation of Oklahoma,6,179,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS179', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS179', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS179'}",America/Chicago +CA,Owens Valley Career Development Center,9,514,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS514', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS514', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS514'}",America/Los_Angeles +AZ,Pascua Yaqui Tribe of Arizona,9,188,0,0,0,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS188', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS188', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS188'}",America/Phoenix +CA,Pechanga Band of Luiseno Mission Indians,9,193,0,0,0,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS193', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS193', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS193'}",America/Los_Angeles +WA,Port Gamble S'Klallam Tribe,10,203,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS203', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS203', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS203'}",America/Los_Angeles +KS,Prairie Band Potawatomi Nation,7,205,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS205', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS205', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS205'}",America/Chicago +NM,Pueblo of Zuni,6,334,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS334', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS334', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS334'}",America/Denver +WA,Quileute Tribe,10,230,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS230', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS230', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS230'}",America/Los_Angeles +WA,Quinault Indian Nation,10,231,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS231', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS231', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS231'}",America/Los_Angeles +WI,Red Cliff Band of Lake Superior Chippewa Indians,5,233,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS233', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS233', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS233'}",America/Chicago +MN,Red Lake Band of Chippewa Indians,5,234,0,0,0,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS234', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS234', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS234'}",America/Chicago +CA,Robinson Rancheria,9,520,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS520', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS520', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS520'}",America/Los_Angeles +CA,Round Valley Indian Tribes,9,241,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS241', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS241', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS241'}",America/Los_Angeles +AZ,Salt River - Pima Maricopa Indian Community,9,248,0,0,0,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS248', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS248', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS248'}",America/Phoenix +AZ,San Carlos Apache Tribe,9,250,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS250', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS250', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS250'}",America/Phoenix +NM,Santo Domingo Pueblo,6,221,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS221', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS221', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS221'}",America/Denver +CA,Scotts Valley Band of Pomo Indians,9,262,0,0,0,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS262', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS262', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS262'}",America/Los_Angeles +CA,Shingle Springs Band of Miwok Indians,9,270,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS270', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS270', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS270'}",America/Los_Angeles +ID,Shoshone-Bannock Tribes,10,273,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS273', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS273', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS273'}",America/Boise +SD,Sisseton-Wahpeton Oyate of the Lake Traverse Reservation,8,275,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS275', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS275', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS275'}",America/Chicago +CA,Soboba Band of Luiseno Indians,9,279,0,0,0,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS279', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS279', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS279'}",America/Los_Angeles +WI,Sokaogon Chippewa Community,5,280,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS280', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS280', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS280'}",America/Chicago +WA,South Puget Inter-Tribal Planning Agency,10,504,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS504', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS504', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS504'}",America/Los_Angeles +CA,"Southern California Tribal Chairmen's Association, Inc.",9,501,0,0,0,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS501', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS501', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS501'}",America/Los_Angeles +WA,Spokane Tribe of Indians,10,282,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS282', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS282', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS282'}",America/Los_Angeles +WI,Stockbridge-Munsee Community Band of Mohican Indians,5,287,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS287', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS287', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS287'}",America/Chicago +AK,Tanana Chiefs Conference,10,806,0,0,0,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS806', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS806', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS806'}",America/Anchorage +CA,Torres Martinez Desert Cahuilla Indians,9,513,0,0,0,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS513', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS513', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS513'}",America/Los_Angeles +WA,Tulalip Tribes,10,305,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS305', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS305', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS305'}",America/Los_Angeles +CA,Tuolumne Band of Me-Wuk Indians,9,307,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS307', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS307', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS307'}",America/Los_Angeles +WA,Upper Skagit Indian Tribe,10,315,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS315', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS315', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS315'}",America/Los_Angeles +NV,Washoe Tribe of Nevada and California,9,321,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS321', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS321', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS321'}",America/Los_Angeles +AZ,White Mountain Apache Tribe,9,322,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS322', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS322', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS322'}",America/Phoenix +NE,Winnebago Tribe of Nebraska,7,324,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS324', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS324', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS324'}",America/Chicago +CA,Yurok Tribe,9,333,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS333', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS333', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS333'}",America/Los_Angeles +OK,Cherokee Nation,6,038,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS038', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS038', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS038'}",America/Chicago +NE,Santee Sioux Nation,7,259,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS259', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS259', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS259'}",America/Chicago +CA,Tolowa Dee-ni' Nation,9,278,0,0,1,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS278', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS278', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS278'}",America/Los_Angeles +MS,Mississippi Band of Choctaw Indians,4,158,0,0,0,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS158', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS158', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS158'}",America/Chicago +WA,"Suquamish Indian Tribe of the Port Madison Reservation, Washington",10,290,0,0,0,"{'Tribal Active Case Data': 'ADS.E2J.FTP1.TS290', 'Tribal Closed Case Data': 'ADS.E2J.FTP2.TS290', 'Tribal Aggregate Data': 'ADS.E2J.FTP3.TS290'}",America/Los_Angeles diff --git a/tdrs-backend/tdpservice/stts/management/commands/populate_stts.py b/tdrs-backend/tdpservice/stts/management/commands/populate_stts.py index d497df7ab..0494c41a9 100644 --- a/tdrs-backend/tdpservice/stts/management/commands/populate_stts.py +++ b/tdrs-backend/tdpservice/stts/management/commands/populate_stts.py @@ -45,6 +45,8 @@ def _load_csv(filename, entity): stt.filenames = json.loads(row["filenames"].replace("'", '"')) stt.ssp = row["SSP"] stt.sample = row["Sample"] + if "Timezone" in row and row["Timezone"]: + stt.timezone = row["Timezone"] # TODO: Was seeing lots of references to STT.objects.filter(pk=... # We could probably one-line this but we'd miss .save() signals # https://stackoverflow.com/questions/41744096/ @@ -103,6 +105,7 @@ def _apply_overrides(overrides_path=None): continue # Only override fields explicitly provided + bool_fields = {"ssp", "sample"} for field in [ "ssp", "sample", @@ -111,9 +114,11 @@ def _apply_overrides(overrides_path=None): "stt_code", "type", "postal_code", + "timezone", ]: if field in override: - setattr(stt, field, _maybe_bool(override[field])) + value = _maybe_bool(override[field]) if field in bool_fields else override[field] + setattr(stt, field, value) stt.save() logger.info("Applied override for STT %s", stt.name) diff --git a/tdrs-backend/tdpservice/stts/migrations/0012_stt_timezone.py b/tdrs-backend/tdpservice/stts/migrations/0012_stt_timezone.py new file mode 100644 index 000000000..d7935be3f --- /dev/null +++ b/tdrs-backend/tdpservice/stts/migrations/0012_stt_timezone.py @@ -0,0 +1,42 @@ +import csv +from pathlib import Path + +from django.db import migrations, models + + +def populate_timezones(apps, schema_editor): + """Populate timezone field for all STTs by reading from the seed CSVs.""" + STT = apps.get_model('stts', 'STT') + data_dir = Path(__file__).resolve().parent.parent / 'management/commands/data' + + tribe_file = 'tribes.csv' + for filename in ('states.csv', 'territories.csv', tribe_file): + chars = 3 if filename == tribe_file else 2 + with open(data_dir / filename) as csvfile: + reader = csv.DictReader(csvfile) + for row in reader: + if 'Timezone' in row and row['Timezone']: + stt_code = str(row['STT_CODE']).zfill(chars) + STT.objects.filter(stt_code=stt_code).update( + timezone=row['Timezone'] + ) + + +def reverse_timezones(apps, schema_editor): + pass + + +class Migration(migrations.Migration): + + dependencies = [ + ('stts', '0011_add_region_name'), + ] + + operations = [ + migrations.AddField( + model_name='stt', + name='timezone', + field=models.CharField(blank=True, default='America/New_York', max_length=63), + ), + migrations.RunPython(populate_timezones, reverse_timezones), + ] diff --git a/tdrs-backend/tdpservice/stts/models.py b/tdrs-backend/tdpservice/stts/models.py index 139e0e992..ae4bbf7f8 100644 --- a/tdrs-backend/tdpservice/stts/models.py +++ b/tdrs-backend/tdpservice/stts/models.py @@ -43,6 +43,7 @@ class EntityType(models.TextChoices): state = models.ForeignKey("self", on_delete=models.CASCADE, blank=True, null=True) ssp = models.BooleanField(default=False, null=True) sample = models.BooleanField(default=False, null=True) + timezone = models.CharField(max_length=63, default="America/New_York", blank=True) @property def num_sections(self): diff --git a/tdrs-backend/tdpservice/stts/test/test_commands.py b/tdrs-backend/tdpservice/stts/test/test_commands.py index 7ea1391e4..55c9d54f8 100644 --- a/tdrs-backend/tdpservice/stts/test/test_commands.py +++ b/tdrs-backend/tdpservice/stts/test/test_commands.py @@ -47,3 +47,36 @@ def test_apply_overrides(tmp_path, stts): rhode_island.refresh_from_db() assert rhode_island.ssp is True + + +@pytest.mark.django_db +def test_populate_stts_sets_timezones(): + """Test that populate_stts populates timezone from CSV data.""" + call_command("populate_stts") + + alaska = STT.objects.get(name="Alaska", type=STT.EntityType.STATE) + assert alaska.timezone == "America/Anchorage" + + navajo = STT.objects.get(name="Navajo Nation", type=STT.EntityType.TRIBE) + assert navajo.timezone == "America/Denver" + + guam = STT.objects.get(name="Guam", type=STT.EntityType.TERRITORY) + assert guam.timezone == "Pacific/Guam" + + +@pytest.mark.django_db +def test_apply_timezone_override(tmp_path, stts): + """Overrides should allow changing an STT's timezone.""" + alaska = STT.objects.get(name="Alaska", type=STT.EntityType.STATE) + assert alaska.timezone == "America/Anchorage" + + overrides_file = tmp_path / "overrides.json" + overrides_file.write_text( + json.dumps([{"name": "Alaska", "timezone": "America/Adak"}]) + ) + + # apply_overrides runs _after_ CSV loading, so the override wins + call_command("populate_stts", apply_overrides=True, overrides=str(overrides_file)) + + alaska.refresh_from_db() + assert alaska.timezone == "America/Adak" diff --git a/tdrs-backend/tdpservice/stts/test/test_models.py b/tdrs-backend/tdpservice/stts/test/test_models.py index b96fc5035..274f76962 100644 --- a/tdrs-backend/tdpservice/stts/test/test_models.py +++ b/tdrs-backend/tdpservice/stts/test/test_models.py @@ -16,3 +16,53 @@ def test_stt_string_representation(stts): """Test STT string representation.""" first_stt = STT.objects.filter(type=STT.EntityType.STATE).first() assert str(first_stt) == f"{first_stt.name} ({first_stt.stt_code})" + + +@pytest.mark.django_db +def test_stt_has_timezone_field(stts): + """Test that STT model has timezone field with a valid IANA timezone.""" + stt = STT.objects.get(name="Alaska") + assert stt.timezone == "America/Anchorage" + + +@pytest.mark.django_db +def test_stt_timezone_default(): + """Test that STT timezone defaults to America/New_York.""" + region = Region.objects.create(id=9999) + stt = STT.objects.create(name="Test STT", region=region, stt_code="99") + assert stt.timezone == "America/New_York" + + +@pytest.mark.django_db +def test_stt_timezone_populated_for_states(stts): + """Test that populate_stts sets timezones for states.""" + arizona = STT.objects.get(name="Arizona") + assert arizona.timezone == "America/Phoenix" + + california = STT.objects.get(name="California") + assert california.timezone == "America/Los_Angeles" + + illinois = STT.objects.get(name="Illinois") + assert illinois.timezone == "America/Chicago" + + +@pytest.mark.django_db +def test_stt_timezone_populated_for_territories(stts): + """Test that populate_stts sets timezones for territories.""" + guam = STT.objects.get(name="Guam") + assert guam.timezone == "Pacific/Guam" + + puerto_rico = STT.objects.get(name="Puerto Rico") + assert puerto_rico.timezone == "America/Puerto_Rico" + + +@pytest.mark.django_db +def test_stt_timezone_populated_for_tribes(stts): + """Test that tribes get their own timezone, not inherited from state.""" + # Navajo Nation is in Arizona but observes DST (America/Denver) + navajo = STT.objects.get(name="Navajo Nation") + assert navajo.timezone == "America/Denver" + + # Hopi Tribe is in Arizona and does NOT observe DST (America/Phoenix) + hopi = STT.objects.get(name="Hopi Tribe") + assert hopi.timezone == "America/Phoenix" diff --git a/tdrs-frontend/cypress/e2e/common-steps/data_files.js b/tdrs-frontend/cypress/e2e/common-steps/data_files.js index 793ccd9ad..46adae2de 100644 --- a/tdrs-frontend/cypress/e2e/common-steps/data_files.js +++ b/tdrs-frontend/cypress/e2e/common-steps/data_files.js @@ -102,6 +102,9 @@ export const fillSttFyQNoProgramSelector = (stt, fy, q) => { }) } +export const fillStt = (stt) => + cy.get('#stt', { timeout: 1000 }).type(stt + '{enter}') + export const fillFyQProgram = (fy, q, program) => { cy.wait(500) cy.get('body').then(($body) => { @@ -122,7 +125,7 @@ export const fillFyQProgram = (fy, q, program) => { return } - if (program === 'SSP') { + if (program === 'PIA') { const hasPia = $body.find( 'label:contains("Program Integrity Audit")' ).length @@ -205,15 +208,19 @@ export const openDataFilesAndSearch = (program, year, quarter, stt = '') => { cy.get('#stt').should('exist').type(`${stt}{enter}`) } if (program === 'SSP') cy.get('label[for="ssp-moe"]').click() + else if (program === 'PIA') + cy.get('label[for="program-integrity-audit"]').click() cy.get('#reportingYears').should('exist').select(year) - cy.get('#quarter').should('exist').select(quarter) // Q1, Q2, Q3, Q4 + + if (program !== 'PIA') cy.get('#quarter').should('exist').select(quarter) // Q1, Q2, Q3, Q4 } export const uploadSectionFile = ( inputSelector, fileName, - shouldRejectInput = false + shouldRejectInput = false, + program = 'TANF' ) => { const filePath = `${TEST_DATA_DIR}/${fileName}` @@ -230,7 +237,16 @@ export const uploadSectionFile = ( 'not.have.class', 'is-loading' ) - cy.get('.usa-alert__text').should('not.exist') + + if (program === 'PIA') { + cy.get('.usa-alert__text') + .contains( + 'For Additional guidance please refer to the Program Instruction for this new reporting requirement.' + ) + .should('exist') + } else { + cy.get('.usa-alert__text').should('not.exist') + } cy.contains('button', 'Submit', { timeout: 5000 }).should( 'have.attr', 'data-has-uploaded-files', @@ -250,16 +266,24 @@ export const openSubmissionHistory = () => { cy.contains('button', 'Submission History').click() } -export const getLatestSubmissionHistoryRow = (section) => { +export const getLatestSubmissionHistoryRow = (section, program = 'TANF') => { const table_captions = { 1: 'Section 1 - Active Case Data', 2: 'Section 2 - Closed Case Data', 3: 'Section 3 - Aggregate Data', 4: 'Section 4 - Stratum Data', + PIA_1: 'Quarter 1 (October - December)', + PIA_2: 'Quarter 2 (January - March)', + } + + let sectionLabel = table_captions[section] + + if (program === 'PIA') { + sectionLabel = table_captions[`PIA_${section}`] } return cy - .contains('caption', table_captions[section]) + .contains('caption', sectionLabel) .parents('table') .find('tbody > tr') .first() @@ -279,9 +303,16 @@ export const downloadErrorReportAndAssert = ( 'Stratum Data', ] + let sectionLabel = ERROR_REPORT_LABELS[section - 1] + + if (programType === 'PIA') { + sectionLabel = ERROR_REPORT_LABELS[0] + } + // Download error report - const programPrefix = programType ? `${programType} ` : '' - const fileName = `${year}-${quarter}-${programPrefix}${ERROR_REPORT_LABELS[section - 1]} Error Report.xlsx` + const programPrefix = + programType && programType !== 'PIA' ? `${programType} ` : 'TANF ' + const fileName = `${year}-${quarter}-${programPrefix}${sectionLabel} Error Report.xlsx` const downloadedFilePath = `${Cypress.config('downloadsFolder')}/${fileName}` cy.intercept('GET', '/v1/data_files/*/download_error_report/').as( diff --git a/tdrs-frontend/cypress/e2e/data-files/file_upload.feature b/tdrs-frontend/cypress/e2e/data-files/file_upload.feature index 29db1ae9d..cfac427a3 100644 --- a/tdrs-frontend/cypress/e2e/data-files/file_upload.feature +++ b/tdrs-frontend/cypress/e2e/data-files/file_upload.feature @@ -44,22 +44,25 @@ Feature: Data file submission # TODO: And Regional Randy gets an email (determine exact) Examples: - | actor | program | section | stt | - | Data Analyst Tim | TANF | 1 | | - | Data Analyst Tim | TANF | 2 | | - | Data Analyst Tim | TANF | 3 | | - | Data Analyst Tim | TANF | 4 | | - | Data Analyst Stefani | SSP | 1 | | - | Data Analyst Stefani | SSP | 2 | | - | Data Analyst Stefani | SSP | 3 | | - | Data Analyst Stefani | SSP | 4 | | - | Data Analyst Tara | TRIBAL | 1 | | - | Data Analyst Tara | TRIBAL | 2 | | - | Data Analyst Tara | TRIBAL | 3 | | + | actor | program | section | stt | + | Data Analyst Tim | TANF | 1 | | + | Data Analyst Tim | TANF | 2 | | + | Data Analyst Tim | TANF | 3 | | + | Data Analyst Tim | TANF | 4 | | + | Data Analyst Stefani | SSP | 1 | | + | Data Analyst Stefani | SSP | 2 | | + | Data Analyst Stefani | SSP | 3 | | + | Data Analyst Stefani | SSP | 4 | | + | Data Analyst Tara | TRIBAL | 1 | | + | Data Analyst Tara | TRIBAL | 2 | | + | Data Analyst Tara | TRIBAL | 3 | | # We're only checking that non Data Analysts can submit a file # No need to test all programs and sections since they were tested above - | DIGIT Diana | TANF | 1 | Alabama | - | DIGIT Diana | TANF | 1 | Arkansas | + | DIGIT Diana | TANF | 1 | Alabama | + | DIGIT Diana | TANF | 1 | Arkansas | + | Data Analyst Tim | PIA | 1 | | + | Data Analyst Tim | PIA | 2 | | + | Admin Alex | PIA | 1 | California | # Edge / failure cases for TANF Data Analyst Tim diff --git a/tdrs-frontend/cypress/e2e/data-files/file_upload.js b/tdrs-frontend/cypress/e2e/data-files/file_upload.js index 878872166..0dfed62b5 100644 --- a/tdrs-frontend/cypress/e2e/data-files/file_upload.js +++ b/tdrs-frontend/cypress/e2e/data-files/file_upload.js @@ -40,6 +40,8 @@ Then('{string} can download the {string} error report', (actor, program) => { df.downloadErrorReport( '2024-Q2-FRA Work Outcomes of TANF Exiters Error Report.xlsx' ) + } else if (program === 'PIA') { + df.downloadErrorReport('2024-Q1-TANF Active Case Data Error Report.xlsx') } }) @@ -112,6 +114,8 @@ const SECTION_INPUT_ID = { 2: '#closed_case_data', 3: '#aggregate_data', 4: '#stratum_data', + PIA_1: '#quarter_1_october_-_december', + PIA_2: '#quarter_2_january_-_march', } const UPLOAD_FILE_INFO = { @@ -136,6 +140,10 @@ const UPLOAD_FILE_INFO = { 2: { fileName: 'small_correct_file.txt', year: '2021', quarter: 'Q1' }, 3: { fileName: 'small_correct_file.txt', year: '2021', quarter: 'Q1' }, }, + PIA: { + 1: { fileName: 'PI_Audit_space-fill.txt', year: '2024', quarter: 'Q1' }, + 2: { fileName: 'PI_Audit_FTANF.txt', year: '2024', quarter: 'Q2' }, + }, } // STEPS ---------- @@ -146,7 +154,12 @@ When( const { year, quarter, fileName } = UPLOAD_FILE_INFO[program][section] df.openDataFilesAndSearch(program, year, quarter, stt) - df.uploadSectionFile(SECTION_INPUT_ID[section], fileName) + + let sectionLabel = SECTION_INPUT_ID[section] + if (program === 'PIA') { + sectionLabel = SECTION_INPUT_ID[`PIA_${section}`] + } + df.uploadSectionFile(sectionLabel, fileName, false, program) } ) @@ -154,7 +167,7 @@ Then( '{string} sees the {string} Section {string} submission in Submission History', (actor, program, section) => { df.openSubmissionHistory() - df.getLatestSubmissionHistoryRow(section) + df.getLatestSubmissionHistoryRow(section, program) .should('exist') .within(() => { cy.contains(UPLOAD_FILE_INFO[program][section]['fileName']).should( @@ -169,7 +182,7 @@ Then( (actor, program, section) => { const { year, quarter } = UPLOAD_FILE_INFO[program][section] - df.getLatestSubmissionHistoryRow(section).within(() => { + df.getLatestSubmissionHistoryRow(section, program).within(() => { const programPrefix = program === 'TANF' ? 'TANF' @@ -179,7 +192,9 @@ Then( ? 'SSP' : program === 'Tribal' || program === 'TRIBAL' ? 'Tribal' - : '' + : program === 'PIA' + ? 'PIA' + : '' df.downloadErrorReportAndAssert(section, year, quarter, programPrefix) }) } diff --git a/tdrs-frontend/cypress/e2e/data-files/submission_history.feature b/tdrs-frontend/cypress/e2e/data-files/submission_history.feature index 8e4de6473..b47819bba 100644 --- a/tdrs-frontend/cypress/e2e/data-files/submission_history.feature +++ b/tdrs-frontend/cypress/e2e/data-files/submission_history.feature @@ -11,6 +11,10 @@ Feature: Data file submission history Given 'Admin Alex' logs in Then Admin Alex can view the Arizona FRA Submission History And Admin Alex can verify the Arizona FRA submission + Scenario: Admin Alex can view Program Integrity Audit Submissions + Given 'Admin Alex' logs in + When Admin Alex views the California PIA Submission History + Then Admin Alex can verify the California PIA submission Scenario: Regional Randy only has view access to submission historys for assigned locations Given FRA Data Analyst Fred submits a file Then 'Regional Randy' logs in diff --git a/tdrs-frontend/cypress/e2e/data-files/submission_history.js b/tdrs-frontend/cypress/e2e/data-files/submission_history.js index 8f7bdd6cb..6c22bccab 100644 --- a/tdrs-frontend/cypress/e2e/data-files/submission_history.js +++ b/tdrs-frontend/cypress/e2e/data-files/submission_history.js @@ -41,6 +41,21 @@ Then('Admin Alex can verify the Arizona FRA submission', () => { df.table_first_row_contains('fra.csv') df.table_first_row_contains('Partially Accepted with Errors') }) + +// PIA steps +When('Admin Alex views the California PIA Submission History', () => { + cy.visit('/data-files') + cy.get('h1').contains('TANF Data Files').should('exist') + df.fillStt('California') + df.fillFyQProgram('2024', '', 'PIA') + cy.get('button').contains('Submission History').click() +}) + +Then('Admin Alex can verify the California PIA submission', () => { + df.table_first_row_contains('PI_Audit_space-fill.txt') + df.table_first_row_contains('Accepted with Errors') +}) + /////////////////////////////////////////////////////////////// /////////////////////// Regional Steps //////////////////////// @@ -95,27 +110,3 @@ Given('FRA Data Analyst Fred submits a file', () => { } }) }) - -/////////////////////////////////////////////////////////////// - -Given('The admin logs in', () => { - cy.visit('/') - cy.adminLogin('cypress-admin@teamraft.com') -}) - -Then('{string} can see submission history', (username) => { - cy.get('h3').contains('Submission History').should('exist', { timeout: 5000 }) - cy.get('caption') - .contains('Section 1 - Active Case Data') - .should('exist', { timeout: 5000 }) -}) - -Then('{string} cannot see the upload form', (username) => { - cy.get('button').contains('Current Submission').should('not.exist') - cy.get('#active_case_data').should('not.exist') -}) - -Then('{string} sees the file in submission history', (username) => { - cy.get('th').contains('small_tanf_section1.txt').should('exist') - cy.get('th').contains('Accepted with Errors').should('exist') -})