diff --git a/.circleci/build-and-test/commands.yml b/.circleci/build-and-test/commands.yml index 307cc183a..a09588d1e 100644 --- a/.circleci/build-and-test/commands.yml +++ b/.circleci/build-and-test/commands.yml @@ -39,16 +39,45 @@ sudo apt-get update sudo apt-get install -y libgbm-dev source /opt/circleci/.nvm/nvm.sh - nvm install v16.13 - nvm alias default v16.13 + nvm install v22.13.0 + nvm alias default v22.13.0 echo 'export NVM_DIR="/opt/circleci/.nvm"' >> $BASH_ENV echo "[ -s \"$NVM_DIR/nvm.sh\" ] && . \"$NVM_DIR/nvm.sh\"" >> $BASH_ENV + echo "nvm use v22.13.0" >> $BASH_ENV + nvm use v22.13.0 + if command -v corepack >/dev/null 2>&1; then + corepack enable + corepack prepare yarn@4.6.0 --activate + else + npm install -g yarn@4.6.0 + fi + node -v + yarn -v - disable-npm-audit: + install-yarn-packages: + description: Install frontend dependencies using Yarn. + parameters: + app-dir: + type: string + default: tdrs-frontend steps: - run: - name: Disable npm audit warnings in CI - command: npm set audit false - - # This allows us to use the node orb to install packages within other commands - install-nodejs-packages: node/install-packages + name: Install Yarn packages + command: | + cd <> + if [ -z "$NVM_DIR" ]; then + export NVM_DIR="/opt/circleci/.nvm" + fi + if [ -s "$NVM_DIR/nvm.sh" ]; then + . "$NVM_DIR/nvm.sh" + nvm use v22.13.0 + fi + if command -v corepack >/dev/null 2>&1; then + corepack enable + corepack prepare yarn@4.6.0 --activate + elif ! command -v yarn >/dev/null 2>&1; then + npm install -g yarn@4.6.0 + fi + node -v + yarn -v + yarn install --immutable diff --git a/.circleci/build-and-test/jobs.yml b/.circleci/build-and-test/jobs.yml index ccb6eaa03..4b9e2c796 100644 --- a/.circleci/build-and-test/jobs.yml +++ b/.circleci/build-and-test/jobs.yml @@ -25,18 +25,17 @@ steps: - checkout - install-nodejs-machine - - disable-npm-audit - - install-nodejs-packages: + - install-yarn-packages: app-dir: tdrs-frontend - run: name: Run ESLint - command: cd tdrs-frontend; npm run lint + command: cd tdrs-frontend; yarn lint - run: name: Run Pa11y Accessibility Tests - command: cd tdrs-frontend; mkdir pa11y-screenshots/; export NODE_OPTIONS=--openssl-legacy-provider; npm run test:accessibility + command: cd tdrs-frontend; mkdir pa11y-screenshots/; export NODE_OPTIONS=--openssl-legacy-provider; yarn test:accessibility - run: name: Run Jest Unit Tests - command: cd tdrs-frontend; npm run test:ci + command: cd tdrs-frontend; yarn test:ci - upload-codecov: component: frontend coverage-report: ./tdrs-frontend/coverage/lcov.info @@ -53,8 +52,7 @@ - docker-compose-up-backend - docker-compose-up-frontend - install-nodejs-machine - - disable-npm-audit - - install-nodejs-packages: + - install-yarn-packages: app-dir: tdrs-frontend - run: name: Wait for backend to become available @@ -82,7 +80,7 @@ docker-compose exec web python manage.py loaddata cypress/users cypress/data_files cypress/regions cypress/profile_editing_regions cypress/profile_editing_users - run: name: Run Cypress e2e tests - command: cd tdrs-frontend; npm run test:e2e-ci + command: cd tdrs-frontend; yarn test:e2e-ci - store_artifacts: path: tdrs-frontend/cypress/screenshots/ - store_artifacts: diff --git a/.circleci/deployment/commands.yml b/.circleci/deployment/commands.yml index b1fb028fa..067d6c76a 100644 --- a/.circleci/deployment/commands.yml +++ b/.circleci/deployment/commands.yml @@ -137,11 +137,12 @@ type: string steps: - install-nodejs: - node-version: "16.13" - - disable-npm-audit - - install-nodejs-packages: + node-version: "22.13.0" + - run: + name: Log Node version + command: node -v + - install-yarn-packages: app-dir: tdrs-frontend - override-ci-command: npm ci --omit=dev --no-fund - get-app-deploy-strategy: appname: <> - run: diff --git a/.circleci/deployment/jobs.yml b/.circleci/deployment/jobs.yml index 1538f464a..4ef3d4d6f 100644 --- a/.circleci/deployment/jobs.yml +++ b/.circleci/deployment/jobs.yml @@ -121,12 +121,11 @@ steps: - checkout - install-nodejs-machine - - disable-npm-audit - - install-nodejs-packages: + - install-yarn-packages: app-dir: tdrs-frontend - run: name: Run Cypress e2e tests - command: cd tdrs-frontend; npm run test:e2e-ci -- --config baseUrl="https://tdp-frontend-develop.acf.hhs.gov" --env cypressToken=$CYPRESS_TOKEN,apiUrl="https://tdp-frontend-develop.acf.hhs.gov/v1",adminUrl="https://tdp-frontend-develop.acf.hhs.gov/admin" + command: cd tdrs-frontend; yarn test:e2e-ci --config baseUrl="https://tdp-frontend-develop.acf.hhs.gov" --env cypressToken=$CYPRESS_TOKEN,apiUrl="https://tdp-frontend-develop.acf.hhs.gov/v1",adminUrl="https://tdp-frontend-develop.acf.hhs.gov/admin" - store_artifacts: path: tdrs-frontend/cypress/screenshots/ - store_artifacts: diff --git a/.github/ISSUE_TEMPLATE/release-tracker-issue-template.md b/.github/ISSUE_TEMPLATE/release-tracker-issue-template.md new file mode 100644 index 000000000..e912da957 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/release-tracker-issue-template.md @@ -0,0 +1,90 @@ +--- +name: Release Tracker +about: Track the release handoff to OFA, staging validation, and production deployment. +title: Release Tracker vX.X.X +labels: '' +assignees: '' + +--- + +# Release Tracker Issue Template + +**TITLE:** Release vX.X.X + +**CONTENT:** + +### πŸ”— Included Pull Requests (Dev Team) +*List the PRs included in this release. **Testing instructions for each feature must be located within these linked PRs.*** +* #XXX - [Feature/Bugfix Title] +* #XXX - [Feature/Bugfix Title] + +### ➑️ URL of GitHub Release Tag: +https://github.com/raft-tech/TANF-app/releases/tag/vX.X.X + +--- + +### πŸ“¦ 1. Preparation & Handoff (Dev / UX / PM) +*Ensuring the release branch is ready and documented before ACF takes over.* + +- [ ] **Release Tagged:** `develop` branch tagged with the new release version. +- [ ] **Release Branch Created:** Branch cut from `develop`. +- [ ] **PR Opened to Staging:** PR opened from the release branch to `HHS:main`. +- [ ] **Testing Instructions Verified:** All linked PRs contain clear testing instructions for ACF validation. +- [ ] **UX/Documentation Check:** UX team has reviewed the PRs, confirmed user-facing changes, and started drafting Release Notes and Knowledge Center guidance. +- [ ] **Migration Flag:** Does this release include a database migration? **[Yes / No]** *(If Yes, rollback from production will be highly complex).* +- [ ] **Handoff Complete:** Issue assigned to @[Alex_username] for Staging validation. + +--- + +### πŸ§ͺ 2. Staging Validation & QASP (ACF / Alex) +*Tracking the status once ACF takes over deployment and testing.* + +- [ ] **Staging Cleared:** Team notified that Staging is about to be updated/restarted. +- [ ] **Deployed to Staging:** PR merged and deployed to the Staging environment. +- [ ] **Feature Validation:** Testing instructions from the linked PRs have been executed and passed. +- [ ] **Regression Validation:** Core workflows (login, submissions, data integrity, etc.) remain functional. + +**πŸ› Bug Tracking Protocol (If issues are found in Staging):** +1. **Non-Dev Team:** Add a comment on this issue describing the bug/unexpected behavior. +2. **Dev Team:** Review the comment, investigate root cause, and open a formal GitHub Issue. +3. **Triage Decision:** + * *Revert:* If isolated to a new feature, revert the PR out of the release candidate. + * *Hotfix:* Warranted **only if** the bug blocks the release entirely AND the production release is needed ASAP. (Dev cuts hotfix PR against the release branch -> merged to `HHS:main` for re-testing). + +- [ ] **Documentation Finalized:** UX team confirms all Release Notes and Knowledge Center updates are finalized and ready for launch. + +--- + +### 🚦 3. Production-Ready Sign-Off +- [ ] ACF/Alex confirms all PRs are validated, no blocking bugs exist, documentation is finalized, and the release is approved for production deployment. + +--- + +### πŸ›Ÿ 4. Rollback & Contingency Reference +*Review before production deployment.* + +* **Pipeline/CircleCI Failure:** Retry the pipeline. If it fails again, requires a hotfix to unblock. +* **Missing Config/Secrets (App crashes on boot):** Do not rollback. ACF updates environment variables in the production console and restarts the app. +* **Third-Party API Blocked in Prod:** Dev provides an emergency hotfix to hide the broken UI component. +* **Performance/Database Lockup Under Load:** Dev writes an emergency hotfix for the offending query. +* **Critical Regression Post-Deploy:** + * *No migrations in release:* Rollback the deployment to the previous stable version. + * *Migrations in release:* Rollback is generally not possible; requires an emergency hotfix. + +--- + +### πŸš€ 5. Production Deployment (ACF / Alex) +*The final deployment executed by ACF.* + +- [ ] **Maintenance Mode:** Alex has enabled the maintenance page to ensure users are out of the system. +- [ ] **Deployed to Prod:** PR opened and merged to `HHS:master`, triggering deployment. +- [ ] **Post-Launch Verification:** Quick check that the production environment is stable post-deploy, and the maintenance page is deactivated. + +--- + +### πŸ“’ 6. Post-Release Communication (PM / UX / ACF) +*Closing the loop with users and stakeholders.* + +- [ ] **Release Notes Published:** (UX team) Plain-language, user-facing release notes and Knowledge Center guidance have been published. +- [ ] **Stakeholders Notified:** (ACF) Any required external communication regarding the new version has been sent. +- [ ] **Close this Issue:** (ACF / PM) The release is fully deployed and stable. diff --git a/.gitignore b/.gitignore index 8bfe5a36a..447283e14 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ tdrs-frontend/yarn-error.log* tdrs-frontend/vars-frontend.yml tdrs-frontend/pa11y-screenshots/* tdrs-frontend/cypress/reports* +tdrs-frontend/test-results*.xml .eslintcache downloads/ @@ -125,8 +126,3 @@ cypress.env.json tdrs-backend/*.pg tdrs-backend/django.log -# Pytest cache -tdrs-backend/.pytest_cache/* - -# frontend cypress test results -tdrs-frontend/test-results*.xml \ No newline at end of file diff --git a/Taskfile.yml b/Taskfile.yml index 3be5319ce..99c9a216c 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -227,21 +227,21 @@ tasks: dir: tdrs-frontend cmds: - docker compose -f docker-compose.local.yml up -d --build tdp-frontend - - docker compose -f docker-compose.local.yml exec tdp-frontend sh -c "npm install" + - docker compose -f docker-compose.local.yml exec tdp-frontend sh -c "corepack prepare yarn@4.6.0 && corepack yarn install" - docker compose -f docker-compose.local.yml down frontend-install: desc: Install frontend dependencies on host (uses local Node) dir: tdrs-frontend cmds: - - npm install + - corepack prepare yarn@4.6.0 --activate && yarn install frontend-update: desc: Rebuild frontend localdev image and install dependencies (no data wipe) dir: tdrs-frontend cmds: - docker compose -f docker-compose.local.yml up -d --build tdp-frontend - - docker compose -f docker-compose.local.yml exec tdp-frontend sh -c "npm install" + - docker compose -f docker-compose.local.yml exec tdp-frontend sh -c "corepack prepare yarn@4.6.0 && corepack yarn install" frontend-test-local: desc: Run frontend unit tests on host (watch-less) @@ -268,7 +268,7 @@ tasks: JEST_ARGS: '{{.JEST_ARGS | default ""}}' cmds: - docker compose -f docker-compose.local.yml up -d --build tdp-frontend-test - - docker compose -f docker-compose.local.yml exec tdp-frontend-test sh -c "npm run test -- {{.JEST_ARGS}}" + - docker compose -f docker-compose.local.yml exec tdp-frontend-test sh -c "corepack prepare yarn@4.6.0 && corepack yarn install --mode=skip-build && CI=1 corepack yarn test --watchAll=false {{.JEST_ARGS}}" frontend-test-cov: desc: 'Run frontend unit tests with coverage in docker. E.g: task frontend-test-cov JEST_ARGS="--testPathPattern=FeedbackReports"' @@ -277,14 +277,14 @@ tasks: JEST_ARGS: '{{.JEST_ARGS | default ""}}' cmds: - docker compose -f docker-compose.local.yml up -d --build tdp-frontend-test - - docker compose -f docker-compose.local.yml exec tdp-frontend-test sh -c "npm run test:cov -- {{.JEST_ARGS}}" + - docker compose -f docker-compose.local.yml exec tdp-frontend-test sh -c "corepack prepare yarn@4.6.0 && corepack yarn install --mode=skip-build && corepack yarn test:cov {{.JEST_ARGS}}" frontend-lint: desc: Run eslint in the frontend test container dir: tdrs-frontend cmds: - docker compose -f docker-compose.local.yml up -d --build tdp-frontend-test --quiet-pull - - docker compose -f docker-compose.local.yml exec -T tdp-frontend-test sh -c "npm run lint" + - docker compose -f docker-compose.local.yml exec -T tdp-frontend-test sh -c "corepack prepare yarn@4.6.0 && corepack yarn install --mode=skip-build && corepack yarn lint" # E2E / Cypress helpers @@ -293,15 +293,44 @@ tasks: dir: tdrs-backend cmds: - export CYPRESS_TOKEN=local-cypress-token - - docker-compose exec web python manage.py delete_cypress_users -usernames new-cypress@teamraft.com cypress-admin@teamraft.com - - docker-compose exec web python manage.py loaddata cypress/users cypress/data_files cypress/regions cypress/profile_editing_regions cypress/profile_editing_users + - docker compose -f docker-compose.yml exec web python manage.py delete_cypress_users -usernames new-cypress@teamraft.com cypress-admin@teamraft.com cypress-data-analyst-dana@teamraft.com cypress-fra-data-analyst-derek@teamraft.com cypress-data-analyst-donna@teamraft.com cypress-fra-data-analyst-david@teamraft.com cypress-fra-ofa-regional-staff-rachel@acf.hhs.gov cypress-fra-ofa-regional-staff-robert@acf.hhs.gov cypress-fra-ofa-regional-staff-rita@acf.hhs.gov cypress-fra-ofa-regional-staff-ryan@acf.hhs.gov + - docker compose -f docker-compose.yml exec web python manage.py loaddata cypress/users cypress/data_files cypress/regions cypress/profile_editing_regions cypress/profile_editing_users frontend-e2e-local: desc: Run Cypress E2E tests locally (Cypress on host, app in docker) dir: tdrs-frontend cmds: - docker compose -f docker-compose.local.yml up -d --build tdp-frontend - - npm run test:e2e + - | + if command -v corepack >/dev/null 2>&1; then + corepack prepare yarn@4.6.0 --activate && env -u ELECTRON_RUN_AS_NODE yarn test:e2e + else + env -u ELECTRON_RUN_AS_NODE npx --yes @yarnpkg/cli-dist@4.6.0 test:e2e + fi + + frontend-e2e-ci-local: + desc: Run Cypress E2E tests in command-line mode (headless, app in docker) + dir: tdrs-frontend + vars: + CYPRESS_TOKEN: '{{.CYPRESS_TOKEN | default "local-cypress-token"}}' + cmds: + - task: backend-up + - task: e2e-env-var-setup + - docker compose -f docker-compose.local.yml up -d --build tdp-frontend + - env -u ELECTRON_RUN_AS_NODE npx cypress verify || env -u ELECTRON_RUN_AS_NODE npx cypress install --force + - | + if command -v corepack >/dev/null 2>&1; then + env -u ELECTRON_RUN_AS_NODE yarn test:e2e-ci + else + env -u ELECTRON_RUN_AS_NODE npx --yes @yarnpkg/cli-dist@4.6.0 test:e2e-ci + fi + + frontend-e2e-cmd: + desc: Run Cypress E2E tests headlessly from the command line (app in docker) + dir: tdrs-frontend + cmds: + - docker compose -f docker-compose.yml up -d --build tdp-frontend + - CYPRESS_TOKEN=${CYPRESS_TOKEN:-local-cypress-token} npm run test:e2e-ci k6: desc: Run k6 performance tests diff --git a/cf-memory-usage.sh b/cf-memory-usage.sh new file mode 100755 index 000000000..be204fcc7 --- /dev/null +++ b/cf-memory-usage.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +# Show memory allocation per app in the current CF space vs the space's quota. + +set -euo pipefail + +if ! command -v cf &>/dev/null; then + echo "Error: cf CLI not found. Install it first." + exit 1 +fi + +if ! cf target &>/dev/null; then + echo "Error: Not logged in to Cloud Foundry. Run 'cf login' first." + exit 1 +fi + +target_info=$(cf target) +org=$(echo "$target_info" | awk '/^org:/{print $2}') +space=$(echo "$target_info" | awk '/^space:/{print $2}') + +echo "Org: $org" +echo "Space: $space" +echo "" + +# Get app names from cf apps (skip header lines) +app_names=$(cf apps | awk 'NR>4 && NF>0 {print $1}') + +if [ -z "$app_names" ]; then + echo "No apps found in this space." + exit 0 +fi + +printf "%-40s %8s %12s %15s %15s\n" "APP" "STATE" "INSTANCES" "ALLOCATED" "USED" +printf "%-40s %8s %12s %15s %15s\n" "---" "-----" "---------" "---------" "----" + +total_allocated_mb=0 + +for app_name in $app_names; do + app_info=$(cf app "$app_name" 2>/dev/null || true) + + state=$(echo "$app_info" | awk '/^requested state:/{print $3}') + instances=$(echo "$app_info" | awk '/^instances:/{print $2}') + mem_alloc=$(echo "$app_info" | awk '/^memory usage:/{print $3}') + + # Parse instance memory usage from the instance line (e.g. "809M of 1G") + mem_used=$(echo "$app_info" | awk '/^#[0-9]/{print $4}' | head -1) + [ -z "$mem_used" ] && mem_used="N/A" + + # Parse allocated memory into MB + alloc_val=$(echo "$mem_alloc" | sed 's/[^0-9.]//g') + alloc_unit=$(echo "$mem_alloc" | sed 's/[0-9.]//g') + case "$alloc_unit" in + G) alloc_mb=$(echo "$alloc_val * 1024" | bc) ;; + M) alloc_mb="$alloc_val" ;; + *) alloc_mb=0 ;; + esac + + # Multiply by total instances + total_inst=$(echo "$instances" | cut -d'/' -f2) + [ -z "$total_inst" ] && total_inst=1 + app_total_mb=$(echo "$alloc_mb * $total_inst" | bc) + + total_allocated_mb=$(echo "$total_allocated_mb + $app_total_mb" | bc) + + printf "%-40s %8s %12s %13sMB %15s\n" "$app_name" "$state" "$instances" "$app_total_mb" "$mem_used" +done + +echo "" +echo "--- Summary ---" +echo "Total memory allocated across all apps: ${total_allocated_mb}MB" + + +echo "" +echo "" +echo "" +echo "--- Space Quota Available ---" + +# Derive env from space name (expects space like "tanf-dev", "tanf-staging", "tanf-prod") +env=$(echo "$space" | awk -F'-' '{print $NF}') +quota_name="${env}_quota" + +echo "Using space quota: $quota_name" +cf space-quota "$quota_name" 2>/dev/null || echo "Could not find space quota '$quota_name'." diff --git a/docs/Technical-Documentation/circle-ci.md b/docs/Technical-Documentation/circle-ci.md index 19c6460bf..f0e29f23d 100644 --- a/docs/Technical-Documentation/circle-ci.md +++ b/docs/Technical-Documentation/circle-ci.md @@ -64,7 +64,7 @@ These all have defaults set in their respective settings modules, but may be ove ## Frontend CI build process ### test-frontend -* Runs most steps directly on the machine executor, utilizing `npm` commands defined in package.json +* Runs most steps directly on the machine executor, utilizing Yarn commands defined in package.json * The exception to the above is the zap scanner step - which runs the frontend via docker-compose, using the nginx target instead of the local dev target * Major steps: * Run ESLint - ensures styling standards are followed @@ -76,13 +76,13 @@ These all have defaults set in their respective settings modules, but may be ove ### deploy-frontend * Called as a step in the `deploy-cloud-dot-gov` command * Runs directly on the machine executor -* Installs Node.JS v12.18 and all project dependencies +* Installs Node.JS v22.13.0 and all project dependencies * Calls script `/scripts/deploy-frontend.sh`, which does the following: * Using cloud.gov application name as an input (`CGHOSTNAME_BACKEND`) it sets the environment variables needed to communicate with the Django backend: * `REACT_APP_BACKEND_HOST` * `REACT_APP_BACKEND_URL` * Only difference in values is whether `/v1` is at the end - * Runs `npm run build` which generates the HTML needed to serve to end users + * Runs `yarn build` which generates the HTML needed to serve to end users * Copies in the nginx configuration for build packs * Uploads the build output to Cloud.gov using `cf push` * Creates and maps the frontend route @@ -122,4 +122,4 @@ The Frontend and Backend deploy Workflows are triggered automatically on pushes * `Deploy with CircleCI` is the prefix part of the label which triggers the build. It needs the environment added as a suffix * To select the environment, add the name after a hyphen following the `Deploy with CircleCI` prefix * e.g. `Deploy with CircleCI-raft` will deploy your branch build to the tanf-dev Cloud Foundry space, tdp-raft environment - * tdp-frontend-raft & tdp-backend-raft \ No newline at end of file + * tdp-frontend-raft & tdp-backend-raft diff --git a/docs/Technical-Documentation/cypress-integration-tests.md b/docs/Technical-Documentation/cypress-integration-tests.md index 47062db90..8e46dfde0 100644 --- a/docs/Technical-Documentation/cypress-integration-tests.md +++ b/docs/Technical-Documentation/cypress-integration-tests.md @@ -26,7 +26,7 @@ All tests added into the `tdrs-frontend/cypress/e2e/` folder will be run against 1. In a new terminal, run the following commands to launch the Cypress runner ```bash cd tdrs-frontend - npm run test:e2e + yarn test:e2e ``` 1. Select "E2E Testing" from the testing type menu ![Select e2e testing](./images/testing/01-e2e-selection.png) @@ -98,4 +98,4 @@ Then('{string} sees a Request Access form', (username) => { For each step implementation, a good rule of thumb is to perform both an action and an [assertion](https://docs.cypress.io/guides/references/assertions#Chai). An action should be something the user can do in the system (click, type, etc.). Assertions help "slow down" the test and limit unexpected behavior when applications run a lot of asynchronous processes. By asserting on something verifiable in each step, we can ensure the test is in a proper state to move forward. This applies to `Given`, `When`, and `Then` steps (though `Then` steps can often omit an action). -Shared step implementations, which apply to all feature files, can be added as [common step definitions](https://github.com/badeball/cypress-cucumber-preprocessor/blob/master/docs/step-definitions.md#example-2-directory-with-common-step-definitions) in `tdrs-frontend/cypress/e2e/common-steps/common-steps.js` \ No newline at end of file +Shared step implementations, which apply to all feature files, can be added as [common step definitions](https://github.com/badeball/cypress-cucumber-preprocessor/blob/master/docs/step-definitions.md#example-2-directory-with-common-step-definitions) in `tdrs-frontend/cypress/e2e/common-steps/common-steps.js` diff --git a/docs/Technical-Documentation/tech-memos/go-parser/architecture_and_integration_plan.md b/docs/Technical-Documentation/tech-memos/go-parser/architecture_and_integration_plan.md new file mode 100644 index 000000000..9b2ad9f55 --- /dev/null +++ b/docs/Technical-Documentation/tech-memos/go-parser/architecture_and_integration_plan.md @@ -0,0 +1,580 @@ +# Go Parser Integration Plan + +High-level architecture plan for integrating the Go parser into the TDP production environment. + +## Table of Contents + +1. [Summary](#summary) +2. [Benefits](#benefits) +3. [System Architecture](#system-architecture) +4. [Data Flow](#data-flow) +5. [Integration Strategy](#integration-strategy) +6. [Configuration Management](#configuration-management) +7. [Infrastructure Requirements](#infrastructure-requirements) +8. [Migration Strategy](#migration-strategy) +9. [Risks, Dependencies, and Open Questions](#risks-dependencies-and-open-questions) + +--- + +## Summary + +The Go parser is a ground-up rewrite of the TDP data file parsing system. It replaces the Python parser's class-based, single-threaded approach with a YAML-driven, parallel pipeline written in Go. The parser handles all four program types (TANF, SSP, Tribal TANF, FRA) across all sections, producing identical database records and parser errors. + +The integration strategy is designed so the Go parser slots into the existing infrastructure with minimal changes to the Django application. It consumes Celery tasks from the same Redis broker, reads files from the same S3-compatible blob storage, and writes to the same PostgreSQL tables. The Django frontend, API, and all downstream systems are unaffected. + +--- + +## Benefits + +### 20x Faster Parsing, Half the Memory + +The Go parser's parallel pipeline delivers roughly 20x the throughput of the Python parser while consuming half the memory. The Python parser processes records single-threaded through the Django ORM; the Go parser decodes, parses, validates, and writes concurrently across worker pools and flushes records in bulk via `COPY FROM`. This isn't a marginal improvement β€” it changes what's operationally feasible. + +### Minutes Instead of Hours for Bulk Reparse + +Admin-triggered reparse events currently queue hundreds of files through the single-threaded Python parser, often taking hours to complete. With the Go parser's throughput, the same reparse operations complete in minutes. This directly improves the feedback loop for administrators correcting data quality issues or reprocessing after rule changes. + +### Automated Error Documentation from Configuration + +Because all schemas, validators, and error messages are defined in YAML configuration files and loaded through the registry at startup, a separate Go program can consume the same registry, filespecs, schemas, and validators to generate a comprehensive document of every possible error a given file type or record type can produce. This gives stakeholders, support staff, and submitters a complete, always-current reference β€” generated directly from the source of truth rather than maintained by hand. + +### Flexible Deployment: Microservice, Lambda, or Celery Worker + +The Go parser's core pipeline has no opinion about how work arrives. The ingestion layer is a thin adapter that can be swapped without touching parsing, validation, or writing logic. This means the parser can run as: + +- **Celery worker** β€” the current integration path, consuming tasks from Redis alongside the Django application +- **HTTP service** β€” accepting file uploads directly, running as a standalone microservice behind a load balancer +- **gRPC service** β€” for high-throughput internal communication between services +- **Lambda function** β€” triggered by S3 upload events/HTTP for serverless, on-demand processing + +Adding a new communication mode is a matter of writing a new entrypoint that calls `Pipeline.ProcessFile()`. The compiled binary, configuration loading, and all pipeline internals remain identical across deployment modes. + +### Local Dry-Run Validation for Submitters + +Because Go compiles to a single static binary with no runtime dependencies, the parser can be distributed directly to data submitters. Users can run the binary against their files locally before uploading to TDP, catching validation errors immediately without waiting for a round-trip through the system. The dry-run mode uses the same schemas, validators, and error messages as production β€” the output is identical to what the server would produce, minus the database writes. This reduces submission cycles, decreases the volume of rejected files, and gives submitters confidence in their data before they hit upload. + +--- + +## System Architecture + +### Current Production Architecture + +``` +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ Frontend │────▢│ Django API │────▢│ S3 / Local β”‚ β”‚ PostgreSQL β”‚ +β”‚ (React) β”‚ β”‚ (upload) β”‚ β”‚ Storage β”‚ β”‚ β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β–²β”€β”€β”€β”€β”€β”€β”˜ + β”‚ β”‚ β”‚ + β–Ό β”‚ β”‚ + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ β”‚ + β”‚ Redis β”‚ β”‚ β”‚ + β”‚ (broker) β”‚ β”‚ β”‚ + β””β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”˜ β”‚ β”‚ + β”‚ β”‚ β”‚ + β–Ό β–Ό β”‚ + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ + β”‚ Python Celery Worker β”‚ β”‚ + β”‚ β”‚β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ parse(data_file_id) task β”‚ + β”‚ - Reads file from S3 β”‚ + β”‚ - Parses records (single-threaded) β”‚ + β”‚ - Validates fields/records β”‚ + β”‚ - Writes to Django ORM β”‚ + β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +``` + +### Target Architecture with Go Parser + +``` +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ Frontend │────▢│ Django API │────▢│ S3 / Local β”‚ β”‚ PostgreSQL β”‚ +β”‚ (React) β”‚ β”‚ (upload) β”‚ β”‚ Storage β”‚ β”‚ β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β–²β”€β”€β”€β”€β”€β”€β”˜ + β”‚ β”‚ β”‚ + β–Ό β”‚ β”‚ + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ β”‚ + β”‚ Redis β”‚ β”‚ β”‚ + β”‚ (broker) β”‚ β”‚ β”‚ + β””β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”˜ β”‚ β”‚ + β”‚ β”‚ β”‚ β”‚ + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ └──────────┐ β”‚ β”‚ + β–Ό β–Ό β–Ό β”‚ + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ + β”‚ Python Celery β”‚ β”‚ Go Parser Worker β”‚ β”‚ + β”‚ Worker β”‚ β”‚ β”‚β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ │──────│ - Reads files from S3 β”‚ + β”‚ post-parse tasks β”‚ β”‚ - Parallel record parsing β”‚ + β”‚ (email, summary β”‚ β”‚ - Parallel validation β”‚ + β”‚ aggregation) β”‚ β”‚ - Parallel Postgres writes β”‚ + β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +``` + +Key architectural decisions: + +- **Go parser runs as its own container** alongside the existing Django/Celery containers +- **Same Redis broker** β€” the Go worker uses `gocelery` to consume tasks from the same Redis instance +- **Same PostgreSQL database** β€” the Go parser writes directly to the existing `search_indexes_*` and `parser_error` tables using `pgx` and `COPY FROM` +- **Same S3 storage** β€” the Go parser uses `aws-sdk-go-v2` to read submitted files from the same bucket +- **Django remains the orchestrator** β€” Django enqueues parse tasks and handles all pre/post-parse logic (DataFile status, DataFileSummary, email notifications, reparsing) + +### Subsystem Overview + +| Subsystem | Package | Responsibility | +|-----------|---------|----------------| +| **Task Consumption** | `celery` | `gocelery` Redis worker receives `parse` tasks dispatched by Django | +| **Blob Storage** | `storage` | Retrieves submitted data files from S3 (or LocalStack in dev) | +| **Pipeline Orchestration** | `pipeline` | Coordinates the full decode β†’ presort β†’ parse β†’ validate β†’ write flow | +| **Decoding** | `decoder` | Reads raw files into `Row` objects (positional, CSV, XLSX) | +| **Pre-sorting** | `parser` | Stable-sorts data rows by case key for streaming group accumulation | +| **Parsing** | `parser` | Record type detection, field extraction, type conversion via worker pool | +| **Validation** | `validation` | Expression-based field and record validation using `expr` engine | +| **Writing** | `writer` | Bulk database writes via `pgx COPY FROM`, routed by record type | +| **Configuration** | `config` | YAML-driven schemas, filespecs, validators, and pipeline tuning | + +--- + +## Data Flow + +### End-to-End: File Submission Through Database Persistence + +``` +1. User uploads file via React frontend + β”‚ + β–Ό +2. Django API receives upload + - Creates DataFile record + - Stores file in S3 (or LocalStack) + - Creates DataFileSummary (PENDING) + - Enqueues Celery task: parse(data_file_id) + β”‚ + β–Ό +3. Redis broker holds the task + β”‚ + β–Ό +4. Go parser worker picks up the task + a. Reads DataFile metadata from PostgreSQL (file path, program, section) + b. Downloads file from S3 into memory + c. Resolves FileSpec from (program, section) via config registry + β”‚ + β–Ό +5. Pipeline.ProcessFile() executes: + + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” + β”‚ PIPELINE STAGES β”‚ + β”‚ β”‚ + β”‚ 5a. DECODE β”‚ + β”‚ Decoder reads raw file β†’ []Row (with original line nums) β”‚ + β”‚ β”‚ β”‚ + β”‚ 5b. PRESORT β”‚ + β”‚ Stable-sort rows by `key_fields` (RPT_MONTH_YEAR, CASE_NUMBER) β”‚ + β”‚ Preserves original line numbers for error reporting β”‚ + β”‚ β”‚ β”‚ + β”‚ 5c. ACCUMULATE β”‚ + β”‚ Stream sorted rows β†’ detect key changes β†’ emit batches β”‚ + β”‚ One active group in memory at a time β”‚ + β”‚ β”‚ β”‚ + β”‚ 5d. PARSE (parallel, N worker goroutines) β”‚ + β”‚ Extract fields per schema, convert types β”‚ + β”‚ Worker pool processes batches in parallel β”‚ + β”‚ β”‚ β”‚ + β”‚ 5e. VALIDATE (parallel, N validator goroutines) β”‚ + β”‚ Field-level validators (category 2) β”‚ + β”‚ Record-level validators (category 1 & 3) β”‚ + β”‚ Group-level validators (category 4) β”‚ + β”‚ Expression engine evaluates YAML-defined rules β”‚ + β”‚ β”‚ β”‚ + β”‚ 5f. WRITE (bulk) β”‚ + β”‚ Convert ParsedRecords β†’ DB model structs β”‚ + β”‚ Bulk insert via pgx COPY FROM β”‚ + β”‚ Flush at configurable thresholds (10k records, 50k errors) β”‚ + β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ + β–Ό +6. Go parser updates DataFileSummary status + β”‚ + β–Ό +7. Python Celery worker handles post-parse tasks + - Email notifications + - Reparse bookkeeping + - Summary aggregation +``` + +### Pipeline Concurrency Model + +``` + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” + β”‚ Accumulator β”‚ + β”‚ (single goroutine) β”‚ + β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ batches (channel, buffer=256) + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β–Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” + β”‚ Parser Pool β”‚ + β”‚ (N worker goroutines) β”‚ + β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ parsed results (channel, buffer=256) + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β–Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” + β”‚ Result Routers β”‚ + β”‚ (N router goroutines) β”‚ + β”‚ - Validate per group β”‚ + β”‚ - Route to writer β”‚ + β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β–Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” + β”‚ Writer β”‚ + β”‚ (Writer pre record type)β”‚ + β”‚ - Buffered flushes β”‚ + β”‚ - 10k record threshold β”‚ + β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +``` + +All pool sizes and buffer depths are configurable via `config/pipeline.yaml`: + +```yaml +pipeline: + num_workers: 16 + work_buffer_size: 256 + result_buffer_size: 256 + num_routers: 16 + num_validators: 16 + pool_prewarm_size: 10000 +writer: + flush_threshold: 10000 + error_flush_threshold: 50000 +database: + max_conns: 10 + min_conns: 4 +``` + +--- + +## Integration Strategy + +### Celery Task Consumption + +The Go parser integrates with the existing Django task dispatch system through a shared Redis broker: + +1. **Django dispatches** β€” `scheduling/parser_task.py` enqueues `parse(data_file_id)` to Redis (unchanged) +2. **Go worker consumes** β€” `internal/celery/redis_worker.go` uses `gocelery` to listen on the same Redis queue +3. **Task routing** β€” Celery task routing directs parse tasks to the Go worker's queue while other tasks (email, reparse) remain on the Python worker's queue + +```python +# Django settings addition for task routing +CELERY_TASK_ROUTES = { + 'tdpservice.scheduling.parser_task.parse': {'queue': 'go_parser'}, +} +``` + +```go +// Go worker registers the parse task handler +tasks := map[string]interface{}{ + "tdpservice.scheduling.parser_task.parse": parseHandler, +} +worker := celery.NewRedisWorker(redisURL, tasks) +worker.Start() +``` + +### Shared Database + +The Go parser writes to the same PostgreSQL tables as the Python parser: + +| Table | Purpose | Write Method | +|-------|---------|--------------| +| `search_indexes_tanf_t1` through `t7` | Parsed TANF records | `COPY FROM` | +| `search_indexes_ssp_m1` through `m7` | Parsed SSP records | `COPY FROM` | +| `search_indexes_tribal_tanf_t1` through `t7` | Parsed Tribal records | `COPY FROM` | +| `search_indexes_fra_te1` | Parsed FRA records | `COPY FROM` | +| `parser_error` | Validation errors | `COPY FROM` | +| `parsers_datafilesummary` | Processing status | UPDATE via query | + +Schema compatibility is enforced by SQLC: `schema.sql` mirrors the Django model definitions, and `sqlc generate` produces Go structs that match the database columns exactly. The `writer/convert` package handles the translation from internal `ParsedRecord` types to these DB model structs. + +### Blob Storage + +The Go parser uses `aws-sdk-go-v2` to access the same S3 bucket (or LocalStack in development): + +- **Production**: Real S3 bucket configured via `AWS_S3_DATAFILES_BUCKET_NAME` +- **Development**: LocalStack S3 at `http://localstack:4566` +- **Credentials**: Same IAM role / environment variables used by the Django application + +### Post-Parse Orchestration + +After the Go parser completes file processing, the Python Celery worker handles remaining tasks that depend on Django ORM and application logic: + +- DataFileSummary status update (accepted/rejected counts, error aggregation) +- Email notifications to submitters +- Reparse bookkeeping and cleanup + +This can be done by having the Go parser enqueue a `post_parse(data_file_id)` task back to the Python worker's queue upon completion, or by having Django poll the DataFileSummary status. + +--- + +## Configuration Management + +### YAML-Driven Architecture + +All parsing behavior is defined in YAML configuration files, not in code. This is a fundamental design difference from the Python parser, where schemas, validators, and field definitions are embedded in Python classes. + +``` +config/ +β”œβ”€β”€ filespecs/ # One YAML per (program, section) +β”‚ β”œβ”€β”€ tanf/ +β”‚ β”‚ β”œβ”€β”€ section1.yaml # TANF Active Case Data +β”‚ β”‚ β”œβ”€β”€ section2.yaml # TANF Closed Case Data +β”‚ β”‚ β”œβ”€β”€ section3.yaml # TANF Aggregate Data +β”‚ β”‚ └── section4.yaml # TANF Stratum Data +β”‚ β”œβ”€β”€ ssp/ # Same structure for SSP +β”‚ β”œβ”€β”€ tribal/ # Same structure for Tribal TANF +β”‚ └── fra/ +β”‚ └── section1.yaml # FRA Exiter Data +β”œβ”€β”€ schemas/ # One YAML per record type +β”‚ β”œβ”€β”€ common/ +β”‚ β”‚ β”œβ”€β”€ header.yaml # Shared HEADER schema +β”‚ β”‚ └── trailer.yaml # Shared TRAILER schema +β”‚ β”œβ”€β”€ tanf/ +β”‚ β”‚ β”œβ”€β”€ t1.yaml through t7.yaml +β”‚ β”œβ”€β”€ ssp/ +β”‚ β”‚ β”œβ”€β”€ m1.yaml through m7.yaml +β”‚ β”œβ”€β”€ tribal_tanf/ +β”‚ β”‚ β”œβ”€β”€ t1.yaml through t7.yaml +β”‚ └── fra/ +β”‚ └── te1.yaml +β”œβ”€β”€ validation/ +β”‚ └── validators.yaml # All field and record validators +└── pipeline.yaml # Worker pool sizes, buffer depths, flush thresholds +``` + +### FileSpecs + +Define which schemas belong to a file, how to detect record types, and how to group records: + +- **Format**: positional (TANF/SSP/Tribal) or columnar (FRA) +- **Record type detection**: prefix matching (positional) or fixed schema (FRA) +- **Accumulator**: key-based grouping for case data (sections 1-2), batching for independent records (sections 3-4, FRA) +- **Presort**: enabled for grouped schemas to guarantee in-memory duplicate detection + +### Schemas + +Define the field layout for each record type: + +- Field name, item number, friendly name (for error messages) +- Byte positions (positional) or column indices (columnar) +- Data types (`string`, `int`) +- Transform functions (e.g., `zero_pad_3`) + +### Validators + +Expression-based validation rules defined in `validators.yaml`: + +- **Field validators**: range checks, date validation, format validation, conditional rules +- **Record validators**: cross-field consistency, sum validations +- **Group Validators**: case-level rules (e.g., `t1_has_t2_or_t3`), valid ssn for federally funded individuals +- Rules are written in `expr` syntax β€” a human-readable expression language +- Error messages are co-located with rules for single-source-of-truth + +```yaml +field_validators: + - id: dateYearIsLargerThan + params: [threshold] + expr: "year(value) >= threshold" + message: "Year must be >= {threshold}" + +group_validators: + - id: t1_has_t2_or_t3 + expr: "RecordCounts['T3'] > 0 or RecordCounts['T2'] > 0" + message: "Every T1 record should have at least one corresponding T2 or T3 record with the same RPT_MONTH_YEAR and CASE_NUMBER" +``` + +### Configuration Loading + +All configuration is loaded once at startup into an immutable registry. Schemas and filespecs are compiled and indexed for O(1) lookup by (program, section) or record type. Expression-based validators are compiled to bytecode at load time, failing fast on syntax errors before any files are processed. + +--- + +## Infrastructure Requirements + +### Docker + +The Go parser requires a new container in the docker-compose stack: + +```yaml +# docker-compose.yml addition +go-parser: + build: + context: ./tdpservice/parsers/go-parser + dockerfile: Dockerfile + environment: + - REDIS_URL=redis://redis-server:6379 + - DATABASE_URL=postgres://tdpuser:tdppass@postgres:5432/tdrs + - AWS_S3_DATAFILES_BUCKET_NAME=tdp-datafiles + - AWS_S3_ENDPOINT=http://localstack:4566 # dev only + depends_on: + - redis-server + - postgres + - localstack +``` + +A `Dockerfile` needs to be created for the Go parser: + +```dockerfile +FROM golang:1.25-alpine AS builder +WORKDIR /app +COPY go.mod go.sum ./ +RUN go mod download +COPY . . +RUN CGO_ENABLED=0 go build -o /go-parser . + +FROM alpine:3.20 +RUN apk add --no-cache ca-certificates +COPY --from=builder /go-parser /go-parser +COPY --from=builder /app/config /config +ENTRYPOINT ["/go-parser"] +``` + +### CI/CD Changes + +| Change | Description | +|--------|-------------| +| **Go build step** | Add `go build` and `go test` to the CI pipeline | +| **SQLC validation** | Run `sqlc diff` to ensure generated code matches schema | +| **Expression validation** | Compile all YAML validators at CI time to catch syntax errors | +| **Docker image build** | Build and push the `go-parser` image alongside the existing `tdp` image | +| **Integration tests** | Run Go integration tests against a test database | + +### Resource Requirements + +| Resource | Recommendation | Notes | +|----------|---------------|-------| +| **CPU** | 2-4 vCPU | Parallel pipeline benefits from multiple cores | +| **Memory** | 1-2 GB | In-memory presort + object pools for large files | +| **Disk** | Minimal | No temp files; config is bundled in image | +| **Network** | Same VPC | Must reach Redis, PostgreSQL, and S3 | + +--- + +## Migration Strategy + +### Phase 1: Shadow Mode (Parallel Validation) + +Deploy the Go parser alongside the Python parser and run both on every submitted file. The Python parser remains the source of truth, writing to production tables as it does today. The Go parser processes the same files concurrently but writes all output β€” parsed records and parser errors β€” to **shadow tables** that mirror the production schema. No production data is affected. + +Shadow tables are structurally identical to their production counterparts (e.g., `shadow_search_indexes_tanf_t1`, `shadow_parser_error`) but exist solely for comparison and analysis. They can be created via a Django migration that duplicates the existing table definitions under a `shadow_` prefix. + +```python +# Django dispatches both parsers for every file +def dispatch_parse(data_file_id): + # Python parser writes to production tables (unchanged) + parse.apply_async(args=[data_file_id], queue="python_parser") + # Go parser writes to shadow tables for analysis + parse.apply_async(args=[data_file_id], queue="go_parser") +``` + +```yaml +# Go parser pipeline.yaml β€” shadow mode configuration +writer: + shadow_mode: true + table_prefix: "shadow_" +``` + +With both parsers processing every file, the team can asynchronously compare results at any cadence: + +- **Record-level diffs**: compare parsed field values between production and shadow tables for the same `datafile_id`, flagging discrepancies +- **Error parity**: compare the set of parser errors generated by each parser β€” missing errors, extra errors, or differing error messages +- **Performance metrics**: measure Go parser latency and resource usage under real production workloads without any risk to production data +- **Edge case discovery**: surface files or record types where the two parsers diverge, driving targeted fixes before the Go parser handles production writes + +Shadow tables can be truncated and rebuilt at will since they carry no production significance. Analysis queries can run against them without impacting production database performance. + +**Exit criteria**: the Go parser produces identical parsed records and equivalent parser errors for all file types over a sustained period, confirmed by automated comparison tooling. + +### Phase 2: Canary Routing (Controlled Cutover) + +Route a small, controlled subset of real submissions to the Go parser writing to production tables. Gradually widen the canary until all traffic is handled by Go. + +Because the Cloud.gov environment does not support network-level traffic splitting, the routing decision is made **programmatically in Django** at task dispatch time. A configuration table or environment-backed setting defines which submissions are routed to the Go parser based on attributes such as program type, STT, or section: + +```python +# Example: routing logic in Django task dispatch +GO_PARSER_CANARY = { + "enabled": True, + "mode": "allowlist", # "allowlist", "percentage", or "all" + "allowlist_stts": ["ST01"], # specific STTs routed to Go + "allowlist_programs": [], # specific program types routed to Go + "percentage": 0, # percentage of remaining traffic routed to Go +} + +def dispatch_parse(data_file_id): + data_file = DataFile.objects.get(id=data_file_id) + if should_use_go_parser(data_file, GO_PARSER_CANARY): + parse.apply_async(args=[data_file_id], queue="go_parser") + else: + parse.apply_async(args=[data_file_id], queue="python_parser") +``` + +Canary progression: + +1. Start with a single STT or program type on the allowlist (smallest blast radius) +2. Monitor record counts, error counts, and DataFileSummary outcomes against historical baselines +3. If the Go parser fails (crashes, times out, returns error), the file can easily be reparsed via the Python parser +4. Widen the allowlist to additional STTs and program types as confidence grows +5. Switch mode to `percentage` and ramp from 10% β†’ 25% β†’ 50% β†’ 100% + +**Exit criteria**: 100% of traffic routed to Go parser with zero fallback events over a sustained period (e.g., 30 days). + +### Phase 3: Python Parser Decommission + +Remove the Python parser from the parsing path entirely. + +1. Remove canary routing logic β€” Go parser is the only consumer of `parse` tasks +2. Keep Python Celery worker for non-parse tasks (email, reparse scheduling, admin). Consider moving the Python Celery worker back into the backend VM +3. Remove Python parser code, schema definitions, and validators from the codebase + +### Feature Parity Checklist + +Before widening the canary beyond the initial allowlist, the Go parser must handle: + +- [ ] All TANF sections (1-4) with all record types (T1-T7) +- [ ] All SSP sections (1-4) with all record types (M1-M7) +- [ ] All Tribal TANF sections (1-4) with all record types (T1-T7) +- [ ] FRA section 1 (CSV and XLSX) +- [ ] HEADER and TRAILER validation +- [ ] All field-level validators +- [ ] All record-level validators +- [ ] All group-level validators +- [ ] Duplicate detection +- [ ] DataFileSummary status updates +- [ ] Parser error records with correct line numbers, field names, and error messages +- [ ] Reparse handling (The Reparse Service should handle the queing, deleting, etc.) + +--- + +## Risks, Dependencies, and Open Questions + +### Risks + +| Risk | Impact | Likelihood | Mitigation | +|------|--------|------------|------------| +| **Schema drift** β€” Django model migrations diverge from SQLC schema | Go parser writes fail or corrupt data | Medium | CI step runs `sqlc diff`; integration tests run against migrated DB | +| **Validation parity gaps** β€” Go parser misses edge cases in Python validators | Incorrect acceptance or rejection of records | Medium | Canary routing with narrow allowlist; automatic Python fallback; comprehensive integration test suite | +| **Memory pressure** β€” Large files during presort exhaust container memory | OOM kill, failed parse | Low | Monitor container memory; set memory limits; most files are well under 100MB | +| **gocelery compatibility** β€” `gocelery` library may not support all Celery protocol features | Task consumption failures | Low | Validate against actual Redis task payloads; the library is used only for basic task/result protocol | +| **Expression engine bugs** β€” `expr` library edge cases in validation | Incorrect validation results | Low | Pinned dependency version; comprehensive validator unit tests | + +### Dependencies + +| Dependency | Owner | Status | +|------------|-------|--------| +| PostgreSQL schema access | DevOps / Django | Existing β€” Go parser needs read/write on existing tables | +| Redis broker access | DevOps | Existing β€” same Redis instance | +| S3 bucket access | DevOps | Existing β€” same IAM role or credentials | +| Go runtime in Docker | DevOps | **New** β€” Dockerfile and CI pipeline needed | +| Celery task routing | Backend team | **New** β€” Django settings change to route parse tasks | +| Canary routing config | Backend team | **New** β€” Django routing logic and canary configuration | + +### Open Questions + +| Question | Context | Proposed Answer | +|----------|---------|-----------------| +| What is the rollback strategy if issues arise during transition? | Production safety | Phase 1 is canary based, and configurable. If issues arise revert config to route 100% of traffic to Python parser β€” no deployment needed. | +| Are there compliance or security review requirements for introducing Go? | FedRAMP / ATO considerations | Go is a compiled, memory-safe language with no additional runtime dependencies. Security review should focus on: dependency audit (`go.mod`), container scanning, and network access patterns (same as Python worker). | +| How will monitoring and observability differ? | Operational visibility | The Go parser should emit structured logs (JSON) and expose Prometheus metrics for: files processed, records parsed, errors generated, pipeline stage latencies, and worker pool utilization. These integrate with the existing Grafana/Prometheus stack already in docker-compose. | +| What is the desired timeline for production readiness? | Planning | Depends on team capacity. Suggested: Phase 1 (canary routing) can begin as soon as the Dockerfile, CI pipeline, and canary routing logic are in place. | diff --git a/docs/Technical-Documentation/tech-memos/upgrading-frontend-dependencies/upgrading-frontend-dependencies.md b/docs/Technical-Documentation/tech-memos/upgrading-frontend-dependencies/upgrading-frontend-dependencies.md index d2b88f7bb..5bf9d4c5d 100644 --- a/docs/Technical-Documentation/tech-memos/upgrading-frontend-dependencies/upgrading-frontend-dependencies.md +++ b/docs/Technical-Documentation/tech-memos/upgrading-frontend-dependencies/upgrading-frontend-dependencies.md @@ -30,28 +30,29 @@ This section outlines the process by which frontend dependencies were upgraded, ```bash -npx npm-check-updates -u -npm install +yarn global add npm-check-updates +ncu -u +yarn install ``` This will bump every package version to the latest available. However, since certain dependencies require specific versions of other packages, this step is not complete until dependency conflicts are resolved. You may receive errors such as this when compiling the application: ```bash -npm ERR! code ERESOLVE -npm ERR! ERESOLVE unable to resolve dependency tree -npm ERR! -npm ERR! While resolving: my-website@0.1.0 -npm ERR! Found: react@16.14.0 -npm ERR! node_modules/react -npm ERR! react@"^16.8.0" from the root project -npm ERR! peer react@"^16.8.0" from @material-ui/core@4.11.0 -npm ERR! node_modules/@material-ui/core -npm ERR! @material-ui/core@"^4.11.0" from the root project -npm ERR! -npm ERR! Could not resolve dependency: -npm ERR! peer react@"17.0.1" from react-dom@17.0.1 -npm ERR! node_modules/react-dom -npm ERR! react-dom@"^17.0.1" from the root project +error code ERESOLVE +error ERESOLVE unable to resolve dependency tree +error +error While resolving: my-website@0.1.0 +error Found: react@16.14.0 +error node_modules/react +error react@"^16.8.0" from the root project +error peer react@"^16.8.0" from @material-ui/core@4.11.0 +error node_modules/@material-ui/core +error @material-ui/core@"^4.11.0" from the root project +error +error Could not resolve dependency: +error peer react@"17.0.1" from react-dom@17.0.1 +error node_modules/react-dom +error react-dom@"^17.0.1" from the root project ``` Downgrade/pin any dependency versions that are in conflict, then recompile. @@ -205,10 +206,10 @@ We will replace `file-type` with the browser-compatible `file-type-checker` ([fi 1. Remove the `file-type` line from `package.json`'s `dependencies` section. 2. Run ```bash - npm i file-type-checker --save + yarn add file-type-checker ``` -* It may be required to delete the `node_modules` folder, then run `npm i` again (if you have cache issues) +* It may be required to delete the `node_modules` folder, then run `yarn install` again (if you have cache issues) * Implementation for this library was slightly different than `file-type`. Consult documentation ### 10. Fix tests @@ -241,7 +242,7 @@ ReactDOMTestUtils.act is deprecated in favor of React.act. Import act from react The following issues (mostly deprecations) were identified and not resolved. Follow-up tickets should be created for this work. 1. Security vulnerabilities - * Running `npm audit` results in the following + * Running `yarn audit` results in the following ```bash 37 vulnerabilities (19 moderate, 17 high, 1 critical) ``` @@ -300,4 +301,4 @@ Most of the frontend is affected by this change, but especially ## Use and Test cases to consider 1. Test the file upload feature thoroughly, including allowed and disallowed file types (extensions and signatures), search form behavior, modals, etc. -2. A11y compliance has not yet been evaluated; test with screenreaders and other a11y tools. \ No newline at end of file +2. A11y compliance has not yet been evaluated; test with screenreaders and other a11y tools. diff --git a/scripts/apply-database-config.sh b/scripts/apply-database-config.sh index a5a066b95..f5fe8da87 100644 --- a/scripts/apply-database-config.sh +++ b/scripts/apply-database-config.sh @@ -20,7 +20,7 @@ sudo apt install -y libpq-dev python3-dev python -m venv ./env source ./env/bin/activate -pip install --upgrade pip pipenv +pip install --upgrade pip pipenv==2026.0.3 pipenv install --dev --system --deploy echo "Done." @@ -73,9 +73,14 @@ if [[ $app == "tdp-backend-prod" ]]; then echo "Creating prod roles and users" python manage.py runscript create_grafana_postgres_role --script-args read_only mr_record_counts_by_tableview stt_section_to_type_mapping fi -python manage.py runscript create_grafana_postgres_role --script-args read_only data_files_datafile django_admin_log parsers_datafilesummary parser_error stts_stt stts_region users_user users_user_groups ssp_m1 ssp_m2 ssp_m3 ssp_m4 ssp_m5 ssp_m6 ssp_m7 tanf_t1 tanf_t2 tanf_t3 tanf_t4 tanf_t5 tanf_t6 tanf_t7 tribal_tanf_t1 tribal_tanf_t2 tribal_tanf_t3 tribal_tanf_t4 tribal_tanf_t5 tribal_tanf_t6 tribal_tanf_t7 +python manage.py runscript create_grafana_postgres_role --script-args read_only data_files_datafile django_admin_log parsers_datafilesummary parser_error stts_stt stts_region users_user users_user_groups user_views python manage.py runscript create_grafana_postgres_role --script-args admin_read_only all -python manage.py runscript create_grafana_readonly_postgres_users --script-args ofa_read_only $OFA_READ_ONLY_PASSWORD read_only ofa_admin_read_only $OFA_ADMIN_READ_ONLY_PASSWORD admin_read_only +python manage.py runscript create_grafana_postgres_role --script-args digit_sensitive data_files_datafile django_admin_log parsers_datafilesummary parser_error stts_stt stts_region users_user users_user_groups user_views admin_views +python manage.py runscript create_grafana_readonly_postgres_users --script-args ofa_read_only $OFA_READ_ONLY_PASSWORD read_only ofa_admin_read_only $OFA_ADMIN_READ_ONLY_PASSWORD admin_read_only ofa_digit_sensitive $OFA_DIGIT_SENSITIVE_PASSWORD digit_sensitive +echo "Done." + +echo "Populating history tables" +python manage.py populate_history --auto echo "Done." diff --git a/scripts/deploy-frontend.sh b/scripts/deploy-frontend.sh index 398c96b32..28c153f15 100755 --- a/scripts/deploy-frontend.sh +++ b/scripts/deploy-frontend.sh @@ -30,8 +30,6 @@ update_frontend() echo "REACT_APP_ENABLE_RUM=true" >> .env.production echo "REACT_APP_FARO_ENDPOINT=https://tanfdata.acf.hhs.gov/collect" >> .env.production echo "REACT_APP_VERSION=v3.8.4" >> .env.production - # PIA - echo "REACT_APP_SHOW_PIA=false" >> .env.production #Nginx echo "BACK_END=" >> .env.production elif [ "$CF_SPACE" = "tanf-staging" ]; then @@ -39,8 +37,6 @@ update_frontend() echo "REACT_APP_FRONTEND_URL=https://$CGHOSTNAME_FRONTEND.acf.hhs.gov" >> .env.development echo "REACT_APP_BACKEND_HOST=https://$CGHOSTNAME_FRONTEND.acf.hhs.gov" >> .env.development echo "REACT_APP_CF_SPACE=$CF_SPACE" >> .env.development - # PIA - echo "REACT_APP_SHOW_PIA=false" >> .env.development cf set-env "$CGHOSTNAME_FRONTEND" ALLOWED_ORIGIN "https://$CGHOSTNAME_FRONTEND.acf.hhs.gov" cf set-env "$CGHOSTNAME_FRONTEND" CONNECT_SRC '*.acf.hhs.gov' @@ -56,7 +52,7 @@ update_frontend() cf set-env "$CGHOSTNAME_FRONTEND" BACKEND_HOST "$CGHOSTNAME_BACKEND" - npm run build:$ENVIRONMENT + yarn build:$ENVIRONMENT unlink .env.production mkdir deployment diff --git a/tdrs-backend/Dockerfile b/tdrs-backend/Dockerfile index 960a00e97..713de2243 100644 --- a/tdrs-backend/Dockerfile +++ b/tdrs-backend/Dockerfile @@ -1,5 +1,5 @@ ARG REGISTRY_OWNER=hhs -ARG BASE_TAG=v1.0.0 +ARG BASE_TAG=v1.1.0 FROM ghcr.io/${REGISTRY_OWNER}/tdp-backend-base:${BASE_TAG} diff --git a/tdrs-backend/Dockerfile.base b/tdrs-backend/Dockerfile.base index 821dcde8f..af9924a4c 100644 --- a/tdrs-backend/Dockerfile.base +++ b/tdrs-backend/Dockerfile.base @@ -20,7 +20,7 @@ RUN apt-get update && \ python3-dev \ curl \ ca-certificates && \ - pip install --no-cache-dir --upgrade pip pipenv && \ + pip install --no-cache-dir --upgrade pip pipenv==2026.0.3 && \ # If we can remove the '--dev' we will save 200MB. However, so much depends on it that it is easier to keep it for now. pipenv install --dev --system --deploy && \ rm -rf /var/lib/apt/lists/* diff --git a/tdrs-backend/Pipfile b/tdrs-backend/Pipfile index 2238f56ab..21103c103 100644 --- a/tdrs-backend/Pipfile +++ b/tdrs-backend/Pipfile @@ -72,6 +72,7 @@ opentelemetry-instrumentation-redis = "==0.60b1" opentelemetry-instrumentation-celery = "==0.60b1" opentelemetry-instrumentation-botocore = "==0.60b1" opentelemetry-instrumentation-requests = "==0.60b1" +django-simple-history = "==3.11.0" [requires] python_version = "3.10.8" diff --git a/tdrs-backend/Pipfile.lock b/tdrs-backend/Pipfile.lock index c08dfeeaa..0f7e48652 100644 --- a/tdrs-backend/Pipfile.lock +++ b/tdrs-backend/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "397cf0a72bfbdef2ec7fbf5859ab3531988ce18c650bbae57d592022b0cd5642" + "sha256": "1e93c51dbd9b0d8dd64352b16c476923d5f85692b949c512999d3e36345cc851" }, "pipfile-spec": 6, "requires": { @@ -86,11 +86,11 @@ }, "certifi": { "hashes": [ - "sha256:9943707519e4add1115f44c2bc244f782c0249876bf51b6599fee1ffbedd685c", - "sha256:ac726dd470482006e014ad384921ed6438c457018f4b3d204aea4281258b2120" + "sha256:027692e4402ad994f1c42e52a4997a9763c646b73e4096e4d5d6db8af1d6f0fa", + "sha256:e887ab5cee78ea814d3472169153c2d12cd43b14bd03329a39a9c6e2e80bfba7" ], "markers": "python_version >= '3.7'", - "version": "==2026.1.4" + "version": "==2026.2.25" }, "cffi": { "hashes": [ @@ -538,6 +538,15 @@ "index": "pypi", "version": "==2.4.1" }, + "django-simple-history": { + "hashes": [ + "sha256:2c587479cf2c3071e9aa555d0d11b73676994db4910770958f57659ade2deffe", + "sha256:f3c298db49e418ffce7fb709a5e83108452ea2179ec5c4b9232484c25427192a" + ], + "index": "pypi", + "markers": "python_version >= '3.10'", + "version": "==3.11.0" + }, "django-storages": { "hashes": [ "sha256:11b7b6200e1cb5ffcd9962bd3673a39c7d6a6109e8096f0e03d46fab3d3aabd9", @@ -616,70 +625,70 @@ }, "grpcio": { "hashes": [ - "sha256:008602fb5bfab98ef9146da9009933d13042c00b219ba79f1e179e83cf10c85c", - "sha256:06881eb8fde0550e0d7c1c7251d11b6b07e2dee502d4241538b8dd152168a233", - "sha256:0ef4693037bbafb5f51f74815dd9e5f0e019c14ba477dbfe5b6eabe1b3560cd1", - "sha256:103df608ce317c590fba50fde69cad502f5fba24e746a22917ca5691b300187c", - "sha256:10e24511de26810a723b54517f5abe3cb14dee7c2ae3c121e3e08f5c6104807b", - "sha256:2b0ef0c6cb638e51ac037f851b755a16088b4004e5eb688f729ffa060eb216ca", - "sha256:2c1cfbf2a172e9249d236ee004d1c51906b4fc8cb43d6dac6bff6eb3a0fbb9e8", - "sha256:2f4b15f132f6b14487c0410066489f775f559db3baef64cc8b0d4a9f1dd166ec", - "sha256:2fa60476242efbfbfefb620371f9e935e033e748b359a7959c16bb3f78333c7e", - "sha256:335e902286649cba6f3937cb39343c99959e5acc31e893ab5e9f700d0d8defdd", - "sha256:3bbf866c7be1095167c62470e1fdc317059b42db97aff1ff71d9237eef0f239e", - "sha256:408a4302e220a39dccfadb41b7b65977518f8953c1ca3ad524ff4ac5de867339", - "sha256:43ce95f92e6f89c5c6cafa7a299e47b35477cf027337b1a1d1e0c71c111b6761", - "sha256:455b16d30abd5f6e364120b297b2b4cb396f93463450d93930d5a5e049194d92", - "sha256:47b5bf9d431b95f627c20f66d573b571f10513022465588c1c0a3913a540f4b7", - "sha256:4e99bbce4f509eb6af4b523109152258043b92bbb945950456a9318eca71ef2e", - "sha256:4fb8b0df1c14dee78f076467c4f581539087f022725a66cbc3970ec91658ea49", - "sha256:54dd38f8edb3d1b693ff29f7ae1c45bb6999f836d0e4c12211bff057529683ac", - "sha256:58bd130450275a87b55d60c7a9185762d6d83a3b1a4401d976e9c6083c7280e0", - "sha256:5ccf4496425b5f5a7a9b801d79fe5e8bfbdf2408b2ab976f291f3e1536d4a3f7", - "sha256:5d39633a0ead39c07d45e00ffb8b262426c5e88ab97bdbc82b2e89ebb6d536cd", - "sha256:5d87f71ce840eb4c13a43d74e59f2d2f83163a71b2c59430598ed983df715a02", - "sha256:6266ce303159899e7f0d545dd9c8edf978f28b79babd3e6aeedec66bb845fb8f", - "sha256:63e69c529121ae6c62a566bde31828dbdd85edf6438610170506dd8b5da6366d", - "sha256:6ba646159dfbd00074e6679103b069d4ef5dc66098cad557e8550feded049b4a", - "sha256:797ec8d482ad7580c29f7dbcc54eebd44d0c1d074c606603aef7eedad3eb61c5", - "sha256:7a3ef091f2082b4ae17463874a6531c01b42e963f164df8ef0c6304f35d9be47", - "sha256:7ab0a68f513620fa34e2dd5428429e0757aac7b3daa9861e5a5a761851ad5767", - "sha256:7fe343a2ccaa3ca48a933e81f4c0a9de37057cf5bc5567864a98775cce570456", - "sha256:86ae01b963762badb8474f0cbf3701cfebaf0cc2cfc860eddd954e974050360b", - "sha256:9944a4cad60e1bf076b025e62157de91aec13216614994038930505a718bae3f", - "sha256:99df1ab7048c18ad08bd8d0e4f81bf69dd1e47b108555aa7afc8befba3f8e62e", - "sha256:a5e2b4a355c92a2263e0e5c8bab97a5516f5e283586e7f83bd9c7579ec2dc9e0", - "sha256:af0b2125bcc19f8ff4274186b48ef9c09d37112e157d2afca4c4dc9ee08eff67", - "sha256:b0c9439c4fc05567ad5e3ae44afb449c0676d8eef5df3ff074e70399883ac7be", - "sha256:b47f176881d6b848f25bdc5b2bcb2c54aa478069ca8339c408015f17f1538f60", - "sha256:b77ee0d0c7abf861fa0b8be9b19a859318dddbf9e6c17437fea781d5205a011b", - "sha256:bf2cf9c2d3919ad9545539c7609e2a7cad48ffddb0b87d58730fec24704057cb", - "sha256:bf84e8dd589bfba77387b4e2ef69380135d3d73d2d322e78ec0f49d388b3ec70", - "sha256:bfb22fefd5cb4a6ac2687d8b314d43f8d3312ed619913270b28524cc4cbbe1dd", - "sha256:c4c3ee365435e6df07ad399ffaabd07f5a0ee8c85a95419a2b658bb0c29e02f0", - "sha256:c670d8f52f4bbddff85cddcf1327255143ded8f6eaf2bdfe25fa58243791f899", - "sha256:c76eab67c341623d52064cf4ef1259184abfba6db85883e481256e40cbbe6b1a", - "sha256:caf8808325a5fdd30cf40d85f3efac965d14b95f92dd8e7d1a7c14ca5e24e67b", - "sha256:ccb40af8fe22e50e8ae9bef0c53840d2297af6c70b2c331b0f61d847082854d8", - "sha256:cf393affd32de39266e2b85b613b5a8420057e55b115774d9adb6546477a8b76", - "sha256:d085ac0245c778bbb32306b3ae477dbe0fc6b58b226d0e54ec934522e336f71e", - "sha256:d0c073d1a6b5de0f550766873e4393f3a0f3b6e1bbb10300735fef4046cbda24", - "sha256:d624592c82a19a5898c5576fbda43c28d7062bac04ea6f33bbd8871bc0639e64", - "sha256:e3364a06f3395853ee926e7cff6c5c2dd1d444f10e071aa4cdbed9db3c59192c", - "sha256:e7bde54ad7bee2d4dbc6d4a351d5b62cc2bfa87c58e9db911ed8a0489192ca9a", - "sha256:ea66e360e5ea032a1f6dde926915ba683edca811ed6f0e0620c52e264ba364e4", - "sha256:f05f444577a7b6b1a8dd83fce602d41e89c00c3524687f947e1f424a547e497c", - "sha256:f16bf178a72dadc61cba3a8b905352076751ee4f5e2cad502091798ce210b4d8", - "sha256:f1999217dd1586236940e3c008f07b76ea25abde4990e2bafb7dbb16da5bbf33", - "sha256:f3890e28a9b9544b03a0c6493b366806ef04296b253fc0d4b4bfee97ecf04f1d", - "sha256:f683813c36e738a5c48661d6945845ee256ea064257161a5ef67d8bd1a23f3e1", - "sha256:f701f14da42d59ee06c993a4e24cc8594fbfdfb373c277dd4580d1de26c98887", - "sha256:f76cf0474eee6cf47cb5ef392e7f15a37941b26100a51adba8d99b3027ac218a", - "sha256:f84ab791751ad5936e0f7f9dce8b29e8ac3efc25a81c8c3780b238726a7face2", - "sha256:fcc6a08f37151bb66785161dc8817b247b85924ef90580d9ee08c72c54598125" - ], - "markers": "python_version >= '3.9'", - "version": "==1.78.0rc2" + "sha256:02b82dcd2fa580f5e82b4cf62ecde1b3c7cc9ba27b946421200706a6e5acaf85", + "sha256:07eb016ea7444a22bef465cce045512756956433f54450aeaa0b443b8563b9ca", + "sha256:09fbd4bcaadb6d8604ed1504b0bdf7ac18e48467e83a9d930a70a7fefa27e862", + "sha256:0fa9943d4c7f4a14a9a876153a4e8ee2bb20a410b65c09f31510b2a42271f41b", + "sha256:13937b28986f45fee342806b07c6344db785ad74a549ebcb00c659142973556f", + "sha256:15f6e636d1152667ddb4022b37534c161c8477274edb26a0b65b215dd0a81e97", + "sha256:1a56bf3ee99af5cf32d469de91bf5de79bdac2e18082b495fc1063ea33f4f2d0", + "sha256:263307118791bc350f4642749a9c8c2d13fec496228ab11070973e568c256bfd", + "sha256:27b5cb669603efb7883a882275db88b6b5d6b6c9f0267d5846ba8699b7ace338", + "sha256:27c625532d33ace45d57e775edf1982e183ff8641c72e4e91ef7ba667a149d72", + "sha256:2b7ad2981550ce999e25ce3f10c8863f718a352a2fd655068d29ea3fd37b4907", + "sha256:2c473b54ef1618f4fb85e82ff4994de18143b74efc088b91b5a935a3a45042ba", + "sha256:34b6cb16f4b67eeb5206250dc5b4d5e8e3db939535e58efc330e4c61341554bd", + "sha256:36aeff5ba8aaf70ceb2cbf6cbba9ad6beef715ad744841f3e0cd977ec02e5966", + "sha256:389b77484959bdaad6a2b7dda44d7d1228381dd669a03f5660392aa0e9385b22", + "sha256:39d21fd30d38a5afb93f0e2e71e2ec2bd894605fb75d41d5a40060c2f98f8d11", + "sha256:39da1680d260c0c619c3b5fa2dc47480ca24d5704c7a548098bca7de7f5dd17f", + "sha256:3a8aa79bc6e004394c0abefd4b034c14affda7b66480085d87f5fbadf43b593b", + "sha256:409bfe22220889b9906739910a0ee4c197a967c21b8dd14b4b06dd477f8819ce", + "sha256:41e4605c923e0e9a84a2718e4948a53a530172bfaf1a6d1ded16ef9c5849fca2", + "sha256:4393bef64cf26dc07cd6f18eaa5170ae4eebaafd4418e7e3a59ca9526a6fa30b", + "sha256:43b930cf4f9c4a2262bb3e5d5bc40df426a72538b4f98e46f158b7eb112d2d70", + "sha256:4b8d7fda614cf2af0f73bbb042f3b7fee2ecd4aea69ec98dbd903590a1083529", + "sha256:4d50329b081c223d444751076bb5b389d4f06c2b32d51b31a1e98172e6cecfb9", + "sha256:5380268ab8513445740f1f77bd966d13043d07e2793487e61fd5b5d0935071eb", + "sha256:5572c5dd1e43dbb452b466be9794f77e3502bdb6aa6a1a7feca72c98c5085ca7", + "sha256:559f58b6823e1abc38f82e157800aff649146f8906f7998c356cd48ae274d512", + "sha256:5ce1855e8cfc217cdf6bcfe0cf046d7cf81ddcc3e6894d6cfd075f87a2d8f460", + "sha256:656a5bd142caeb8b1efe1fe0b4434ecc7781f44c97cfc7927f6608627cf178c0", + "sha256:716a544969660ed609164aff27b2effd3ff84e54ac81aa4ce77b1607ca917d22", + "sha256:75fa92c47d048d696f12b81a775316fca68385ffc6e6cb1ed1d76c8562579f74", + "sha256:7e836778c13ff70edada16567e8da0c431e8818eaae85b80d11c1ba5782eccbb", + "sha256:849cc62eb989bc3be5629d4f3acef79be0d0ff15622201ed251a86d17fef6494", + "sha256:86edb3966778fa05bfdb333688fde5dc9079f9e2a9aa6a5c42e9564b7656ba04", + "sha256:888ceb7821acd925b1c90f0cdceaed1386e69cfe25e496e0771f6c35a156132f", + "sha256:8942bdfc143b467c264b048862090c4ba9a0223c52ae28c9ae97754361372e42", + "sha256:8991c2add0d8505178ff6c3ae54bd9386279e712be82fa3733c54067aae9eda1", + "sha256:8e1fcb419da5811deb47b7749b8049f7c62b993ba17822e3c7231e3e0ba65b79", + "sha256:8f27683ca68359bd3f0eb4925824d71e538f84338b3ae337ead2ae43977d7541", + "sha256:917047c19cd120b40aab9a4b8a22e9ce3562f4a1343c0d62b3cd2d5199da3d67", + "sha256:99550e344482e3c21950c034f74668fccf8a546d50c1ecb4f717543bbdc071ba", + "sha256:9a00992d6fafe19d648b9ccb4952200c50d8e36d0cce8cf026c56ed3fdc28465", + "sha256:9dee66d142f4a8cca36b5b98a38f006419138c3c89e72071747f8fca415a6d8f", + "sha256:a40515b69ac50792f9b8ead260f194ba2bb3285375b6c40c7ff938f14c3df17d", + "sha256:a6afd191551fd72e632367dfb083e33cd185bf9ead565f2476bba8ab864ae496", + "sha256:b071dccac245c32cd6b1dd96b722283b855881ca0bf1c685cf843185f5d5d51e", + "sha256:b2acd83186305c0802dbc4d81ed0ec2f3e8658d7fde97cfba2f78d7372f05b89", + "sha256:b5d5881d72a09b8336a8f874784a8eeffacde44a7bc1a148bce5a0243a265ef0", + "sha256:ca6aebae928383e971d5eace4f1a217fd7aadaf18d5ddd3163d80354105e9068", + "sha256:cd26048d066b51f39fe9206e2bcc2cea869a5e5b2d13c8d523f4179193047ebd", + "sha256:d101fe49b1e0fb4a7aa36ed0c3821a0f67a5956ef572745452d2cd790d723a3f", + "sha256:d6fb962947e4fe321eeef3be1ba5ba49d32dea9233c825fcbade8e858c14aaf4", + "sha256:db681513a1bdd879c0b24a5a6a70398da5eaaba0e077a306410dc6008426847a", + "sha256:e2a6b33d1050dce2c6f563c5caf7f7cbeebf7fba8cde37ffe3803d50526900d1", + "sha256:e49e720cd6b092504ec7bb2f60eb459aaaf4ce0e5fe20521c201b179e93b5d5d", + "sha256:e840405a3f1249509892be2399f668c59b9d492068a2cf326d661a8c79e5e747", + "sha256:ebeec1383aed86530a5f39646984e92d6596c050629982ac54eeb4e2f6ead668", + "sha256:f81816faa426da461e9a597a178832a351d6f1078102590a4b32c77d251b71eb", + "sha256:f8759a1347f3b4f03d9a9d4ce8f9f31ad5e5d0144ba06ccfb1ffaeb0ba4c1e20", + "sha256:ff7de398bb3528d44d17e6913a7cfe639e3b15c65595a71155322df16978c5e1", + "sha256:ffbb760df1cd49e0989f9826b2fd48930700db6846ac171eaff404f3cfbe5c28" + ], + "markers": "python_version >= '3.9'", + "version": "==1.78.1" }, "gunicorn": { "hashes": [ @@ -1032,100 +1041,100 @@ }, "pillow": { "hashes": [ - "sha256:00162e9ca6d22b7c3ee8e61faa3c3253cd19b6a37f126cad04f2f88b306f557d", - "sha256:079af2fb0c599c2ec144ba2c02766d1b55498e373b3ac64687e43849fbbef5bc", - "sha256:0b022eaaf709541b391ee069f0022ee5b36c709df71986e3f7be312e46f42c84", - "sha256:0c27407a2d1b96774cbc4a7594129cc027339fd800cd081e44497722ea1179de", - "sha256:0ddedfaa8b5f0b4ffbc2fa87b556dc59f6bb4ecb14a53b33f9189713ae8053c0", - "sha256:0deedf2ea233722476b3a81e8cdfbad786f7adbed5d848469fa59fe52396e4ef", - "sha256:0ed07dca4a8464bada6139ab38f5382f83e5f111698caf3191cb8dbf27d908b4", - "sha256:0fde7ec5538ab5095cc02df38ee99b0443ff0e1c847a045554cf5f9af1f4aa82", - "sha256:15c794d74303828eaa957ff8070846d0efe8c630901a1c753fdc63850e19ecd9", - "sha256:1a949604f73eb07a8adab38c4fe50791f9919344398bdc8ac6b307f755fc7030", - "sha256:1f345e7bc9d7f368887c712aa5054558bad44d2a301ddf9248599f4161abc7c0", - "sha256:1fcc52d86ce7a34fd17cb04e87cfdb164648a3662a6f20565910a99653d66c18", - "sha256:21e686a21078b0f9cb8c8a961d99e6a4ddb88e0fc5ea6e130172ddddc2e5221a", - "sha256:2415373395a831f53933c23ce051021e79c8cd7979822d8cc478547a3f4da8ef", - "sha256:277518bf4fe74aa91489e1b20577473b19ee70fb97c374aa50830b279f25841b", - "sha256:27b9baecb428899db6c0de572d6d305cfaf38ca1596b5c0542a5182e3e74e8c6", - "sha256:29a4cef9cb672363926f0470afc516dbf7305a14d8c54f7abbb5c199cd8f8179", - "sha256:3413c2ae377550f5487991d444428f1a8ae92784aac79caa8b1e3b89b175f77e", - "sha256:351889afef0f485b84078ea40fe33727a0492b9af3904661b0abbafee0355b72", - "sha256:3ffaa2f0659e2f740473bcf03c702c39a8d4b2b7ffc629052028764324842c64", - "sha256:40a8e3b9e8773876d6e30daed22f016509e3987bab61b3b7fe309d7019a87451", - "sha256:414b9a78e14ffeb98128863314e62c3f24b8a86081066625700b7985b3f529bd", - "sha256:43aca0a55ce1eefc0aefa6253661cb54571857b1a7b2964bd8a1e3ef4b729924", - "sha256:43b4899cfd091a9693a1278c4982f3e50f7fb7cff5153b05174b4afc9593b616", - "sha256:461f9dfdafa394c59cd6d818bdfdbab4028b83b02caadaff0ffd433faf4c9a7a", - "sha256:4f9f6a650743f0ddee5593ac9e954ba1bdbc5e150bc066586d4f26127853ab94", - "sha256:53d8b764726d3af1a138dd353116f774e3862ec7e3794e0c8781e30db0f35dfc", - "sha256:565c986f4b45c020f5421a4cea13ef294dde9509a8577f29b2fc5edc7587fff8", - "sha256:5c5ae0a06e9ea030ab786b0251b32c7e4ce10e58d983c0d5c56029455180b5b9", - "sha256:5cb7bc1966d031aec37ddb9dcf15c2da5b2e9f7cc3ca7c54473a20a927e1eb91", - "sha256:5da841d81b1a05ef940a8567da92decaa15bc4d7dedb540a8c219ad83d91808a", - "sha256:5fee4c04aad8932da9f8f710af2c1a15a83582cfb884152a9caa79d4efcdbf9c", - "sha256:609e89d9f90b581c8d16358c9087df76024cf058fa693dd3e1e1620823f39670", - "sha256:6258f3260986990ba2fa8a874f8b6e808cf5abb51a94015ca3dc3c68aa4f30ea", - "sha256:64efdf00c09e31efd754448a383ea241f55a994fd079866b92d2bbff598aad91", - "sha256:65b80c1ee7e14a87d6a068dd3b0aea268ffcabfe0498d38661b00c5b4b22e74c", - "sha256:6741e6f3074a35e47c77b23a4e4f2d90db3ed905cb1c5e6e0d49bff2045632bc", - "sha256:681088909d7e8fa9e31b9799aaa59ba5234c58e5e4f1951b4c4d1082a2e980e0", - "sha256:6b7a9d1db5dad90e2991645874f708e87d9a3c370c243c2d7684d28f7e133e6b", - "sha256:7315f9137087c4e0ee73a761b163fc9aa3b19f5f606a7fc08d83fd3e4379af65", - "sha256:742aea052cf5ab5034a53c3846165bc3ce88d7c38e954120db0ab867ca242661", - "sha256:75af0b4c229ac519b155028fa1be632d812a519abba9b46b20e50c6caa184f19", - "sha256:7b5dd7cbae20285cdb597b10eb5a2c13aa9de6cde9bb64a3c1317427b1db1ae1", - "sha256:7d6daa89a00b58c37cb1747ec9fb7ac3bc5ffd5949f5888657dfddde6d1312e0", - "sha256:800429ac32c9b72909c671aaf17ecd13110f823ddb7db4dfef412a5587c2c24e", - "sha256:806f3987ffe10e867bab0ddad45df1148a2b98221798457fa097ad85d6e8bc75", - "sha256:808b99604f7873c800c4840f55ff389936ef1948e4e87645eaf3fccbc8477ac4", - "sha256:80941e6d573197a0c28f394753de529bb436b1ca990ed6e765cf42426abc39f8", - "sha256:84cabc7095dd535ca934d57e9ce2a72ffd216e435a84acb06b2277b1de2689bd", - "sha256:8637e29d13f478bc4f153d8daa9ffb16455f0a6cb287da1b432fdad2bfbd66c7", - "sha256:896866d2d436563fa2a43a9d72f417874f16b5545955c54a64941e87c1376c61", - "sha256:8e178e3e99d3c0ea8fc64b88447f7cac8ccf058af422a6cedc690d0eadd98c51", - "sha256:907bfa8a9cb790748a9aa4513e37c88c59660da3bcfffbd24a7d9e6abf224551", - "sha256:9212d6b86917a2300669511ed094a9406888362e085f2431a7da985a6b124f45", - "sha256:92a7fe4225365c5e3a8e598982269c6d6698d3e783b3b1ae979e7819f9cd55c1", - "sha256:935b9d1aed48fcfb3f838caac506f38e29621b44ccc4f8a64d575cb1b2a88644", - "sha256:97e9993d5ed946aba26baf9c1e8cf18adbab584b99f452ee72f7ee8acb882796", - "sha256:983976c2ab753166dc66d36af6e8ec15bb511e4a25856e2227e5f7e00a160587", - "sha256:9f5fefaca968e700ad1a4a9de98bf0869a94e397fe3524c4c9450c1445252304", - "sha256:a332ac4ccb84b6dde65dbace8431f3af08874bf9770719d32a635c4ef411b18b", - "sha256:a40905599d8079e09f25027423aed94f2823adaf2868940de991e53a449e14a8", - "sha256:a6dfc2af5b082b635af6e08e0d1f9f1c4e04d17d4e2ca0ef96131e85eda6eb17", - "sha256:a786bf667724d84aa29b5db1c61b7bfdde380202aaca12c3461afd6b71743171", - "sha256:a83e0850cb8f5ac975291ebfc4170ba481f41a28065277f7f735c202cd8e0af3", - "sha256:aa0c9cc0b82b14766a99fbe6084409972266e82f459821cd26997a488a7261a7", - "sha256:b17fbdbe01c196e7e159aacb889e091f28e61020a8abeac07b68079b6e626988", - "sha256:b63e13dd27da389ed9475b3d28510f0f954bca0041e8e551b2a4eb1eab56a39a", - "sha256:b6e53e82ec2db0717eabb276aa56cf4e500c9a7cec2c2e189b55c24f65a3e8c0", - "sha256:bb0984b30e973f7e2884362b7d23d0a348c7143ee559f38ef3eaab640144204c", - "sha256:bc11908616c8a283cf7d664f77411a5ed2a02009b0097ff8abbba5e79128ccf2", - "sha256:bdec5e43377761c5dbca620efb69a77f6855c5a379e32ac5b158f54c84212b14", - "sha256:bef9768cab184e7ae6e559c032e95ba8d07b3023c289f79a2bd36e8bf85605a5", - "sha256:c990547452ee2800d8506c4150280757f88532f3de2a58e3022e9b179107862a", - "sha256:ca94b6aac0d7af2a10ba08c0f888b3d5114439b6b3ef39968378723622fed377", - "sha256:cad302dc10fac357d3467a74a9561c90609768a6f73a1923b0fd851b6486f8b0", - "sha256:d0a7735df32ccbcc98b98a1ac785cc4b19b580be1bdf0aeb5c03223220ea09d5", - "sha256:d70347c8a5b7ccd803ec0c85c8709f036e6348f1e6a5bf048ecd9c64d3550b8b", - "sha256:d70534cea9e7966169ad29a903b99fc507e932069a881d0965a1a84bb57f6c6d", - "sha256:db44d5c160a90df2d24a24760bbd37607d53da0b34fb546c4c232af7192298ac", - "sha256:e115c15e3bc727b1ca3e641a909f77f8ca72a64fff150f666fcc85e57701c26c", - "sha256:e2479c7f02f9d505682dc47df8c0ea1fc5e264c4d1629a5d63fe3e2334b89554", - "sha256:e5dcbe95016e88437ecf33544ba5db21ef1b8dd6e1b434a2cb2a3d605299e643", - "sha256:e6bdb408f7c9dd2a5ff2b14a3b0bb6d4deb29fb9961e6eb3ae2031ae9a5cec13", - "sha256:e75d3dba8fc1ddfec0cd752108f93b83b4f8d6ab40e524a95d35f016b9683b09", - "sha256:efdc140e7b63b8f739d09a99033aa430accce485ff78e6d311973a67b6bf3208", - "sha256:f10c98f49227ed8383d28174ee95155a675c4ed7f85e2e573b04414f7e371bda", - "sha256:f188028b5af6b8fb2e9a76ac0f841a575bd1bd396e46ef0840d9b88a48fdbcea", - "sha256:f188d580bd870cda1e15183790d1cc2fa78f666e76077d103edf048eed9c356e", - "sha256:f45bd71d1fa5e5749587613037b172e0b3b23159d1c00ef2fc920da6f470e6f0", - "sha256:f61333d817698bdcdd0f9d7793e365ac3d2a21c1f1eb02b32ad6aefb8d8ea831", - "sha256:fb125d860738a09d363a88daa0f59c4533529a90e564785e20fe875b200b6dbd" + "sha256:02f84dfad02693676692746df05b89cf25597560db2857363a208e393429f5e9", + "sha256:0330d233c1a0ead844fc097a7d16c0abff4c12e856c0b325f231820fee1f39da", + "sha256:03edcc34d688572014ff223c125a3f77fb08091e4607e7745002fc214070b35f", + "sha256:097690ba1f2efdeb165a20469d59d8bb03c55fb6621eb2041a060ae8ea3e9642", + "sha256:178aa072084bd88ec759052feca8e56cbb14a60b39322b99a049e58090479713", + "sha256:18e5bddd742a44b7e6b1e773ab5db102bd7a94c32555ba656e76d319d19c3850", + "sha256:1a9b0ee305220b392e1124a764ee4265bd063e54a751a6b62eff69992f457fa9", + "sha256:1f1625b72740fdda5d77b4def688eb8fd6490975d06b909fd19f13f391e077e0", + "sha256:1f1be78ce9466a7ee64bfda57bdba0f7cc499d9794d518b854816c41bf0aa4e9", + "sha256:1f90cff8aa76835cba5769f0b3121a22bd4eb9e6884cfe338216e557a9a548b8", + "sha256:21329ec8c96c6e979cd0dfd29406c40c1d52521a90544463057d2aaa937d66a6", + "sha256:2815a87ab27848db0321fb78c7f0b2c8649dee134b7f2b80c6a45c6831d75ccd", + "sha256:2c1fc0f2ca5f96a3c8407e41cca26a16e46b21060fe6d5b099d2cb01412222f5", + "sha256:2e0c664be47252947d870ac0d327fea7e63985a08794758aa8af5b6cb6ec0c9c", + "sha256:339ffdcb7cbeaa08221cd401d517d4b1fe7a9ed5d400e4a8039719238620ca35", + "sha256:344cf1e3dab3be4b1fa08e449323d98a2a3f819ad20f4b22e77a0ede31f0faa1", + "sha256:36341d06738a9f66c8287cf8b876d24b18db9bd8740fa0672c74e259ad408cff", + "sha256:365b10bb9417dd4498c0e3b128018c4a624dc11c7b97d8cc54effe3b096f4c38", + "sha256:3a5cbdcddad0af3da87cb16b60d23648bc3b51967eb07223e9fed77a82b457c4", + "sha256:417423db963cb4be8bac3fc1204fe61610f6abeed1580a7a2cbb2fbda20f12af", + "sha256:42fc1f4677106188ad9a55562bbade416f8b55456f522430fadab3cef7cd4e60", + "sha256:44ce27545b6efcf0fdbdceb31c9a5bdea9333e664cda58a7e674bb74608b3986", + "sha256:472a8d7ded663e6162dafdf20015c486a7009483ca671cece7a9279b512fcb13", + "sha256:47b94983da0c642de92ced1702c5b6c292a84bd3a8e1d1702ff923f183594717", + "sha256:495c302af3aad1ca67420ddd5c7bd480c8867ad173528767d906428057a11f0e", + "sha256:4ceb838d4bd9dab43e06c363cab2eebf63846d6a4aeaea283bbdfd8f1a8ed58b", + "sha256:50480dcd74fa63b8e78235957d302d98d98d82ccbfac4c7e12108ba9ecbdba15", + "sha256:518a48c2aab7ce596d3bf79d0e275661b846e86e4d0e7dec34712c30fe07f02a", + "sha256:559b38da23606e68681337ad74622c4dbba02254fc9cb4488a305dd5975c7eeb", + "sha256:578510d88c6229d735855e1f278aa305270438d36a05031dfaae5067cc8eb04d", + "sha256:597bd9c8419bc7c6af5604e55847789b69123bbe25d65cc6ad3012b4f3c98d8b", + "sha256:5a8eb7ed8d4198bccbd07058416eeec51686b498e784eda166395a23eb99138e", + "sha256:5c0dd1636633e7e6a0afe7bf6a51a14992b7f8e60de5789018ebbdfae55b040a", + "sha256:5cb1785d97b0c3d1d1a16bc1d710c4a0049daefc4935f3a8f31f827f4d3d2e7f", + "sha256:5d1f9575a12bed9e9eedd9a4972834b08c97a352bd17955ccdebfeca5913fa0a", + "sha256:5d8c41325b382c07799a3682c1c258469ea2ff97103c53717b7893862d0c98ce", + "sha256:5dae5f21afb91322f2ff791895ddd8889e5e947ff59f71b46041c8ce6db790bc", + "sha256:600fd103672b925fe62ed08e0d874ea34d692474df6f4bf7ebe148b30f89f39f", + "sha256:6408a7b064595afcab0a49393a413732a35788f2a5092fdc6266952ed67de586", + "sha256:652a2c9ccfb556235b2b501a3a7cf3742148cd22e04b5625c5fe057ea3e3191f", + "sha256:665e1b916b043cef294bc54d47bf02d87e13f769bc4bc5fa225a24b3a6c5aca9", + "sha256:691ab2ac363b8217f7d31b3497108fb1f50faab2f75dfb03284ec2f217e87bf8", + "sha256:6c52f062424c523d6c4db85518774cc3d50f5539dd6eed32b8f6229b26f24d40", + "sha256:6c6db3b84c87d48d0088943bf33440e0c42370b99b1c2a7989216f7b42eede60", + "sha256:7311c0a0dcadb89b36b7025dfd8326ecfa36964e29913074d47382706e516a7c", + "sha256:7aac39bcf8d4770d089588a2e1dd111cbaa42df5a94be3114222057d68336bd0", + "sha256:7b03048319bfc6170e93bd60728a1af51d3dd7704935feb228c4d4faab35d334", + "sha256:7e7976bf1910a8116b523b9f9f58bf410f3e8aa330cd9a2bb2953f9266ab49af", + "sha256:8089c852a56c2966cf18835db62d9b34fef7ba74c726ad943928d494fa7f4735", + "sha256:86172b0831b82ce4f7877f280055892b31179e1576aa00d0df3bb1bbf8c3e524", + "sha256:89b54027a766529136a06cfebeecb3a04900397a3590fd252160b888479517bf", + "sha256:89c7e895002bbe49cdc5426150377cbbc04767d7547ed145473f496dfa40408b", + "sha256:8b7e5304e34942bf62e15184219a7b5ad4ff7f3bb5cca4d984f37df1a0e1aee2", + "sha256:8fd420ef0c52c88b5a035a0886f367748c72147b2b8f384c9d12656678dfdfa9", + "sha256:98edb152429ab62a1818039744d8fbb3ccab98a7c29fc3d5fcef158f3f1f68b7", + "sha256:99c1506ea77c11531d75e3a412832a13a71c7ebc8192ab9e4b2e355555920e3e", + "sha256:9ad8fa5937ab05218e2b6a4cff30295ad35afd2f83ac592e68c0d871bb0fdbc4", + "sha256:9f51079765661884a486727f0729d29054242f74b46186026582b4e4769918e4", + "sha256:a003d7422449f6d1e3a34e3dd4110c22148336918ddbfc6a32581cd54b2e0b2b", + "sha256:a0b1cd6232e2b618adcc54d9882e4e662a089d5768cd188f7c245b4c8c44a397", + "sha256:a285e3eb7a5a45a2ff504e31f4a8d1b12ef62e84e5411c6804a42197c1cf586c", + "sha256:a37691702ed687799de29a518d63d4682d9016932db66d4e90c345831b02fb4e", + "sha256:a550ae29b95c6dc13cf69e2c9dc5747f814c54eeb2e32d683e5e93af56caa029", + "sha256:ab174cd7d29a62dd139c44bf74b698039328f45cb03b4596c43473a46656b2f3", + "sha256:ab323b787d6e18b3d91a72fc99b1a2c28651e4358749842b8f8dfacd28ef2052", + "sha256:adebb5bee0f0af4909c30db0d890c773d1a92ffe83da908e2e9e720f8edf3984", + "sha256:aee2810642b2898bb187ced9b349e95d2a7272930796e022efaf12e99dccd293", + "sha256:af9a332e572978f0218686636610555ae3defd1633597be015ed50289a03c523", + "sha256:b574c51cf7d5d62e9be37ba446224b59a2da26dc4c1bb2ecbe936a4fb1a7cb7f", + "sha256:b66e95d05ba806247aaa1561f080abc7975daf715c30780ff92a20e4ec546e1b", + "sha256:b81b5e3511211631b3f672a595e3221252c90af017e399056d0faabb9538aa80", + "sha256:b957b71c6b2387610f556a7eb0828afbe40b4a98036fc0d2acfa5a44a0c2036f", + "sha256:bb66b7cc26f50977108790e2456b7921e773f23db5630261102233eb355a3b79", + "sha256:c6008de247150668a705a6338156efb92334113421ceecf7438a12c9a12dab23", + "sha256:c7697918b5be27424e9ce568193efd13d925c4481dd364e43f5dff72d33e10f8", + "sha256:cb9bb857b2d057c6dfc72ac5f3b44836924ba15721882ef103cecb40d002d80e", + "sha256:cc7d296b5ea4d29e6570dabeaed58d31c3fea35a633a69679fb03d7664f43fb3", + "sha256:d242e8ac078781f1de88bf823d70c1a9b3c7950a44cdf4b7c012e22ccbcd8e4e", + "sha256:d2912fd8114fc5545aa3a4b5576512f64c55a03f3ebcca4c10194d593d43ea36", + "sha256:d470ab1178551dd17fdba0fef463359c41aaa613cdcd7ff8373f54be629f9f8f", + "sha256:d4ce8e329c93845720cd2014659ca67eac35f6433fd3050393d85f3ecef0dad5", + "sha256:d6e4571eedf43af33d0fc233a382a76e849badbccdf1ac438841308652a08e1f", + "sha256:e65498daf4b583091ccbb2556c7000abf0f3349fcd57ef7adc9a84a394ed29f6", + "sha256:e879bb6cd5c73848ef3b2b48b8af9ff08c5b71ecda8048b7dd22d8a33f60be32", + "sha256:e9e8064fb1cc019296958595f6db671fba95209e3ceb0c4734c9baf97de04b20", + "sha256:f7ed2c6543bad5a7d5530eb9e78c53132f93dfa44a28492db88b41cdab885202", + "sha256:f95c00d5d6700b2b890479664a06e754974848afaae5e21beb4d83c106923fd0", + "sha256:f975aa7ef9684ce7e2c18a3aa8f8e2106ce1e46b94ab713d156b2898811651d3", + "sha256:fbfa2a7c10cc2623f412753cddf391c7f971c52ca40a3f65dc5039b2939e8563", + "sha256:fc354a04072b765eccf2204f588a7a532c9511e8b9c7f900e1b64e3e33487090", + "sha256:fc44ef1f3de4f45b50ccf9136999d71abb99dca7706bc75d222ed350b9fd2289" ], "markers": "python_version >= '3.10'", - "version": "==12.1.0" + "version": "==12.1.1" }, "prometheus-client": { "hashes": [ @@ -1478,19 +1487,19 @@ }, "wcwidth": { "hashes": [ - "sha256:53123b7af053c74e9fe2e92ac810301f6139e64379031f7124574212fb3b4091", - "sha256:d584eff31cd4753e1e5ff6c12e1edfdb324c995713f75d26c29807bb84bf649e" + "sha256:1a3a1e510b553315f8e146c54764f4fb6264ffad731b3d78088cdb1478ffbdad", + "sha256:cdc4e4262d6ef9a1a57e018384cbeb1208d8abbc64176027e2c2455c81313159" ], "markers": "python_version >= '3.8'", - "version": "==0.5.3" + "version": "==0.6.0" }, "werkzeug": { "hashes": [ - "sha256:5111e36e91086ece91f93268bb39b4a35c1e6f1feac762c9c822ded0a4e322dc", - "sha256:6a548b0e88955dd07ccb25539d7d0cc97417ee9e179677d22c7041c8f078ce67" + "sha256:210c6bede5a420a913956b4791a7f4d6843a43b6fcee4dfa08a65e93007d0d25", + "sha256:7ddf3357bb9564e407607f988f683d72038551200c704012bb9a4c523d42f131" ], "markers": "python_version >= '3.9'", - "version": "==3.1.5" + "version": "==3.1.6" }, "wrapt": { "hashes": [ @@ -1662,101 +1671,115 @@ "toml" ], "hashes": [ - "sha256:00d34b29a59d2076e6f318b30a00a69bf63687e30cd882984ed444e753990cc1", - "sha256:00dd3f02de6d5f5c9c3d95e3e036c3c2e2a669f8bf2d3ceb92505c4ce7838f67", - "sha256:01119735c690786b6966a1e9f098da4cd7ca9174c4cfe076d04e653105488395", - "sha256:03a6e5e1e50819d6d7436f5bc40c92ded7e484e400716886ac921e35c133149d", - "sha256:05dd25b21afffe545e808265897c35f32d3e4437663923e0d256d9ab5031fb14", - "sha256:06d316dbb3d9fd44cca05b2dbcfbef22948493d63a1f28e828d43e6cc505fed8", - "sha256:06e49c5897cb12e3f7ecdc111d44e97c4f6d0557b81a7a0204ed70a8b038f86f", - "sha256:0b4f345f7265cdbdb5ec2521ffff15fa49de6d6c39abf89fc7ad68aa9e3a55f0", - "sha256:0c2be202a83dde768937a61cdc5d06bf9fb204048ca199d93479488e6247656c", - "sha256:0f45e32ef383ce56e0ca099b2e02fcdf7950be4b1b56afaab27b4ad790befe5b", - "sha256:123ceaf2b9d8c614f01110f908a341e05b1b305d6b2ada98763b9a5a59756051", - "sha256:16d23d6579cf80a474ad160ca14d8b319abaa6db62759d6eef53b2fc979b58c8", - "sha256:2213a8d88ed35459bda71597599d4eec7c2ebad201c88f0bfc2c26fd9b0dd2ea", - "sha256:24db3959de8ee394eeeca89ccb8ba25305c2da9a668dd44173394cbd5aa0777f", - "sha256:284e06eadfe15ddfee2f4ee56631f164ef897a7d7d5a15bca5f0bb88889fc5ba", - "sha256:299d66e9218193f9dc6e4880629ed7c4cd23486005166247c283fb98531656c3", - "sha256:2d098709621d0819039f3f1e471ee554f55a0b2ac0d816883c765b14129b5627", - "sha256:2f5e731627a3d5ef11a2a35aa0c6f7c435867c7ccbc391268eb4f2ca5dbdcc10", - "sha256:303d38b19626c1981e1bb067a9928236d88eb0e4479b18a74812f05a82071508", - "sha256:318002f1fd819bdc1651c619268aa5bc853c35fa5cc6d1e8c96bd9cd6c828b75", - "sha256:318b2e4753cbf611061e01b6cc81477e1cdfeb69c36c4a14e6595e674caadb56", - "sha256:31b6e889c53d4e6687ca63706148049494aace140cffece1c4dc6acadb70a7b3", - "sha256:343aaeb5f8bb7bcd38620fd7bc56e6ee8207847d8c6103a1e7b72322d381ba4a", - "sha256:3d1aed4f4e837a832df2f3b4f68a690eede0de4560a2dbc214ea0bc55aabcdb4", - "sha256:3f379b02c18a64de78c4ccdddf1c81c2c5ae1956c72dacb9133d7dd7809794ab", - "sha256:44f14a62f5da2e9aedf9080e01d2cda61df39197d48e323538ec037336d68da8", - "sha256:46d29926349b5c4f1ea4fca95e8c892835515f3600995a383fa9a923b5739ea4", - "sha256:51c4c42c0e7d09a822b08b6cf79b3c4db8333fffde7450da946719ba0d45730f", - "sha256:53be4aab8ddef18beb6188f3a3fdbf4d1af2277d098d4e618be3a8e6c88e74be", - "sha256:562136b0d401992118d9b49fbee5454e16f95f85b120a4226a04d816e33fe024", - "sha256:5907605ee20e126eeee2abe14aae137043c2c8af2fa9b38d2ab3b7a6b8137f73", - "sha256:59224bfb2e9b37c1335ae35d00daa3a5b4e0b1a20f530be208fff1ecfa436f43", - "sha256:5b1ad2e0dc672625c44bc4fe34514602a9fd8b10d52ddc414dc585f74453516c", - "sha256:5badd7e596e6b0c89aa8ec6d37f4473e4357f982ce57f9a2942b0221cd9cf60c", - "sha256:5d67b9ed6f7b5527b209b24b3df9f2e5bf0198c1bbf99c6971b0e2dcb7e2a107", - "sha256:65436cde5ecabe26fb2f0bf598962f0a054d3f23ad529361326ac002c61a2a1e", - "sha256:6ed2e787249b922a93cd95c671cc9f4c9797a106e81b455c83a9ddb9d34590c0", - "sha256:71295f2d1d170b9977dc386d46a7a1b7cbb30e5405492529b4c930113a33f895", - "sha256:75b3c0300f3fa15809bd62d9ca8b170eb21fcf0100eb4b4154d6dc8b3a5bbd43", - "sha256:79f2670c7e772f4917895c3d89aad59e01f3dbe68a4ed2d0373b431fad1dcfba", - "sha256:7a482f2da9086971efb12daca1d6547007ede3674ea06e16d7663414445c683e", - "sha256:7bbb5aa9016c4c29e3432e087aa29ebee3f8fda089cfbfb4e6d64bd292dcd1c2", - "sha256:7df8759ee57b9f3f7b66799b7660c282f4375bef620ade1686d6a7b03699e75f", - "sha256:824bb95cd71604031ae9a48edb91fd6effde669522f960375668ed21b36e3ec4", - "sha256:853c3d3c79ff0db65797aad79dee6be020efd218ac4510f15a205f1e8d13ce25", - "sha256:87ff33b652b3556b05e204ae20793d1f872161b0fa5ec8a9ac76f8430e152ed6", - "sha256:8bb09e83c603f152d855f666d70a71765ca8e67332e5829e62cb9466c176af23", - "sha256:8f1010029a5b52dc427c8e2a8dbddb2303ddd180b806687d1acd1bb1d06649e7", - "sha256:8f2adf4bcffbbec41f366f2e6dffb9d24e8172d16e91da5799c9b7ed6b5716e6", - "sha256:90a8af9dba6429b2573199622d72e0ebf024d6276f16abce394ad4d181bb0910", - "sha256:94d2ac94bd0cc57c5626f52f8c2fffed1444b5ae8c9fc68320306cc2b255e155", - "sha256:96c3be8bae9d0333e403cc1a8eb078a7f928b5650bae94a18fb4820cc993fb9b", - "sha256:989aa158c0eb19d83c76c26f4ba00dbb272485c56e452010a3450bdbc9daafd9", - "sha256:99fee45adbb1caeb914da16f70e557fb7ff6ddc9e4b14de665bd41af631367ef", - "sha256:9db3a3285d91c0b70fab9f39f0a4aa37d375873677efe4e71e58d8321e8c5d39", - "sha256:9f9efbbaf79f935d5fbe3ad814825cbce4f6cdb3054384cb49f0c0f496125fa0", - "sha256:a2f7589c6132c44c53f6e705e1a6677e2b7821378c22f7703b2cf5388d0d4587", - "sha256:a88705500988c8acad8b8fd86c2a933d3aa96bec1ddc4bc5cb256360db7bbd00", - "sha256:ab6d72bffac9deb6e6cb0f61042e748de3f9f8e98afb0375a8e64b0b6e11746b", - "sha256:ae9306b5299e31e31e0d3b908c66bcb6e7e3ddca143dea0266e9ce6c667346d3", - "sha256:b2182129f4c101272ff5f2f18038d7b698db1bf8e7aa9e615cb48440899ad32e", - "sha256:b2beb64c145593a50d90db5c7178f55daeae129123b0d265bdb3cbec83e5194a", - "sha256:b607a40cba795cfac6d130220d25962931ce101f2f478a29822b19755377fb34", - "sha256:be14d0622125edef21b3a4d8cd2d138c4872bf6e38adc90fd92385e3312f406a", - "sha256:bfeee64ad8b4aae3233abb77eb6b52b51b05fa89da9645518671b9939a78732b", - "sha256:c5e9787cec750793a19a28df7edd85ac4e49d3fb91721afcdc3b86f6c08d9aa8", - "sha256:c672d4e2f0575a4ca2bf2aa0c5ced5188220ab806c1bb6d7179f70a11a017222", - "sha256:c6f6169bbdbdb85aab8ac0392d776948907267fcc91deeacf6f9d55f7a83ae3b", - "sha256:ca46e5c3be3b195098dd88711890b8011a9fa4feca942292bb84714ce5eab5d3", - "sha256:cc7fd0f726795420f3678ac82ff882c7fc33770bd0074463b5aef7293285ace9", - "sha256:cd5dee4fd7659d8306ffa79eeaaafd91fa30a302dac3af723b9b469e549247e0", - "sha256:d1a049b5c51b3b679928dd35e47c4a2235e0b6128b479a7596d0ef5b42fa6301", - "sha256:d358dc408edc28730aed5477a69338e444e62fba0b7e9e4a131c505fadad691e", - "sha256:d3a16d6398666510a6886f67f43d9537bfd0e13aca299688a19daa84f543122f", - "sha256:d401f0864a1d3198422816878e4e84ca89ec1c1bf166ecc0ae01380a39b888cd", - "sha256:d6f4a21328ea49d38565b55599e1c02834e76583a6953e5586d65cb1efebd8f8", - "sha256:db83b77f97129813dbd463a67e5335adc6a6a91db652cc085d60c2d512746f96", - "sha256:debf29e0b157769843dff0981cc76f79e0ed04e36bb773c6cac5f6029054bd8a", - "sha256:dfb428e41377e6b9ba1b0a32df6db5409cb089a0ed1d0a672dc4953ec110d84f", - "sha256:e129328ad1258e49cae0123a3b5fcb93d6c2fa90d540f0b4c7cdcdc019aaa3dc", - "sha256:e5b86db331c682fd0e4be7098e6acee5e8a293f824d41487c667a93705d415ca", - "sha256:ed48b4170caa2c4420e0cd27dc977caaffc7eecc317355751df8373dddcef595", - "sha256:edc7754932682d52cf6e7a71806e529ecd5ce660e630e8bd1d37109a2e5f63ba", - "sha256:f45c9bcb16bee25a798ccba8a2f6a1251b19de6a0d617bb365d7d2f386c4e20e", - "sha256:f75695e157c83d374f88dcc646a60cb94173304a9258b2e74ba5a66b7614a51a", - "sha256:f7f153d0184d45f3873b3ad3ad22694fd73aadcb8cdbc4337ab4b41ea6b4dff1", - "sha256:f7f6182d3dfb8802c1747eacbfe611b669455b69b7c037484bb1efbbb56711ac", - "sha256:f9bada7bc660d20b23d7d312ebe29e927b655cf414dadcdb6335a2075695bd86", - "sha256:fae6a21537519c2af00245e834e5bf2884699cc7c1055738fd0f9dc37a3644ad", - "sha256:fb25061a66802df9fc13a9ba1967d25faa4dae0418db469264fd9860a921dde4", - "sha256:fc970575799a9d17d5c3fafd83a0f6ccf5d5117cdc9ad6fbd791e9ead82418b0", - "sha256:fcda51c918c7a13ad93b5f89a58d56e3a072c9e0ba5c231b0ed81404bf2648fb" + "sha256:01d4cbc3c283a17fc1e42d614a119f7f438eabb593391283adca8dc86eff1246", + "sha256:02231499b08dabbe2b96612993e5fc34217cdae907a51b906ac7fca8027a4459", + "sha256:0dd7ab8278f0d58a0128ba2fca25824321f05d059c1441800e934ff2efa52129", + "sha256:0e086334e8537ddd17e5f16a344777c1ab8194986ec533711cbe6c41cde841b6", + "sha256:0fc31c787a84f8cd6027eba44010517020e0d18487064cd3d8968941856d1415", + "sha256:14375934243ee05f56c45393fe2ce81fe5cc503c07cee2bdf1725fb8bef3ffaf", + "sha256:1731dc33dc276dafc410a885cbf5992f1ff171393e48a21453b78727d090de80", + "sha256:19bc3c88078789f8ef36acb014d7241961dbf883fd2533d18cb1e7a5b4e28b11", + "sha256:1af1641e57cf7ba1bd67d677c9abdbcd6cc2ab7da3bca7fa1e2b7e50e65f2ad0", + "sha256:1d4be36a5114c499f9f1f9195e95ebf979460dbe2d88e6816ea202010ba1c34b", + "sha256:200dea7d1e8095cc6e98cdabe3fd1d21ab17d3cee6dab00cadbb2fe35d9c15b9", + "sha256:23e3f687cf945070d1c90f85db66d11e3025665d8dafa831301a0e0038f3db9b", + "sha256:2421d591f8ca05b308cf0092807308b2facbefe54af7c02ac22548b88b95c98f", + "sha256:245e37f664d89861cf2329c9afa2c1fe9e6d4e1a09d872c947e70718aeeac505", + "sha256:25381386e80ae727608e662474db537d4df1ecd42379b5ba33c84633a2b36d47", + "sha256:25a41c3104d08edb094d9db0d905ca54d0cd41c928bb6be3c4c799a54753af55", + "sha256:296f8b0af861d3970c2a4d8c91d48eb4dd4771bcef9baedec6a9b515d7de3def", + "sha256:29e3220258d682b6226a9b0925bc563ed9a1ebcff3cad30f043eceea7eaf2689", + "sha256:2a09cfa6a5862bc2fc6ca7c3def5b2926194a56b8ab78ffcf617d28911123012", + "sha256:2b0f6ccf3dbe577170bebfce1318707d0e8c3650003cb4b3a9dd744575daa8b5", + "sha256:2c048ea43875fbf8b45d476ad79f179809c590ec7b79e2035c662e7afa3192e3", + "sha256:2cb0f1e000ebc419632bbe04366a8990b6e32c4e0b51543a6484ffe15eaeda95", + "sha256:2fa8d5f8de70688a28240de9e139fa16b153cc3cbb01c5f16d88d6505ebdadf9", + "sha256:300deaee342f90696ed186e3a00c71b5b3d27bffe9e827677954f4ee56969601", + "sha256:30b8d0512f2dc8c8747557e8fb459d6176a2c9e5731e2b74d311c03b78451997", + "sha256:33901f604424145c6e9c2398684b92e176c0b12df77d52db81c20abd48c3794c", + "sha256:3599eb3992d814d23b35c536c28df1a882caa950f8f507cef23d1cbf334995ac", + "sha256:391ee8f19bef69210978363ca930f7328081c6a0152f1166c91f0b5fdd2a773c", + "sha256:3998e5a32e62fdf410c0dbd3115df86297995d6e3429af80b8798aad894ca7aa", + "sha256:3c06f0f1337c667b971ca2f975523347e63ec5e500b9aa5882d91931cd3ef750", + "sha256:40aa8808140e55dc022b15d8aa7f651b6b3d68b365ea0398f1441e0b04d859c3", + "sha256:40d74da8e6c4b9ac18b15331c4b5ebc35a17069410cad462ad4f40dcd2d50c0d", + "sha256:4223b4230a376138939a9173f1bdd6521994f2aff8047fae100d6d94d50c5a12", + "sha256:48685fee12c2eb3b27c62f2658e7ea21e9c3239cba5a8a242801a0a3f6a8c62a", + "sha256:4c7d3cc01e7350f2f0f6f7036caaf5673fb56b6998889ccfe9e1c1fe75a9c932", + "sha256:4e83efc079eb39480e6346a15a1bcb3e9b04759c5202d157e1dd4303cd619356", + "sha256:4fc7fa81bbaf5a02801b65346c8b3e657f1d93763e58c0abdf7c992addd81a92", + "sha256:53d133df809c743eb8bce33b24bcababb371f4441340578cd406e084d94a6148", + "sha256:590c0ed4bf8e85f745e6b805b2e1c457b2e33d5255dd9729743165253bc9ad39", + "sha256:5b856a8ccf749480024ff3bd7310adaef57bf31fd17e1bfc404b7940b6986634", + "sha256:65dfcbe305c3dfe658492df2d85259e0d79ead4177f9ae724b6fb245198f55d6", + "sha256:6f01afcff62bf9a08fb32b2c1d6e924236c0383c02c790732b6537269e466a72", + "sha256:6fdef321fdfbb30a197efa02d48fcd9981f0d8ad2ae8903ac318adc653f5df98", + "sha256:71ca20079dd8f27fcf808817e281e90220475cd75115162218d0e27549f95fef", + "sha256:725d985c5ab621268b2edb8e50dfe57633dc69bda071abc470fed55a14935fd3", + "sha256:75eab1ebe4f2f64d9509b984f9314d4aa788540368218b858dad56dc8f3e5eb9", + "sha256:75fcd519f2a5765db3f0e391eb3b7d150cce1a771bf4c9f861aeab86c767a3c0", + "sha256:76451d1978b95ba6507a039090ba076105c87cc76fc3efd5d35d72093964d49a", + "sha256:784fc3cf8be001197b652d51d3fd259b1e2262888693a4636e18879f613a62a9", + "sha256:78cdf0d578b15148b009ccf18c686aa4f719d887e76e6b40c38ffb61d264a552", + "sha256:79be69cf7f3bf9b0deeeb062eab7ac7f36cd4cc4c4dd694bd28921ba4d8596cc", + "sha256:79e73a76b854d9c6088fe5d8b2ebe745f8681c55f7397c3c0a016192d681045f", + "sha256:7b322db1284a2ed3aa28ffd8ebe3db91c929b7a333c0820abec3d838ef5b3525", + "sha256:7d41eead3cc673cbd38a4417deb7fd0b4ca26954ff7dc6078e33f6ff97bed940", + "sha256:7eda778067ad7ffccd23ecffce537dface96212576a07924cbf0d8799d2ded5a", + "sha256:7f57b33491e281e962021de110b451ab8a24182589be17e12a22c79047935e23", + "sha256:8041b6c5bfdc03257666e9881d33b1abc88daccaf73f7b6340fb7946655cd10f", + "sha256:8248977c2e33aecb2ced42fef99f2d319e9904a36e55a8a68b69207fb7e43edc", + "sha256:845f352911777a8e722bfce168958214951e07e47e5d5d9744109fa5fe77f79b", + "sha256:85480adfb35ffc32d40918aad81b89c69c9cc5661a9b8a81476d3e645321a056", + "sha256:8e264226ec98e01a8e1054314af91ee6cde0eacac4f465cc93b03dbe0bce2fd7", + "sha256:8e798c266c378da2bd819b0677df41ab46d78065fb2a399558f3f6cae78b2fbb", + "sha256:9181a3ccead280b828fae232df12b16652702b49d41e99d657f46cc7b1f6ec7a", + "sha256:9351229c8c8407645840edcc277f4a2d44814d1bc34a2128c11c2a031d45a5dd", + "sha256:93550784d9281e374fb5a12bf1324cc8a963fd63b2d2f223503ef0fd4aa339ea", + "sha256:9401ebc7ef522f01d01d45532c68c5ac40fb27113019b6b7d8b208f6e9baa126", + "sha256:94eb63f9b363180aff17de3e7c8760c3ba94664ea2695c52f10111244d16a299", + "sha256:9d107aff57a83222ddbd8d9ee705ede2af2cc926608b57abed8ef96b50b7e8f9", + "sha256:a32ebc02a1805adf637fc8dec324b5cdacd2e493515424f70ee33799573d661b", + "sha256:a3aa4e7b9e416774b21797365b358a6e827ffadaaca81b69ee02946852449f00", + "sha256:a6f94a7d00eb18f1b6d403c91a88fd58cfc92d4b16080dfdb774afc8294469bf", + "sha256:aa3feb8db2e87ff5e6d00d7e1480ae241876286691265657b500886c98f38bda", + "sha256:ad27098a189e5838900ce4c2a99f2fe42a0bf0c2093c17c69b45a71579e8d4a2", + "sha256:ae4578f8528569d3cf303fef2ea569c7f4c4059a38c8667ccef15c6e1f118aa5", + "sha256:b1ec7b6b6e93255f952e27ab58fbc68dcc468844b16ecbee881aeb29b6ab4d8d", + "sha256:b507778ae8a4c915436ed5c2e05b4a6cecfa70f734e19c22a005152a11c7b6a9", + "sha256:b66a2da594b6068b48b2692f043f35d4d3693fb639d5ea8b39533c2ad9ac3ab9", + "sha256:b720ce6a88a2755f7c697c23268ddc47a571b88052e6b155224347389fdf6a3b", + "sha256:b7b38448866e83176e28086674fe7368ab8590e4610fb662b44e345b86d63ffa", + "sha256:b8eb931ee8e6d8243e253e5ed7336deea6904369d2fd8ae6e43f68abbf167092", + "sha256:bb28c0f2cf2782508a40cec377935829d5fcc3ad9a3681375af4e84eb34b6b58", + "sha256:bd60d4fe2f6fa7dff9223ca1bbc9f05d2b6697bc5961072e5d3b952d46e1b1ea", + "sha256:c35eb28c1d085eb7d8c9b3296567a1bebe03ce72962e932431b9a61f28facf26", + "sha256:c4240e7eded42d131a2d2c4dec70374b781b043ddc79a9de4d55ca71f8e98aea", + "sha256:caa421e2684e382c5d8973ac55e4f36bed6821a9bad5c953494de960c74595c9", + "sha256:d490ba50c3f35dd7c17953c68f3270e7ccd1c6642e2d2afe2d8e720b98f5a053", + "sha256:d65b2d373032411e86960604dc4edac91fdfb5dca539461cf2cbe78327d1e64f", + "sha256:dae88bc0fc77edaa65c14be099bd57ee140cf507e6bfdeea7938457ab387efb0", + "sha256:de6defc1c9badbf8b9e67ae90fd00519186d6ab64e5cc5f3d21359c2a9b2c1d3", + "sha256:e101609bcbbfb04605ea1027b10dc3735c094d12d40826a60f897b98b1c30256", + "sha256:e24f9156097ff9dc286f2f913df3a7f63c0e333dcafa3c196f2c18b4175ca09a", + "sha256:e2f25215f1a359ab17320b47bcdaca3e6e6356652e8256f2441e4ef972052903", + "sha256:e5c8f6ed1e61a8b2dcdf31eb0b9bbf0130750ca79c1c49eb898e2ad86f5ccc91", + "sha256:e6f70dec1cc557e52df5306d051ef56003f74d56e9c4dd7ddb07e07ef32a84dd", + "sha256:e856bf6616714c3a9fbc270ab54103f4e685ba236fa98c054e8f87f266c93505", + "sha256:e87f6c587c3f34356c3759f0420693e35e7eb0e2e41e4c011cb6ec6ecbbf1db7", + "sha256:eb30bf180de3f632cd043322dad5751390e5385108b2807368997d1a92a509d0", + "sha256:eb88b316ec33760714a4720feb2816a3a59180fd58c1985012054fa7aebee4c2", + "sha256:eb9078108fbf0bcdde37c3f4779303673c2fa1fe8f7956e68d447d0dd426d38a", + "sha256:ecae9737b72408d6a950f7e525f30aca12d4bd8dd95e37342e5beb3a2a8c4f71", + "sha256:ee756f00726693e5ba94d6df2bdfd64d4852d23b09bb0bc700e3b30e6f333985", + "sha256:f4594c67d8a7c89cf922d9df0438c7c7bb022ad506eddb0fdb2863359ff78242", + "sha256:f53d492307962561ac7de4cd1de3e363589b000ab69617c6156a16ba7237998d", + "sha256:fb07dc5da7e849e2ad31a5d74e9bece81f30ecf5a42909d0a695f8bd1874d6af", + "sha256:fb26a934946a6afe0e326aebe0730cdff393a8bc0bbb65a2f41e30feddca399c", + "sha256:fdfc1e28e7c7cdce44985b3043bc13bbd9c747520f94a4d7164af8260b3d91f0" ], "markers": "python_version >= '3.10'", - "version": "==7.13.3" + "version": "==7.13.4" }, "django": { "hashes": [ @@ -1811,11 +1834,11 @@ }, "faker": { "hashes": [ - "sha256:93503165c165d330260e4379fd6dc07c94da90c611ed3191a0174d2ab9966a42", - "sha256:b76a68163aa5f171d260fc24827a8349bc1db672f6a665359e8d0095e8135d30" + "sha256:70222361cd82aa10cb86066d1a4e8f47f2bcdc919615c412045a69c4e6da0cd3", + "sha256:c69640c1e13bad49b4bcebcbf1b52f9f1a872b6ea186c248ada34d798f1661bf" ], "markers": "python_version >= '3.10'", - "version": "==40.1.2" + "version": "==40.5.1" }, "flake8": { "hashes": [ @@ -2044,11 +2067,11 @@ }, "platformdirs": { "hashes": [ - "sha256:61d5cdcc6065745cdd94f0f878977f8de9437be93de97c1c12f853c9c0cdcbda", - "sha256:d03afa3963c806a9bed9d5125c8f4cb2fdaf74a55ab60e5d59b3fde758104d31" + "sha256:9170634f126f8efdae22fb58ae8a0eaa86f38365bc57897a6c4f781d1f5875bd", + "sha256:9a33809944b9db043ad67ca0db94b14bf452cc6aeaac46a88ea55b26e2e9d291" ], "markers": "python_version >= '3.10'", - "version": "==4.5.1" + "version": "==4.9.2" }, "pluggy": { "hashes": [ diff --git a/tdrs-backend/celery_start.sh b/tdrs-backend/celery_start.sh index 83ecbbcc6..4c351a823 100755 --- a/tdrs-backend/celery_start.sh +++ b/tdrs-backend/celery_start.sh @@ -3,20 +3,46 @@ echo "Starting celery" if [[ $1 == "cloud" ]]; then - # Get the computed URI from VCAP_SERVICES - REDIS_URI=$(echo $VCAP_SERVICES | jq -r '."aws-elasticache-redis"[0].credentials.uri') + # Build the broker URL with TLS scheme and correct DB number (matching cloudgov.py logic) + REDIS_BASE=$(echo $VCAP_SERVICES | jq -r '."aws-elasticache-redis"[0].credentials | "rediss://:\(.password)@\(.host):\(.port)"') + ENV_NAME=$(echo $VCAP_APPLICATION | jq -r '.application_name | split("-") | last') + # Running python method, to not duplicate logic for determining the DB number + BROKER_DB=$(python -c "from tdpservice.common.util import get_cloudgov_broker_db_numbers; print(get_cloudgov_broker_db_numbers('${ENV_NAME}')['celery'])") + REDIS_URI="${REDIS_BASE}/${BROKER_DB}" + + # Log the broker config (redact password) + echo "REDIS_URI: $(echo $REDIS_URI | sed 's/:.*@/:\/\/***@/')" + echo "ENV_NAME: ${ENV_NAME}, BROKER_DB: ${BROKER_DB}" + + if [[ -z "$REDIS_BASE" || "$REDIS_BASE" == "rediss://:@:" ]]; then + echo "ERROR: Failed to extract Redis credentials from VCAP_SERVICES" + exit 1 + fi + if [[ -z "$BROKER_DB" ]]; then + echo "ERROR: Failed to determine broker DB number for env '${ENV_NAME}'" + exit 1 + fi echo "Starting Alloy" mkdir /home/vcap/app/alloy-data wget https://github.com/grafana/alloy/releases/download/v1.9.1/alloy-boringcrypto-linux-amd64.zip - unzip -a alloy-boringcrypto-linux-amd64.zip && rm -rf alloy-boringcrypto-linux-amd64.zip - chmod +x alloy-boringcrypto-linux-amd64 - ./alloy-boringcrypto-linux-amd64 run --server.http.listen-addr=0.0.0.0:12345 --storage.path=/home/vcap/app/alloy-data /home/vcap/app/plg/alloy/alloy.config & + if ! unzip -a alloy-boringcrypto-linux-amd64.zip; then + echo "ERROR: Failed to unzip Alloy" + else + rm -rf alloy-boringcrypto-linux-amd64.zip + chmod +x alloy-boringcrypto-linux-amd64 + ./alloy-boringcrypto-linux-amd64 run --server.http.listen-addr=0.0.0.0:12345 --storage.path=/home/vcap/app/alloy-data /home/vcap/app/plg/alloy/alloy.config & + echo "Alloy started (PID: $!)" + fi echo "Starting the Celery Exporter" - curl -L https://github.com/danihodovic/celery-exporter/releases/download/latest/celery-exporter -o ./celery-exporter - chmod +x ./celery-exporter - ./celery-exporter --broker-url="$REDIS_URI" --port 9808 & + if ! curl -fL https://github.com/danihodovic/celery-exporter/releases/download/latest/celery-exporter -o ./celery-exporter; then + echo "ERROR: Failed to download celery-exporter" + else + chmod +x ./celery-exporter + ./celery-exporter --broker-url="$REDIS_URI" --port 9808 & + echo "Celery Exporter started (PID: $!)" + fi fi # Celery worker config can be found here: https://docs.celeryq.dev/en/stable/userguide/workers.html#:~:text=The-,hostname,-argument%20can%20expand diff --git a/tdrs-backend/docker-compose.yml b/tdrs-backend/docker-compose.yml index 8c55267f5..c89553852 100644 --- a/tdrs-backend/docker-compose.yml +++ b/tdrs-backend/docker-compose.yml @@ -229,6 +229,7 @@ services: ./manage.py makemigrations && ./manage.py migrate && ./manage.py populate_stts && + ./manage.py populate_history --auto && python ./plg/grafana_views/generate_views.py --all && ./manage.py runscript apply_grafana_views && ./gunicorn_start.sh" diff --git a/tdrs-backend/plg/alloy/alloy.config b/tdrs-backend/plg/alloy/alloy.config index d2072d91c..75456094c 100644 --- a/tdrs-backend/plg/alloy/alloy.config +++ b/tdrs-backend/plg/alloy/alloy.config @@ -26,6 +26,12 @@ loki.process "backend_process" { stage.regex { expression = "^(?P