v4.14.0 release to prod

Taskfile.yml

@@ -1,7 +1,6 @@
-version: '3'
+version: "3"
 
 tasks:
-
   gitcfg:
     desc: Configure git
     cmds:
@@ -61,7 +60,6 @@ tasks:
       # 3) Apply all migrations to initialize the new database
       - docker compose -f docker-compose.yml exec web sh -c "python manage.py migrate"
 
-
   backend-restart:
     desc: Restart backend web server
     dir: tdrs-backend
@@ -84,7 +82,7 @@ tasks:
     desc: Execute a manage.py command in the backend container."
     dir: tdrs-backend
     vars:
-      CMD: '{{.CMD}}'
+      CMD: "{{.CMD}}"
     cmds:
       - docker compose -f docker-compose.yml exec web sh -c "python manage.py {{.CMD}}"
 
@@ -112,6 +110,14 @@ tasks:
       - task: backend-up
       - docker compose -f docker-compose.yml exec web sh -c "python manage.py migrate"
 
+  backend-update:
+    desc: Rebuild backend image and apply migrations (no data wipe)
+    dir: tdrs-backend
+    cmds:
+      - task: create-network
+      - docker compose -f docker-compose.yml up -d --build
+      - docker compose -f docker-compose.yml exec web sh -c "python manage.py migrate"
+
   backend-pytest:
     desc: 'Run pytest in the backend container. E.g: task backend-pytest PYTEST_ARGS="tdpservice/test/ -s -vv"'
     dir: tdrs-backend
@@ -230,6 +236,31 @@ tasks:
     cmds:
       - npm install
 
+  frontend-update:
+    desc: Rebuild frontend localdev image and install dependencies (no data wipe)
+    dir: tdrs-frontend
+    cmds:
+      - docker compose -f docker-compose.local.yml up -d --build tdp-frontend
+      - docker compose -f docker-compose.local.yml exec tdp-frontend sh -c "npm install"
+
+  frontend-test-local:
+    desc: Run frontend unit tests on host (watch-less)
+    dir: tdrs-frontend
+    cmds:
+      - CI=true npm run test
+
+  frontend-test-watch:
+    desc: Run frontend unit tests on host (watch)
+    dir: tdrs-frontend
+    cmds:
+      - npm run test
+
+  frontend-test-cov-local:
+    desc: Run frontend unit tests with coverage on host
+    dir: tdrs-frontend
+    cmds:
+      - npm run test:cov
+
   frontend-test:
     desc: 'Run frontend unit tests in docker (watch-less). E.g: task frontend-test JEST_ARGS="--testPathPattern=FeedbackReports"'
     dir: tdrs-frontend
@@ -263,7 +294,7 @@ tasks:
     cmds:
       - export CYPRESS_TOKEN=local-cypress-token
       - docker-compose exec web python manage.py delete_cypress_users -usernames new-cypress@teamraft.com cypress-admin@teamraft.com
-      - docker-compose exec web python manage.py loaddata cypress/users cypress/data_files cypress/regions
+      - docker-compose exec web python manage.py loaddata cypress/users cypress/data_files cypress/regions cypress/profile_editing_regions cypress/profile_editing_users
 
   frontend-e2e-local:
     desc: Run Cypress E2E tests locally (Cypress on host, app in docker)
@@ -276,10 +307,10 @@ tasks:
     desc: Run k6 performance tests
     dir: performance-tests
     vars:
-      SCRIPT: '{{.SCRIPT}}'
+      SCRIPT: "{{.SCRIPT}}"
       CYPRESS_TOKEN: '{{.CYPRESS_TOKEN | default "local-cypress-token"}}'
       BASE_URL: '{{.BASE_URL | default "http://localhost:3000"}}'
-      SCENARIO: '{{.SCENARIO}}'
+      SCENARIO: "{{.SCENARIO}}"
     cmds:
       - k6 run -e BASE_URL={{.BASE_URL}} -e CYPRESS_TOKEN={{.CYPRESS_TOKEN}} -e SCENARIO={{.SCENARIO}} {{.SCRIPT}}
 
@@ -289,6 +320,12 @@ tasks:
       - task: backend-up
       - task: frontend-up
 
+  update-deps:
+    desc: Rebuild backend/frontend and apply dependency updates (no data wipe)
+    cmds:
+      - task: backend-update
+      - task: frontend-update
+
   down:
     desc: Stop both frontend and backend web servers
     cmds:

docs/Technical-Documentation/circle-ci.md

@@ -11,12 +11,19 @@
     * Executors: build environments used for jobs
     * Jobs: a collection of steps run on an executor
     * Orbs: reusable code that can be imported in to circle config - similar to pip packages, etc
-* We currently have 5 workflows:
+* Automated workflows:
     * `build-and-test`: Runs jobs `secrets-check`, `test-frontend` and `test-backend` on every commit
     * `dev-deployment`: Deploys a PR to the dev space. Triggered by a GitHub action whenever one of the relevant deployment labels is assigned via an API call to Circle CI with the pipeline parameter `run_dev_deployment`.
     * `nightly`: Runs every night at UTC midnight and performs an OWASP scan against the staging site for both backend and frontend then stores the results in Django using a Cloud Foundry task.
     * `owasp-scan`: Runs an OWASP scan against the backend and frontend for a given PR. Triggered by a GitHub action whenever the `QASP Review` label is assigned via an API call to Circle CI with the pipeline parameter `run_owasp_scan`.
-    * `staging-deployment`: Deploys the main branch to the staging space in Cloud.gov. Triggered via merges to the branch `develop`.
+    * `staging-deployment`: Deploys the main branch to the staging space in Cloud.gov. Triggered via merges to the branch `main`.
+* Manual deployment
+    * Select the desired branch from the branch dropdown on the CircleCI project page
+    * Click the "Trigger Pipeline" button
+    * Enter the `target_env`: e.g. `qasp`, `raft`, `a11y`
+    * Set `triggered` to true
+    * Set `run_dev_deployment` to true
+    * Click the `Run Pipeline` button
 
 ## How are environment variables supplied in CI?
 We manually set some environment variables in the project settings for Circle CI. From there, they are used in several places:

scripts/apply-database-config.sh

@@ -81,6 +81,7 @@ echo "Done."
 
 if [[ $app == "tdp-backend-develop" || $space == "tanf-dev" ]]; then
     echo "Applying e2e test data"
+    python manage.py populate_stts
     python manage.py loaddata cypress/users cypress/data_files cypress/regions cypress/profile_editing_regions cypress/profile_editing_users
     echo "Done."
 fi

tdrs-backend/celery_start.sh

@@ -20,7 +20,7 @@ if [[ $1 == "cloud" ]]; then
 fi
 
 # Celery worker config can be found here: https://docs.celeryq.dev/en/stable/userguide/workers.html#:~:text=The-,hostname,-argument%20can%20expand
-celery -A tdpservice.settings worker --loglevel=INFO --concurrency=1 --max-tasks-per-child=1 -n worker1@%h & # tune -c ?
+celery -A tdpservice.settings worker --loglevel=INFO --concurrency=1 --max-tasks-per-child=100 -n worker1@%h & # tune -c ?
 sleep 5
 
 # TODO: Uncomment the following line to add flower service when memory limitation is resolved

tdrs-backend/plg/grafana_views/generate_views.py

@@ -99,7 +99,7 @@ def handle_field(field, formatted_fields, is_admin):
             f''' --
         -- Calculate if SSN is valid
         CASE
-            WHEN "{field}" IS NOT NULL AND "{field}" !~ '{regex_str}' AND "{field}" LIKE '%[^0-9]%' THEN 1
+            WHEN "{field}" IS NOT NULL AND "{field}" ~ '^[0-9]{{9}}$' AND "{field}" !~ '{regex_str}' THEN 1
             ELSE 0
         END AS "SSN_VALID"'''.strip()
         )

tdrs-backend/tdpservice/common/test/test_util.py

@@ -0,0 +1,26 @@
+"""Test the common utils."""
+
+import pytest
+
+from tdpservice.common.util import get_cloudgov_broker_db_numbers
+
+
+@pytest.mark.parametrize(
+    "env,expected",
+    [
+        # dev envs
+        ("raft", {"celery": "0", "caches": {"stts": "1", "feature-flags": "2"}}),
+        ("qasp", {"celery": "3", "caches": {"stts": "4", "feature-flags": "5"}}),
+        ("a11y", {"celery": "6", "caches": {"stts": "7", "feature-flags": "8"}}),
+        # staging
+        ("develop", {"celery": "0", "caches": {"stts": "1", "feature-flags": "2"}}),
+        ("staging", {"celery": "3", "caches": {"stts": "4", "feature-flags": "5"}}),
+        # prod
+        ("prod", {"celery": "0", "caches": {"stts": "1", "feature-flags": "2"}}),
+    ],
+)
+@pytest.mark.django_db
+def test_get_cloudgov_broker_db_numbers(env, expected):
+    """Test redis broker db number generation for deployed envs."""
+    result = get_cloudgov_broker_db_numbers(env)
+    assert result == expected

tdrs-backend/tdpservice/common/util.py

@@ -0,0 +1,32 @@
+"""Utilities for the application."""
+
+
+def get_cloudgov_broker_db_numbers(cloudgov_name):
+    """
+    Get the appropriate redis broker db numbers for an environment.
+
+    Returns an object of {"celery": str, "caches": {"cache_name": str}}
+    """
+    spaces = {
+        "dev": ["raft", "qasp", "a11y"],
+        "staging": ["develop", "staging"],
+        "prod": ["prod"],
+    }
+    cache_options = ["stts", "feature-flags"]
+
+    broker_nums = {}
+
+    for space, envs in spaces.items():
+        incr = 0
+
+        for env in envs:
+            celery = str(incr)
+            caches = {}
+            for c in cache_options:
+                incr += 1
+                caches[c] = str(incr)
+
+            broker_nums[env] = {"celery": celery, "caches": caches}
+            incr += 1
+
+    return broker_nums[cloudgov_name]

tdrs-backend/tdpservice/conftest.py

@@ -198,6 +198,18 @@ def stt(region):
     return stt
 
 
+@pytest.fixture
+def tribe_stt(region):
+    """Return a Tribal STT."""
+    stt, _ = STT.objects.get_or_create(
+        name="Blackfeet Nation",
+        region=region,
+        stt_code="020",
+        type=STT.EntityType.TRIBE,
+    )
+    return stt
+
+
 @pytest.fixture
 def region():
     """Return a region."""

tdrs-backend/tdpservice/core/models.py

@@ -2,7 +2,10 @@
 
 from django.contrib.auth.models import Permission
 from django.contrib.contenttypes.models import ContentType
+from django.core.cache import caches
 from django.db import models
+from django.db.models.signals import post_delete, post_migrate, post_save
+from django.dispatch import receiver
 
 
 class FeatureFlag(models.Model):
@@ -11,9 +14,9 @@ class FeatureFlag(models.Model):
     class Meta:
         """Metadata."""
 
-        ordering = ['feature_name']
-        verbose_name = 'Feature Flag'
-        verbose_name_plural = 'Feature Flags'
+        ordering = ["feature_name"]
+        verbose_name = "Feature Flag"
+        verbose_name_plural = "Feature Flags"
 
     feature_name = models.CharField(max_length=100, unique=True, db_index=True)
     enabled = models.BooleanField(default=False)
@@ -28,6 +31,18 @@ def __str__(self) -> str:
         return f"{self.feature_name} ({status})"
 
 
+@receiver([post_delete, post_migrate, post_save], sender=FeatureFlag)
+def clear_feature_flag_cache(sender, instance, **kwargs):
+    """Invalidate the cache after any changes to feature flags.
+
+    This depends on the cache being separated by feature, so the entire cache can be deleted.
+    There are too many options for headers/cookies to determine the key programatically,
+    so we segment the different featuers into separate caches to be able to invalidate efficiently
+    """
+    cache = caches["feature-flags"]
+    cache.clear()
+
+
 """Global permissions
 
 Allows for the creation of permissions that are

tdrs-backend/tdpservice/core/serializers.py

@@ -0,0 +1,20 @@
+"""Serialize core model data."""
+
+from rest_framework import serializers
+
+from tdpservice.core.models import FeatureFlag
+
+
+class FeatureFlagSerializer(serializers.ModelSerializer):
+    """FeatureFlag serializer."""
+
+    class Meta:
+        """Metadata."""
+
+        model = FeatureFlag
+        fields = [
+            "feature_name",
+            "enabled",
+            "config",
+            "description",
+        ]