diff --git a/Taskfile.yml b/Taskfile.yml index fb334a6d2..3be5319ce 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -1,7 +1,6 @@ -version: '3' +version: "3" tasks: - gitcfg: desc: Configure git cmds: @@ -61,7 +60,6 @@ tasks: # 3) Apply all migrations to initialize the new database - docker compose -f docker-compose.yml exec web sh -c "python manage.py migrate" - backend-restart: desc: Restart backend web server dir: tdrs-backend @@ -84,7 +82,7 @@ tasks: desc: Execute a manage.py command in the backend container." dir: tdrs-backend vars: - CMD: '{{.CMD}}' + CMD: "{{.CMD}}" cmds: - docker compose -f docker-compose.yml exec web sh -c "python manage.py {{.CMD}}" @@ -112,6 +110,14 @@ tasks: - task: backend-up - docker compose -f docker-compose.yml exec web sh -c "python manage.py migrate" + backend-update: + desc: Rebuild backend image and apply migrations (no data wipe) + dir: tdrs-backend + cmds: + - task: create-network + - docker compose -f docker-compose.yml up -d --build + - docker compose -f docker-compose.yml exec web sh -c "python manage.py migrate" + backend-pytest: desc: 'Run pytest in the backend container. E.g: task backend-pytest PYTEST_ARGS="tdpservice/test/ -s -vv"' dir: tdrs-backend @@ -230,6 +236,31 @@ tasks: cmds: - npm install + frontend-update: + desc: Rebuild frontend localdev image and install dependencies (no data wipe) + dir: tdrs-frontend + cmds: + - docker compose -f docker-compose.local.yml up -d --build tdp-frontend + - docker compose -f docker-compose.local.yml exec tdp-frontend sh -c "npm install" + + frontend-test-local: + desc: Run frontend unit tests on host (watch-less) + dir: tdrs-frontend + cmds: + - CI=true npm run test + + frontend-test-watch: + desc: Run frontend unit tests on host (watch) + dir: tdrs-frontend + cmds: + - npm run test + + frontend-test-cov-local: + desc: Run frontend unit tests with coverage on host + dir: tdrs-frontend + cmds: + - npm run test:cov + frontend-test: desc: 'Run frontend unit tests in docker (watch-less). E.g: task frontend-test JEST_ARGS="--testPathPattern=FeedbackReports"' dir: tdrs-frontend @@ -263,7 +294,7 @@ tasks: cmds: - export CYPRESS_TOKEN=local-cypress-token - docker-compose exec web python manage.py delete_cypress_users -usernames new-cypress@teamraft.com cypress-admin@teamraft.com - - docker-compose exec web python manage.py loaddata cypress/users cypress/data_files cypress/regions + - docker-compose exec web python manage.py loaddata cypress/users cypress/data_files cypress/regions cypress/profile_editing_regions cypress/profile_editing_users frontend-e2e-local: desc: Run Cypress E2E tests locally (Cypress on host, app in docker) @@ -276,10 +307,10 @@ tasks: desc: Run k6 performance tests dir: performance-tests vars: - SCRIPT: '{{.SCRIPT}}' + SCRIPT: "{{.SCRIPT}}" CYPRESS_TOKEN: '{{.CYPRESS_TOKEN | default "local-cypress-token"}}' BASE_URL: '{{.BASE_URL | default "http://localhost:3000"}}' - SCENARIO: '{{.SCENARIO}}' + SCENARIO: "{{.SCENARIO}}" cmds: - k6 run -e BASE_URL={{.BASE_URL}} -e CYPRESS_TOKEN={{.CYPRESS_TOKEN}} -e SCENARIO={{.SCENARIO}} {{.SCRIPT}} @@ -289,6 +320,12 @@ tasks: - task: backend-up - task: frontend-up + update-deps: + desc: Rebuild backend/frontend and apply dependency updates (no data wipe) + cmds: + - task: backend-update + - task: frontend-update + down: desc: Stop both frontend and backend web servers cmds: diff --git a/docs/Technical-Documentation/circle-ci.md b/docs/Technical-Documentation/circle-ci.md index be8b5edd4..19c6460bf 100644 --- a/docs/Technical-Documentation/circle-ci.md +++ b/docs/Technical-Documentation/circle-ci.md @@ -11,12 +11,19 @@ * Executors: build environments used for jobs * Jobs: a collection of steps run on an executor * Orbs: reusable code that can be imported in to circle config - similar to pip packages, etc -* We currently have 5 workflows: +* Automated workflows: * `build-and-test`: Runs jobs `secrets-check`, `test-frontend` and `test-backend` on every commit * `dev-deployment`: Deploys a PR to the dev space. Triggered by a GitHub action whenever one of the relevant deployment labels is assigned via an API call to Circle CI with the pipeline parameter `run_dev_deployment`. * `nightly`: Runs every night at UTC midnight and performs an OWASP scan against the staging site for both backend and frontend then stores the results in Django using a Cloud Foundry task. * `owasp-scan`: Runs an OWASP scan against the backend and frontend for a given PR. Triggered by a GitHub action whenever the `QASP Review` label is assigned via an API call to Circle CI with the pipeline parameter `run_owasp_scan`. - * `staging-deployment`: Deploys the main branch to the staging space in Cloud.gov. Triggered via merges to the branch `develop`. + * `staging-deployment`: Deploys the main branch to the staging space in Cloud.gov. Triggered via merges to the branch `main`. +* Manual deployment + * Select the desired branch from the branch dropdown on the CircleCI project page + * Click the "Trigger Pipeline" button + * Enter the `target_env`: e.g. `qasp`, `raft`, `a11y` + * Set `triggered` to true + * Set `run_dev_deployment` to true + * Click the `Run Pipeline` button ## How are environment variables supplied in CI? We manually set some environment variables in the project settings for Circle CI. From there, they are used in several places: diff --git a/scripts/apply-database-config.sh b/scripts/apply-database-config.sh index 80422556b..a5a066b95 100644 --- a/scripts/apply-database-config.sh +++ b/scripts/apply-database-config.sh @@ -81,6 +81,7 @@ echo "Done." if [[ $app == "tdp-backend-develop" || $space == "tanf-dev" ]]; then echo "Applying e2e test data" + python manage.py populate_stts python manage.py loaddata cypress/users cypress/data_files cypress/regions cypress/profile_editing_regions cypress/profile_editing_users echo "Done." fi diff --git a/tdrs-backend/celery_start.sh b/tdrs-backend/celery_start.sh index d9da2e5b6..83ecbbcc6 100755 --- a/tdrs-backend/celery_start.sh +++ b/tdrs-backend/celery_start.sh @@ -20,7 +20,7 @@ if [[ $1 == "cloud" ]]; then fi # Celery worker config can be found here: https://docs.celeryq.dev/en/stable/userguide/workers.html#:~:text=The-,hostname,-argument%20can%20expand -celery -A tdpservice.settings worker --loglevel=INFO --concurrency=1 --max-tasks-per-child=1 -n worker1@%h & # tune -c ? +celery -A tdpservice.settings worker --loglevel=INFO --concurrency=1 --max-tasks-per-child=100 -n worker1@%h & # tune -c ? sleep 5 # TODO: Uncomment the following line to add flower service when memory limitation is resolved diff --git a/tdrs-backend/plg/grafana_views/generate_views.py b/tdrs-backend/plg/grafana_views/generate_views.py index 781e9f346..fecf56074 100644 --- a/tdrs-backend/plg/grafana_views/generate_views.py +++ b/tdrs-backend/plg/grafana_views/generate_views.py @@ -99,7 +99,7 @@ def handle_field(field, formatted_fields, is_admin): f''' -- -- Calculate if SSN is valid CASE - WHEN "{field}" IS NOT NULL AND "{field}" !~ '{regex_str}' AND "{field}" LIKE '%[^0-9]%' THEN 1 + WHEN "{field}" IS NOT NULL AND "{field}" ~ '^[0-9]{{9}}$' AND "{field}" !~ '{regex_str}' THEN 1 ELSE 0 END AS "SSN_VALID"'''.strip() ) diff --git a/tdrs-backend/tdpservice/common/test/__init__.py b/tdrs-backend/tdpservice/common/test/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tdrs-backend/tdpservice/common/test/test_util.py b/tdrs-backend/tdpservice/common/test/test_util.py new file mode 100644 index 000000000..70d033e05 --- /dev/null +++ b/tdrs-backend/tdpservice/common/test/test_util.py @@ -0,0 +1,26 @@ +"""Test the common utils.""" + +import pytest + +from tdpservice.common.util import get_cloudgov_broker_db_numbers + + +@pytest.mark.parametrize( + "env,expected", + [ + # dev envs + ("raft", {"celery": "0", "caches": {"stts": "1", "feature-flags": "2"}}), + ("qasp", {"celery": "3", "caches": {"stts": "4", "feature-flags": "5"}}), + ("a11y", {"celery": "6", "caches": {"stts": "7", "feature-flags": "8"}}), + # staging + ("develop", {"celery": "0", "caches": {"stts": "1", "feature-flags": "2"}}), + ("staging", {"celery": "3", "caches": {"stts": "4", "feature-flags": "5"}}), + # prod + ("prod", {"celery": "0", "caches": {"stts": "1", "feature-flags": "2"}}), + ], +) +@pytest.mark.django_db +def test_get_cloudgov_broker_db_numbers(env, expected): + """Test redis broker db number generation for deployed envs.""" + result = get_cloudgov_broker_db_numbers(env) + assert result == expected diff --git a/tdrs-backend/tdpservice/common/util.py b/tdrs-backend/tdpservice/common/util.py new file mode 100644 index 000000000..a21bfd078 --- /dev/null +++ b/tdrs-backend/tdpservice/common/util.py @@ -0,0 +1,32 @@ +"""Utilities for the application.""" + + +def get_cloudgov_broker_db_numbers(cloudgov_name): + """ + Get the appropriate redis broker db numbers for an environment. + + Returns an object of {"celery": str, "caches": {"cache_name": str}} + """ + spaces = { + "dev": ["raft", "qasp", "a11y"], + "staging": ["develop", "staging"], + "prod": ["prod"], + } + cache_options = ["stts", "feature-flags"] + + broker_nums = {} + + for space, envs in spaces.items(): + incr = 0 + + for env in envs: + celery = str(incr) + caches = {} + for c in cache_options: + incr += 1 + caches[c] = str(incr) + + broker_nums[env] = {"celery": celery, "caches": caches} + incr += 1 + + return broker_nums[cloudgov_name] diff --git a/tdrs-backend/tdpservice/conftest.py b/tdrs-backend/tdpservice/conftest.py index affe81273..29850dcf3 100644 --- a/tdrs-backend/tdpservice/conftest.py +++ b/tdrs-backend/tdpservice/conftest.py @@ -198,6 +198,18 @@ def stt(region): return stt +@pytest.fixture +def tribe_stt(region): + """Return a Tribal STT.""" + stt, _ = STT.objects.get_or_create( + name="Blackfeet Nation", + region=region, + stt_code="020", + type=STT.EntityType.TRIBE, + ) + return stt + + @pytest.fixture def region(): """Return a region.""" diff --git a/tdrs-backend/tdpservice/core/models.py b/tdrs-backend/tdpservice/core/models.py index 254c8e4c7..2a14684b7 100644 --- a/tdrs-backend/tdpservice/core/models.py +++ b/tdrs-backend/tdpservice/core/models.py @@ -2,7 +2,10 @@ from django.contrib.auth.models import Permission from django.contrib.contenttypes.models import ContentType +from django.core.cache import caches from django.db import models +from django.db.models.signals import post_delete, post_migrate, post_save +from django.dispatch import receiver class FeatureFlag(models.Model): @@ -11,9 +14,9 @@ class FeatureFlag(models.Model): class Meta: """Metadata.""" - ordering = ['feature_name'] - verbose_name = 'Feature Flag' - verbose_name_plural = 'Feature Flags' + ordering = ["feature_name"] + verbose_name = "Feature Flag" + verbose_name_plural = "Feature Flags" feature_name = models.CharField(max_length=100, unique=True, db_index=True) enabled = models.BooleanField(default=False) @@ -28,6 +31,18 @@ def __str__(self) -> str: return f"{self.feature_name} ({status})" +@receiver([post_delete, post_migrate, post_save], sender=FeatureFlag) +def clear_feature_flag_cache(sender, instance, **kwargs): + """Invalidate the cache after any changes to feature flags. + + This depends on the cache being separated by feature, so the entire cache can be deleted. + There are too many options for headers/cookies to determine the key programatically, + so we segment the different featuers into separate caches to be able to invalidate efficiently + """ + cache = caches["feature-flags"] + cache.clear() + + """Global permissions Allows for the creation of permissions that are diff --git a/tdrs-backend/tdpservice/core/serializers.py b/tdrs-backend/tdpservice/core/serializers.py new file mode 100644 index 000000000..f2e9a73b1 --- /dev/null +++ b/tdrs-backend/tdpservice/core/serializers.py @@ -0,0 +1,20 @@ +"""Serialize core model data.""" + +from rest_framework import serializers + +from tdpservice.core.models import FeatureFlag + + +class FeatureFlagSerializer(serializers.ModelSerializer): + """FeatureFlag serializer.""" + + class Meta: + """Metadata.""" + + model = FeatureFlag + fields = [ + "feature_name", + "enabled", + "config", + "description", + ] diff --git a/tdrs-backend/tdpservice/core/test/test_api.py b/tdrs-backend/tdpservice/core/test/test_api.py index 7896da5db..ad497512d 100644 --- a/tdrs-backend/tdpservice/core/test/test_api.py +++ b/tdrs-backend/tdpservice/core/test/test_api.py @@ -1,12 +1,21 @@ """Core API tests.""" + import uuid +from unittest.mock import MagicMock, patch from django.contrib.admin.models import LogEntry from django.contrib.contenttypes.models import ContentType +from django.core.cache import caches +from django.test import TestCase, override_settings +from django.urls import reverse import pytest from rest_framework import status +from rest_framework.test import APIClient +from tdpservice.conftest import UserFactory +from tdpservice.core.models import FeatureFlag +from tdpservice.core.views import FeatureFlagViewset from tdpservice.data_files.models import DataFile @@ -78,3 +87,119 @@ def test_log_entry_creation(api_client, data_file_instance): content_type_id=ContentType.objects.get_for_model(DataFile).pk, object_id=data_file_instance.pk, ).exists() + + +@override_settings( + CACHES={ + "feature-flags": { + "BACKEND": "django.core.cache.backends.locmem.LocMemCache", + "LOCATION": "unique-test-cache-location", # Unique location to avoid conflicts + "KEY_PREFIX": "test", + }, + } +) +class TestFeatureFlagViewset(TestCase): + """Tests for the FeatureFlagViewset class.""" + + api_client = APIClient() + + def setUp(self): + """Run before all tests in TestCase.""" + super().setUp() + cache = caches["feature-flags"] + cache.clear() + + user = UserFactory.create() + self.api_client.login(username=user.username, password="test_password") + + def test_existing_list_cache_avoids_lookup(self): + """Test that no lookup is performed if flags exist in the cache.""" + mock_queryset = MagicMock() + with patch.object( + FeatureFlagViewset, "get_queryset", return_value=mock_queryset + ) as mock_method: + # request and check the cache was cold + response = self.api_client.get(reverse("feature-flag-list")) + assert response.status_code == status.HTTP_200_OK + assert mock_method.called + + mock_method.reset_mock() + + # the cache should be warm now, request again + response = self.api_client.get(reverse("feature-flag-list")) + assert response.status_code == status.HTTP_200_OK + assert not mock_method.called + + def test_no_list_cache_forces_lookup(self): + """Test that a lookup is performed if there are no flags in the cache.""" + mock_queryset = MagicMock() + with patch.object( + FeatureFlagViewset, "get_queryset", return_value=mock_queryset + ) as mock_method: + # request and check the cache was cold + response = self.api_client.get(reverse("feature-flag-list")) + assert response.status_code == status.HTTP_200_OK + assert mock_method.called + + def test_saving_flag_invalidates_cache(self): + """Test saving a feature flag invalidates existing cache.""" + mock_queryset = MagicMock() + with patch.object( + FeatureFlagViewset, "get_queryset", return_value=mock_queryset + ) as mock_method: + # request and check the cache was cold + response = self.api_client.get(reverse("feature-flag-list")) + assert response.status_code == status.HTTP_200_OK + assert mock_method.called + + mock_method.reset_mock() + + # the cache should be warm now, request again + response = self.api_client.get(reverse("feature-flag-list")) + assert response.status_code == status.HTTP_200_OK + assert not mock_method.called + + mock_method.reset_mock() + + # create a new feature flag + FeatureFlag.objects.create(feature_name="unit-test") + + # check that the cache was invalidated + response = self.api_client.get(reverse("feature-flag-list")) + assert response.status_code == status.HTTP_200_OK + assert mock_method.called + + def test_existing_single_cache_avoids_lookup(self): + """Test that no lookup is performed if flags exist in the cache.""" + FeatureFlag.objects.create(feature_name="test1") + with patch.object( + FeatureFlagViewset, "get_queryset", return_value=FeatureFlag.objects.all() + ) as mock_method: + # request and check the cache was cold + response = self.api_client.get( + reverse("feature-flag-detail", args=("test1",)) + ) + assert response.status_code == status.HTTP_200_OK + assert mock_method.called + + mock_method.reset_mock() + + # the cache should be warm now, request again + response = self.api_client.get( + reverse("feature-flag-detail", args=("test1",)) + ) + assert response.status_code == status.HTTP_200_OK + assert not mock_method.called + + def test_no_single_cache_forces_lookup(self): + """Test that a lookup is performed if there are no flags in the cache.""" + FeatureFlag.objects.create(feature_name="test2") + with patch.object( + FeatureFlagViewset, "get_queryset", return_value=FeatureFlag.objects.all() + ) as mock_method: + # request and check the cache was cold + response = self.api_client.get( + reverse("feature-flag-detail", args=("test2",)) + ) + assert response.status_code == status.HTTP_200_OK + assert mock_method.called diff --git a/tdrs-backend/tdpservice/core/views.py b/tdrs-backend/tdpservice/core/views.py index fb1e2f323..3dc1e4639 100644 --- a/tdrs-backend/tdpservice/core/views.py +++ b/tdrs-backend/tdpservice/core/views.py @@ -1,13 +1,20 @@ """Define core, generic views of the app.""" + import logging +from django.conf import settings from django.contrib.admin.models import ADDITION, LogEntry from django.contrib.contenttypes.models import ContentType +from django.utils.decorators import method_decorator +from django.views.decorators.cache import cache_page +from rest_framework import viewsets from rest_framework.decorators import api_view, permission_classes from rest_framework.permissions import IsAuthenticated from rest_framework.response import Response +from tdpservice.core.models import FeatureFlag +from tdpservice.core.serializers import FeatureFlagSerializer from tdpservice.data_files.models import DataFile logger = logging.getLogger() @@ -56,3 +63,39 @@ def write_logs(request): ) return Response("Success") + + +class FeatureFlagViewset(viewsets.ReadOnlyModelViewSet): + """List and Get endpoints for FeatureFlag.""" + + pagination_class = None + permission_classes = [IsAuthenticated] + queryset = FeatureFlag.objects.all() + serializer_class = FeatureFlagSerializer + lookup_field = "feature_name" + + @method_decorator( + [ + cache_page( + settings.DEFAULT_CACHE_TIMEOUT, + cache="feature-flags", + key_prefix="list", + ), + ] + ) + def list(self, request): + """Get the feature flag list from the cache if available, else fetch the queryset.""" + return super().list(request) + + @method_decorator( + [ + cache_page( + settings.DEFAULT_CACHE_TIMEOUT, + cache="feature-flags", + key_prefix="value", + ), + ] + ) # should these be individually cached? would be cached anyway with above impl + def retrieve(self, request, *args, **kwargs): + """Get the feature flag from cache if available, fallback to db.""" + return super().retrieve(request, *args, **kwargs) diff --git a/tdrs-backend/tdpservice/data_files/test/test_admin_filters.py b/tdrs-backend/tdpservice/data_files/test/test_admin_filters.py new file mode 100644 index 000000000..5c5d193c5 --- /dev/null +++ b/tdrs-backend/tdpservice/data_files/test/test_admin_filters.py @@ -0,0 +1,197 @@ +"""Tests for admin list filters on DataFiles.""" + +import pytest +from django.contrib import admin +from django.test import RequestFactory + +from tdpservice.data_files.admin.filters import LatestReparseEvent, VersionFilter +from tdpservice.data_files.models import DataFile +from tdpservice.data_files.test.factories import DataFileFactory +from tdpservice.search_indexes.models.reparse_meta import ReparseMeta +from tdpservice.stts.test.factories import STTFactory + + +class DummyChangeList: + """Minimal changelist stub for filter choice testing.""" + + def get_query_string(self, new_params, remove): + """Return a simplified query string.""" + return "&".join([f"{key}={value}" for key, value in new_params.items()]) + + +@pytest.mark.django_db +def test_latest_reparse_event_lookups_and_choices(): + """Return lookups and selection states for latest reparse filter.""" + request = RequestFactory().get( + "/", {LatestReparseEvent.parameter_name: "latest"} + ) + model_admin = admin.ModelAdmin(DataFile, admin.site) + + filter_instance = LatestReparseEvent( + request, + request.GET.dict(), + DataFile, + model_admin, + ) + filter_instance.used_parameters[filter_instance.parameter_name] = "latest" + + assert filter_instance.lookups(request, model_admin) == ( + (None, "All"), + ("latest", "Latest"), + ) + + choices = list(filter_instance.choices(DummyChangeList())) + selected = {choice["display"]: choice["selected"] for choice in choices} + assert selected["Latest"] is True + assert selected["All"] is False + + +@pytest.mark.django_db +def test_latest_reparse_event_queryset_filters_latest(stt): + """Filter datafiles to latest reparse event when selected.""" + request = RequestFactory().get("/") + model_admin = admin.ModelAdmin(DataFile, admin.site) + + filter_instance = LatestReparseEvent( + request, + {LatestReparseEvent.parameter_name: "latest"}, + DataFile, + model_admin, + ) + + meta_old = ReparseMeta.objects.create(db_backup_location="s3://old") + meta_new = ReparseMeta.objects.create(db_backup_location="s3://new") + + old_file = DataFileFactory(stt=stt, version=1) + new_file = DataFileFactory(stt=stt, version=2) + old_file.reparses.add(meta_old) + new_file.reparses.add(meta_new) + + filtered = filter_instance.queryset(request, DataFile.objects.all()) + + assert set(filtered.values_list("id", flat=True)) == {new_file.id} + + +@pytest.mark.django_db +def test_latest_reparse_event_queryset_no_latest_returns_all(monkeypatch): + """Return unfiltered queryset when latest meta is missing.""" + request = RequestFactory().get("/") + model_admin = admin.ModelAdmin(DataFile, admin.site) + + filter_instance = LatestReparseEvent( + request, + {LatestReparseEvent.parameter_name: "latest"}, + DataFile, + model_admin, + ) + + monkeypatch.setattr(ReparseMeta, "get_latest", staticmethod(lambda: None)) + + datafile = DataFileFactory() + queryset = DataFile.objects.all() + + assert set(filter_instance.queryset(request, queryset)) == {datafile} + + +@pytest.mark.django_db +def test_latest_reparse_event_queryset_no_value_returns_all(): + """Return unfiltered queryset when filter value is not set.""" + request = RequestFactory().get("/") + model_admin = admin.ModelAdmin(DataFile, admin.site) + + filter_instance = LatestReparseEvent( + request, + {}, + DataFile, + model_admin, + ) + + datafile = DataFileFactory() + queryset = DataFile.objects.all() + + assert set(filter_instance.queryset(request, queryset)) == {datafile} + + +@pytest.mark.django_db +def test_latest_reparse_event_queryset_empty_is_noop(): + """Keep empty queryset unchanged.""" + request = RequestFactory().get("/") + model_admin = admin.ModelAdmin(DataFile, admin.site) + + filter_instance = LatestReparseEvent( + request, + {LatestReparseEvent.parameter_name: "latest"}, + DataFile, + model_admin, + ) + + queryset = DataFile.objects.none() + assert list(filter_instance.queryset(request, queryset)) == [] + + +@pytest.mark.django_db +def test_version_filter_returns_latest_versions(): + """Return only latest versions per file group.""" + request = RequestFactory().get("/") + model_admin = admin.ModelAdmin(DataFile, admin.site) + + stt = STTFactory() + base_kwargs = { + "stt": stt, + "year": 2022, + "quarter": "Q1", + "program_type": DataFile.ProgramType.TANF, + "section": DataFile.Section.ACTIVE_CASE_DATA, + "is_program_audit": False, + } + old_version = DataFileFactory(version=1, **base_kwargs) + new_version = DataFileFactory(version=2, **base_kwargs) + other_group = DataFileFactory( + version=1, + stt=stt, + year=2022, + quarter="Q2", + program_type=DataFile.ProgramType.TANF, + section=DataFile.Section.ACTIVE_CASE_DATA, + is_program_audit=False, + ) + + filter_instance = VersionFilter(request, {}, DataFile, model_admin) + filtered = filter_instance.queryset(request, DataFile.objects.all()) + + assert set(filtered.values_list("id", flat=True)) == { + new_version.id, + other_group.id, + } + assert old_version.id not in filtered.values_list("id", flat=True) + + +@pytest.mark.django_db +def test_version_filter_with_value_returns_unfiltered(stt): + """Return unfiltered queryset when filter has a value.""" + request = RequestFactory().get("/") + model_admin = admin.ModelAdmin(DataFile, admin.site) + + datafile = DataFileFactory(stt=stt) + other_file = DataFileFactory(stt=stt, version=2) + queryset = DataFile.objects.all() + + filter_instance = VersionFilter( + request, + {VersionFilter.parameter_name: "1"}, + DataFile, + model_admin, + ) + + assert set(filter_instance.queryset(request, queryset)) == {datafile, other_file} + + +@pytest.mark.django_db +def test_version_filter_empty_queryset_is_noop(): + """Keep empty queryset unchanged for version filter.""" + request = RequestFactory().get("/") + model_admin = admin.ModelAdmin(DataFile, admin.site) + + filter_instance = VersionFilter(request, {}, DataFile, model_admin) + + assert list(filter_instance.queryset(request, DataFile.objects.none())) == [] diff --git a/tdrs-backend/tdpservice/data_files/test/test_api.py b/tdrs-backend/tdpservice/data_files/test/test_api.py index 2b216914a..fcb9ec08c 100644 --- a/tdrs-backend/tdpservice/data_files/test/test_api.py +++ b/tdrs-backend/tdpservice/data_files/test/test_api.py @@ -713,123 +713,204 @@ def test_list_ofa_admin_data_file_years_no_self_stt( assert response.data == [2020, 2021, 2022] +program_type_options = [i[0] for i in DataFile.ProgramType.choices] +year_options = [2021, 2022] +quarter_options = [i[0] for i in DataFile.Quarter.choices] + +fra_section_options = DataFile.get_fra_section_list() +tanf_section_options = [ + i[0] + for i in DataFile.Section.choices + if not i[0] in DataFile.get_fra_section_list() +] + + +def get_file_types(program_type): + """Return the search api's `file_type`s for a given program.""" + if program_type == DataFile.ProgramType.FRA.value: + return fra_section_options + elif program_type == DataFile.ProgramType.SSP.value: + return ["ssp-moe"] + return ["tanf"] + + +def make_parametrize_set(): + """Return a set of programs/file types/sections to use in `parametrize`.""" + options = [] + for program_type in program_type_options: + file_types = get_file_types(program_type) + + for file_type in file_types: + for year in year_options: + for quarter in quarter_options: + options.append((program_type, file_type, year, quarter)) + + return options + + +@pytest.mark.django_db class TestDataFileQuerysetFiltering: - """Tests for the get_queryset filtering logic with program_type and is_program_audit combinations. + """Tests for the get_queryset filtering logic with program_type, stt, section, year, quarter, and is_program_audit combinations. Note: Program integrity audit (is_program_audit=True) only applies to TANF files. Tribal, SSP, and FRA files do not have audit variants. + + FRA files also query differently than TANF files, the `file_type` url param corresponds to the section for FRA files, but to the program type for TANF/SSP files. Tribal TANF files use `file_type=tanf` but from a tribal STT. """ root_url = "/v1/data_files/" - @pytest.fixture - def tanf_non_audit_file(self, data_analyst, stt): - """Create a TANF file with is_program_audit=False.""" - return DataFile.create_new_version( - { - "original_filename": "tanf_non_audit.txt", - "user": data_analyst, - "stt": stt, - "year": 2024, - "quarter": "Q1", - "section": "Active Case Data", - "program_type": DataFile.ProgramType.TANF, - "is_program_audit": False, - } - ) + def should_test_pia(self, program_type): + """Return true if a file should be tested for program integrity audit.""" + return program_type == DataFile.ProgramType.TANF.value - @pytest.fixture - def tanf_audit_file(self, data_analyst, stt): - """Create a TANF file with is_program_audit=True.""" - return DataFile.create_new_version( - { - "original_filename": "tanf_audit.txt", - "user": data_analyst, - "stt": stt, - "year": 2024, - "quarter": "Q1", - "section": "Active Case Data", - "program_type": DataFile.ProgramType.TANF, - "is_program_audit": True, - } - ) + def get_section_options(self, program_type): + """Return the allowed sections for a given program type.""" + if program_type == DataFile.ProgramType.FRA.value: + return fra_section_options + return tanf_section_options - @pytest.fixture - def tribal_file(self, data_analyst, stt): - """Create a TRIBAL file (is_program_audit is always False for Tribal).""" + def get_location(self, program_type, stt, tribe_stt): + """Return the submitting location for a given program type.""" + if program_type == DataFile.ProgramType.TRIBAL.value: + return tribe_stt + return stt + + def create_file( + self, + program_type, + section, + year, + quarter, + location, + user, + pia=False, + ): + """Create a DataFile object in the db for the test.""" + pia_part = "pia" if pia else "spl" return DataFile.create_new_version( { - "original_filename": "tribal.txt", - "user": data_analyst, - "stt": stt, - "year": 2024, - "quarter": "Q1", - "section": "Active Case Data", - "program_type": DataFile.ProgramType.TRIBAL, - "is_program_audit": False, + "original_filename": f"{program_type}-{section}-{year}-{quarter}-{pia_part}.txt", + "user": user, + "stt": location, + "year": year, + "quarter": quarter, + "section": section, + "program_type": program_type, + "is_program_audit": pia, } ) - @pytest.mark.django_db - def test_tanf_non_audit_appears_in_regular_list( - self, api_client, data_analyst, stt, tanf_non_audit_file, tanf_audit_file - ): - """Test TANF file with is_program_audit=False appears in regular file list.""" - api_client.login(username=data_analyst.username, password="test_password") - - response = api_client.get(f"{self.root_url}?stt={stt.id}&year=2024&quarter=Q1") + def gen_key(self, program_type, year, quarter): + """Generate a key for a given file.""" + k = f"{program_type}{year}{quarter}" + return k + def _make_get_request(self, api_client, url_param_str): + root_url = "/v1/data_files/" + response = api_client.get(f"{root_url}?{url_param_str}") assert response.status_code == status.HTTP_200_OK - file_ids = [f["id"] for f in response.data] - assert len(file_ids) == 1 - assert tanf_non_audit_file.id in file_ids - assert tanf_audit_file.id not in file_ids - - @pytest.mark.django_db - def test_tanf_audit_appears_in_program_integrity_audit_list( - self, api_client, data_analyst, stt, tanf_non_audit_file, tanf_audit_file - ): - """Test TANF file with is_program_audit=True appears in program-integrity-audit list.""" - api_client.login(username=data_analyst.username, password="test_password") - response = api_client.get( - f"{self.root_url}?stt={stt.id}&year=2024&quarter=Q1&file_type=program-integrity-audit" - ) + response_file_ids = [f["id"] for f in response.data] + return response_file_ids - assert response.status_code == status.HTTP_200_OK - file_ids = [f["id"] for f in response.data] - assert len(file_ids) == 1 - assert tanf_audit_file.id in file_ids - assert tanf_non_audit_file.id not in file_ids - - @pytest.mark.django_db - def test_tribal_appears_in_regular_list( - self, api_client, data_analyst, stt, tribal_file, tanf_audit_file - ): - """Test TRIBAL file appears in regular file list alongside TANF non-audit files.""" - api_client.login(username=data_analyst.username, password="test_password") + def _assert_tanf(self, k, non_pia_files, response_file_ids, section_options): + assert len(response_file_ids) == len(section_options) + for f in non_pia_files[k]: + assert f.id in response_file_ids - response = api_client.get(f"{self.root_url}?stt={stt.id}&year=2024&quarter=Q1") + def _assert_fra(self, k, non_pia_files, response_file_ids): + assert len(response_file_ids) == 1 + assert response_file_ids[0] in [f.id for f in non_pia_files[k]] + for f in non_pia_files[k]: + assert f.program_type == DataFile.ProgramType.FRA.value - assert response.status_code == status.HTTP_200_OK - file_ids = [f["id"] for f in response.data] - assert len(file_ids) == 1 - assert tribal_file.id in file_ids - assert tanf_audit_file.id not in file_ids - - @pytest.mark.django_db - def test_tribal_excluded_from_program_integrity_audit_list( - self, api_client, data_analyst, stt, tribal_file, tanf_audit_file + def _assert_pia(self, k, pia_files, response_file_ids, section_options): + assert len(response_file_ids) == len(section_options) + for f in pia_files[k]: + assert f.id in response_file_ids + + @pytest.fixture + def filter_test_data(self, stt, tribe_stt, ofa_system_admin): + """Create a file for each program, section, year, quarter, pia combo.""" + non_pia_files = {} + pia_files = {} + + for program_type in program_type_options: + section_options = self.get_section_options(program_type) + location = self.get_location(program_type, stt, tribe_stt) + + for section in section_options: + for year in year_options: + for quarter in quarter_options: + k = self.gen_key(program_type, year, quarter) + + if k not in non_pia_files: + non_pia_files[k] = [] + + non_pia_files[k].append( + self.create_file( + program_type, + section, + year, + quarter, + location, + ofa_system_admin, + ) + ) + if self.should_test_pia(program_type): + if k not in pia_files: + pia_files[k] = [] + + pia_files[k].append( + self.create_file( + program_type, + section, + year, + quarter, + location, + ofa_system_admin, + pia=True, + ) + ) + + return (stt, tribe_stt, ofa_system_admin, non_pia_files, pia_files) + + @pytest.mark.parametrize( + "program_type,file_type,year,quarter", make_parametrize_set() + ) + def test_get_api_filtering( + self, + api_client, + filter_test_data, + program_type, + file_type, + year, + quarter, ): - """Test TRIBAL files are excluded from program-integrity-audit list (only TANF audit files appear).""" - api_client.login(username=data_analyst.username, password="test_password") + """Check that requests made to the filter endpoint contain every expected file and only files for the requested combination.""" + stt, tribe_stt, ofa_system_admin, non_pia_files, pia_files = filter_test_data + + # check the endpoint filtering + api_client.login(username=ofa_system_admin.username, password="test_password") + location = self.get_location(program_type, stt, tribe_stt) + section_options = self.get_section_options(program_type) - response = api_client.get( - f"{self.root_url}?stt={stt.id}&year=2024&quarter=Q1&file_type=program-integrity-audit" + k = self.gen_key(program_type, year, quarter) + + non_pia_file_ids = self._make_get_request( + api_client, + f"stt={location.id}&year={year}&quarter={quarter}&file_type={file_type}", ) - assert response.status_code == status.HTTP_200_OK - file_ids = [f["id"] for f in response.data] - assert len(file_ids) == 1 - assert tanf_audit_file.id in file_ids - assert tribal_file.id not in file_ids + if program_type == DataFile.ProgramType.FRA.value: + self._assert_fra(k, non_pia_files, non_pia_file_ids) + else: + self._assert_tanf(k, non_pia_files, non_pia_file_ids, section_options) + + if self.should_test_pia(program_type): + pia_file_ids = self._make_get_request( + api_client, + f"stt={location.id}&year={year}&quarter={quarter}&file_type=program-integrity-audit", + ) + self._assert_pia(k, pia_files, pia_file_ids, section_options) diff --git a/tdrs-backend/tdpservice/email/tasks.py b/tdrs-backend/tdpservice/email/tasks.py index 2f47f097f..dc387058d 100644 --- a/tdrs-backend/tdpservice/email/tasks.py +++ b/tdrs-backend/tdpservice/email/tasks.py @@ -21,7 +21,6 @@ send_deactivation_warning_email, ) from tdpservice.email.helpers.admin_notifications import email_admin_deactivated_user -from tdpservice.parsers.util import calendar_to_fiscal from tdpservice.stts.models import STT from tdpservice.users.models import ( AccountApprovalStatusChoices, @@ -172,8 +171,7 @@ def email_admin_num_access_requests(): @shared_task def send_data_submission_reminder(due_date, reporting_period, fiscal_quarter): """Send all Data Analysts a reminder to submit if they have not already.""" - now = datetime.now() - fiscal_year = calendar_to_fiscal(now.year, fiscal_quarter) + fiscal_year = datetime.now().year all_locations = STT.objects.all() @@ -184,19 +182,22 @@ def send_data_submission_reminder(due_date, reporting_period, fiscal_quarter): ) for loc in all_locations: - submitted_sections = ( + submitted_programs_sections = ( year_quarter_files.filter(stt=loc) - .values_list("section", flat=True) + .values_list("program_type", "section") .distinct() ) - required_sections = loc.filenames.keys() - submitted_all_sections = True - for s in required_sections: - if s not in submitted_sections: - submitted_all_sections = False + submitted_programs_sections = [f"{ps[0]} {ps[1]}".upper() for ps in submitted_programs_sections] - if not submitted_all_sections: + required_program_sections = loc.filenames.keys() + required_program_sections = [ps.upper() for ps in required_program_sections] + + submitted_all_programs_sections = all( + ps in submitted_programs_sections for ps in required_program_sections + ) + + if not submitted_all_programs_sections: reminder_locations.append(loc) template_path = DataFileEmail.UPCOMING_SUBMISSION_DEADLINE.value diff --git a/tdrs-backend/tdpservice/email/test/test_upcoming_deadline_email.py b/tdrs-backend/tdpservice/email/test/test_upcoming_deadline_email.py index fb91c4c7b..8a1bd08d0 100644 --- a/tdrs-backend/tdpservice/email/test/test_upcoming_deadline_email.py +++ b/tdrs-backend/tdpservice/email/test/test_upcoming_deadline_email.py @@ -13,7 +13,7 @@ from tdpservice.users.models import User -@pytest.mark.parametrize( +QUARTERLY_PARAMS = pytest.mark.parametrize( "due_date, reporting_period, fiscal_quarter", [ ("February 14", "Oct - Dec", "Q1"), @@ -22,139 +22,293 @@ ("November 14th", "Jul - Sep", "Q4"), ], ) -@pytest.mark.django_db -def test_upcoming_deadline_sends_no_sections_submitted( - due_date, reporting_period, fiscal_quarter -): - """Test that the send_deactivation_warning_email function runs when no sections have been submitted.""" - stt = STT.objects.create( - name="Arkansas", - filenames={ - "Active Case Data": "test-filename.txt", - "Closed Case Data": "test-filename-closed.txt", - }, - ) + +def _create_stt_with_analyst(name, filenames, ssp=False): + """Create an STT and an approved Data Analyst assigned to it.""" + stt = STT.objects.create(name=name, filenames=filenames, ssp=ssp) data_analyst = User.objects.create( - username="test@email.com", stt=stt, account_approval_status="Approved" + username=f"{name.lower().replace(' ', '')}@test.com", + stt=stt, + account_approval_status="Approved", ) data_analyst.groups.add(Group.objects.get(name="Data Analyst")) data_analyst.save() + return stt, data_analyst - send_data_submission_reminder(due_date, reporting_period, fiscal_quarter) - assert len(mail.outbox) == 1 - assert ( - mail.outbox[0].subject == "Action Requested: Please submit your TANF data files" +def _submit_file(stt, user, program_type, section, fiscal_quarter): + """Create a DataFile for the current fiscal year and given quarter.""" + return DataFile.create_new_version( + { + "section": section, + "program_type": program_type, + "quarter": fiscal_quarter, + "year": datetime.now().year, + "stt": stt, + "user": user, + "is_program_audit": False, + } ) -@pytest.mark.parametrize( - "due_date, reporting_period, fiscal_quarter", - [ - ("February 14", "Oct - Dec", "Q1"), - ("May 15th", "Jan - Mar", "Q2"), - ("August 14th", "Apr - Jun", "Q3"), - ("November 14th", "Jul - Sep", "Q4"), - ], -) +@QUARTERLY_PARAMS @pytest.mark.django_db -def test_upcoming_deadline_sends_some_sections_submitted( +def test_upcoming_deadline_sends_no_sections_submitted( due_date, reporting_period, fiscal_quarter ): - """Test that the send_deactivation_warning_email function runs when some sections have been submitted.""" - stt = STT.objects.create( - name="Arkansas", - filenames={ - "Active Case Data": "test-filename.txt", - "Closed Case Data": "test-filename-closed.txt", + """Reminder is sent when no sections have been submitted.""" + stt, _ = _create_stt_with_analyst( + "Arkansas", + { + "TAN Active Case Data": "test-filename.txt", + "TAN Closed Case Data": "test-filename-closed.txt", }, ) - data_analyst = User.objects.create( - username="test@email.com", stt=stt, account_approval_status="Approved" + send_data_submission_reminder(due_date, reporting_period, fiscal_quarter) + + assert len(mail.outbox) == 1 + assert ( + mail.outbox[0].subject + == "Action Requested: Please submit your TANF data files" ) - data_analyst.groups.add(Group.objects.get(name="Data Analyst")) - data_analyst.save() - now = datetime.now() - fiscal_year = now.year - 1 if fiscal_quarter == "Q1" else now.year - _ = DataFile.create_new_version( +@QUARTERLY_PARAMS +@pytest.mark.django_db +def test_upcoming_deadline_sends_some_sections_submitted( + due_date, reporting_period, fiscal_quarter +): + """Reminder is sent when only some sections have been submitted.""" + stt, analyst = _create_stt_with_analyst( + "Arkansas", { - "section": "Active Case Data", - "program_type": "TAN", - "quarter": fiscal_quarter, - "year": fiscal_year, - "stt": stt, - "user": data_analyst, - "is_program_audit": False, - } + "TAN Active Case Data": "test-filename.txt", + "TAN Closed Case Data": "test-filename-closed.txt", + }, ) + _submit_file(stt, analyst, "TAN", "Active Case Data", fiscal_quarter) + send_data_submission_reminder(due_date, reporting_period, fiscal_quarter) assert len(mail.outbox) == 1 assert ( - mail.outbox[0].subject == "Action Requested: Please submit your TANF data files" + mail.outbox[0].subject + == "Action Requested: Please submit your TANF data files" ) -@pytest.mark.parametrize( - "due_date, reporting_period, fiscal_quarter", - [ - ("February 14", "Oct - Dec", "Q1"), - ("May 15th", "Jan - Mar", "Q2"), - ("August 14th", "Apr - Jun", "Q3"), - ("November 14th", "Jul - Sep", "Q4"), - ], -) +@QUARTERLY_PARAMS @pytest.mark.django_db def test_upcoming_deadline_no_send_when_all_sections_complete( due_date, reporting_period, fiscal_quarter ): - """Test that the send_deactivation_warning_email function does not run when all sections have been submitted.""" - stt = STT.objects.create( - name="Arkansas", - filenames={ - "Active Case Data": "test-filename.txt", - "Closed Case Data": "test-filename-closed.txt", + """No reminder is sent when all required sections have been submitted.""" + stt, analyst = _create_stt_with_analyst( + "Arkansas", + { + "TAN Active Case Data": "test-filename.txt", + "TAN Closed Case Data": "test-filename-closed.txt", }, ) - data_analyst = User.objects.create( - username="test@email.com", stt=stt, account_approval_status="Approved" + _submit_file(stt, analyst, "TAN", "Active Case Data", fiscal_quarter) + _submit_file(stt, analyst, "TAN", "Closed Case Data", fiscal_quarter) + + send_data_submission_reminder(due_date, reporting_period, fiscal_quarter) + + assert len(mail.outbox) == 0 + + +@pytest.mark.django_db +def test_q1_files_found_using_current_year(): + """Q1 DataFiles stored with the current year are found by the task.""" + stt, analyst = _create_stt_with_analyst( + "TestState", + { + "TAN Active Case Data": "file1.txt", + "TAN Closed Case Data": "file2.txt", + }, ) - data_analyst.groups.add(Group.objects.get(name="Data Analyst")) - data_analyst.save() - now = datetime.now() - fiscal_year = now.year - 1 if fiscal_quarter == "Q1" else now.year + # Submit all required files for Q1 using the current year + _submit_file(stt, analyst, "TAN", "Active Case Data", "Q1") + _submit_file(stt, analyst, "TAN", "Closed Case Data", "Q1") - _ = DataFile.create_new_version( + send_data_submission_reminder("February 14", "Oct - Dec", "Q1") + + # All files submitted → no reminder should be sent + assert len(mail.outbox) == 0 + + +@pytest.mark.django_db +def test_q1_files_with_previous_year_are_not_matched(): + """Data files stored with the previous year should not satisfy the current period.""" + stt, analyst = _create_stt_with_analyst( + "TestState", + { + "TAN Active Case Data": "file1.txt", + "TAN Closed Case Data": "file2.txt", + }, + ) + + # Submit files with LAST year — these should not count + last_year = datetime.now().year - 1 + DataFile.create_new_version( { "section": "Active Case Data", "program_type": "TAN", - "quarter": fiscal_quarter, - "year": fiscal_year, + "quarter": "Q1", + "year": last_year, "stt": stt, - "user": data_analyst, + "user": analyst, "is_program_audit": False, } ) - - _ = DataFile.create_new_version( + DataFile.create_new_version( { "section": "Closed Case Data", "program_type": "TAN", - "quarter": fiscal_quarter, - "year": fiscal_year, + "quarter": "Q1", + "year": last_year, "stt": stt, - "user": data_analyst, + "user": analyst, "is_program_audit": False, } ) - send_data_submission_reminder(due_date, reporting_period, fiscal_quarter) + send_data_submission_reminder("February 14", "Oct - Dec", "Q1") + + # Files are for the wrong year → reminder should still be sent + assert len(mail.outbox) == 1 + + +@pytest.mark.django_db +def test_tribal_submission_does_not_satisfy_tanf_requirement(): + """A Tribal DataFile should not count toward TANF filing requirements. + + Regression: the old code only checked section name, so a Tribal 'Active + Case Data' submission would satisfy a TANF 'Active Case Data' requirement. + """ + stt, analyst = _create_stt_with_analyst( + "TestState", + { + "TAN Active Case Data": "file1.txt", + "TAN Closed Case Data": "file2.txt", + }, + ) + + # Submit Tribal files — wrong program type for this STT + _submit_file(stt, analyst, "TRIBAL", "Active Case Data", "Q1") + _submit_file(stt, analyst, "TRIBAL", "Closed Case Data", "Q1") + + send_data_submission_reminder("February 14", "Oct - Dec", "Q1") + + # Tribal files don't satisfy TANF requirement → reminder sent + assert len(mail.outbox) == 1 + + +@pytest.mark.django_db +def test_tanf_submission_does_not_satisfy_tribal_requirement(): + """A TANF DataFile should not count toward Tribal filing requirements.""" + stt, analyst = _create_stt_with_analyst( + "TestTribe", + { + "Tribal Active Case Data": "file1.txt", + "Tribal Closed Case Data": "file2.txt", + "Tribal Aggregate Data": "file3.txt", + }, + ) + + # Submit TANF files — wrong program type for a tribal STT + _submit_file(stt, analyst, "TAN", "Active Case Data", "Q1") + _submit_file(stt, analyst, "TAN", "Closed Case Data", "Q1") + _submit_file(stt, analyst, "TAN", "Aggregate Data", "Q1") + + send_data_submission_reminder("February 14", "Oct - Dec", "Q1") + + # TANF files don't satisfy Tribal requirement → reminder sent + assert len(mail.outbox) == 1 + + +@pytest.mark.django_db +def test_tribal_stt_no_reminder_when_all_tribal_sections_submitted(): + """A Tribal STT that has submitted all Tribal files should not get a reminder.""" + stt, analyst = _create_stt_with_analyst( + "TestTribe", + { + "Tribal Active Case Data": "file1.txt", + "Tribal Closed Case Data": "file2.txt", + "Tribal Aggregate Data": "file3.txt", + }, + ) + + _submit_file(stt, analyst, "TRIBAL", "Active Case Data", "Q1") + _submit_file(stt, analyst, "TRIBAL", "Closed Case Data", "Q1") + _submit_file(stt, analyst, "TRIBAL", "Aggregate Data", "Q1") + + send_data_submission_reminder("February 14", "Oct - Dec", "Q1") + + assert len(mail.outbox) == 0 + + +@pytest.mark.django_db +def test_ssp_stt_requires_both_tanf_and_ssp_submissions(): + """An SSP state must submit both TANF and SSP files to avoid a reminder. + + Regression: the old code ignored program type, so submitting only TANF + files could falsely satisfy SSP requirements (since both have the same + section names). + """ + stt, analyst = _create_stt_with_analyst( + "RhodeIsland", + { + "TAN Active Case Data": "tanf1.txt", + "TAN Closed Case Data": "tanf2.txt", + "TAN Aggregate Data": "tanf3.txt", + "SSP Active Case Data": "ssp1.txt", + "SSP Closed Case Data": "ssp2.txt", + "SSP Aggregate Data": "ssp3.txt", + }, + ssp=True, + ) + + # Only submit TANF files — SSP requirements are not satisfied + _submit_file(stt, analyst, "TAN", "Active Case Data", "Q2") + _submit_file(stt, analyst, "TAN", "Closed Case Data", "Q2") + _submit_file(stt, analyst, "TAN", "Aggregate Data", "Q2") + + send_data_submission_reminder("May 15th", "Jan - Mar", "Q2") + + # Missing SSP files → reminder sent + assert len(mail.outbox) == 1 + assert "SSP" in mail.outbox[0].subject + + +@pytest.mark.django_db +def test_ssp_stt_no_reminder_when_all_programs_submitted(): + """An SSP state that has submitted all TANF and SSP files should not get a reminder.""" + stt, analyst = _create_stt_with_analyst( + "RhodeIsland", + { + "TAN Active Case Data": "tanf1.txt", + "TAN Closed Case Data": "tanf2.txt", + "TAN Aggregate Data": "tanf3.txt", + "SSP Active Case Data": "ssp1.txt", + "SSP Closed Case Data": "ssp2.txt", + "SSP Aggregate Data": "ssp3.txt", + }, + ssp=True, + ) + + # Submit both TANF and SSP files + for program_type in ("TAN", "SSP"): + _submit_file(stt, analyst, program_type, "Active Case Data", "Q2") + _submit_file(stt, analyst, program_type, "Closed Case Data", "Q2") + _submit_file(stt, analyst, program_type, "Aggregate Data", "Q2") + + send_data_submission_reminder("May 15th", "Jan - Mar", "Q2") assert len(mail.outbox) == 0 diff --git a/tdrs-backend/tdpservice/parsers/aggregates.py b/tdrs-backend/tdpservice/parsers/aggregates.py index 3dbbb5fcc..24c117a55 100644 --- a/tdrs-backend/tdpservice/parsers/aggregates.py +++ b/tdrs-backend/tdpservice/parsers/aggregates.py @@ -23,15 +23,14 @@ def case_aggregates_by_month(df, dfs_status): aggregate_data = {"months": [], "rejected": 0} all_errors = ParserError.objects.filter(file=df, deprecated=False) + + rpt_month_years = [] for month in month_list: - total = 0 - cases_with_errors = 0 - accepted = 0 month_int = month_to_int(month) - rpt_month_year = int(f"{calendar_year}{month_int}") - if dfs_status == "Rejected": - # we need to be careful here on examples of bad headers or empty files, since no month will be found - # but we can rely on the frontend submitted year-quarter to still generate the list of months + rpt_month_years.append(int(f"{calendar_year}{month_int}")) + + if dfs_status == "Rejected": + for month in month_list: aggregate_data["months"].append( { "accepted_with_errors": "N/A", @@ -39,34 +38,50 @@ def case_aggregates_by_month(df, dfs_status): "month": month, } ) - continue - - case_numbers = set() + else: + # Collect all unique case numbers per month across all record types (e.g. T1, T2, T3). + # A case can span multiple record types, so we union them into a single set per month. + # Only cases that were successfully parsed and saved to the model are included here. + case_numbers_by_month = {rmy: set() for rmy in rpt_month_years} for schema in schemas.values(): schema = schema[0] - - curr_case_numbers = set( - schema.model.objects.filter(datafile=df, RPT_MONTH_YEAR=rpt_month_year) - .distinct("CASE_NUMBER") - .values_list("CASE_NUMBER", flat=True) + records = ( + schema.model.objects.filter(datafile=df, RPT_MONTH_YEAR__in=rpt_month_years) + .values_list("CASE_NUMBER", "RPT_MONTH_YEAR") + .distinct() ) - case_numbers = case_numbers.union(curr_case_numbers) - - total += len(case_numbers) - cases_with_errors += ( - all_errors.filter(case_number__in=case_numbers) - .distinct("case_number") - .count() - ) - accepted = total - cases_with_errors - - aggregate_data["months"].append( - { - "month": month, - "accepted_without_errors": accepted, - "accepted_with_errors": cases_with_errors, - } + for case_number, rmy in records: + case_numbers_by_month[rmy].add(case_number) + + # Find which of those case numbers have errors, grouped by month. + # We query errors once using the full set of case numbers across all months, then + # bucket the results by rpt_month_year so each error is attributed to the correct month. + # Note: this does bring all distinct case numbers for each month into memory. If we ever get files with a huge + # number of cases, we could get OOM. If that happens we can move to a raw SQL query to union all the tables. + all_case_numbers = set().union(*case_numbers_by_month.values()) + error_cases_by_month = {rmy: set() for rmy in rpt_month_years} + error_records = ( + all_errors.filter(case_number__in=all_case_numbers, rpt_month_year__in=rpt_month_years) + .values_list("case_number", "rpt_month_year") + .distinct() ) + for case_number, rmy in error_records: + error_cases_by_month[rmy].add(case_number) + + # Compute per-month aggregates. + for month, rmy in zip(month_list, rpt_month_years): + total = len(case_numbers_by_month[rmy]) + # Intersect to exclude rejected records that generated cat1/cat4 errors since they aren't serialized + cases_with_errors = len(error_cases_by_month[rmy] & case_numbers_by_month[rmy]) + accepted = total - cases_with_errors + + aggregate_data["months"].append( + { + "month": month, + "accepted_without_errors": accepted, + "accepted_with_errors": cases_with_errors, + } + ) error_type_query = ( Query(error_type=ParserErrorCategoryChoices.PRE_CHECK) diff --git a/tdrs-backend/tdpservice/parsers/test/conftest.py b/tdrs-backend/tdpservice/parsers/test/conftest.py index 96ed02990..7c1667a21 100644 --- a/tdrs-backend/tdpservice/parsers/test/conftest.py +++ b/tdrs-backend/tdpservice/parsers/test/conftest.py @@ -67,6 +67,10 @@ def big_file(stt_user, stt): """Fixture for ADS.E2J.FTP1.TS06.""" return util.create_test_datafile("ADS.E2J.FTP1.TS06", stt_user, stt) +@pytest.fixture +def case_aggregates_edge_case(stt_user, stt): + """Fixture for cases_across_months_with_error.txt which ensures the case aggregates algorithm doesn't double count errors across cases.""" + return util.create_test_datafile("cases_across_months_with_error.txt", stt_user, stt) @pytest.fixture def bad_test_file(stt_user, stt): diff --git a/tdrs-backend/tdpservice/parsers/test/data/cases_across_months_with_error.txt b/tdrs-backend/tdpservice/parsers/test/data/cases_across_months_with_error.txt new file mode 100644 index 000000000..4da14184a --- /dev/null +++ b/tdrs-backend/tdpservice/parsers/test/data/cases_across_months_with_error.txt @@ -0,0 +1,16 @@ +HEADER20254A21000TAN1 D +T12025100101022891A19501415671120332100418000000000000004500780000000000000005000000000000000000202222000000002220020000000000000000000000000000000000000000 +T22025100101022891A32199508211112233332222212222222013212110200001069945000000000000000000000000000000000000000000000000000000000000398900000000000000000000 +T32025100101022891A12020052111223333422222112207398100000000120181110111223333222221222073981000000000000000000000000000000000000000000000000000000000000000 +T12025110101022891A19501415671110532100015000000000000002930790000000000000005000000000000000000202222000000002220020000000000000000000000000000000000000000 +T22025110101022891A32199508211122333352222212222222013212110200001069942000000000000000000000000000000000000000000000000000000000000372200000000000000000000 +T22025110101022891A52199410311122333312222211222222020212190000003069900000000000000000000000000000000000000000000000000000000000000000000000000000000000000 +T22025110101022891A52199410311122333312222211222222020212190000003069900000000000000000000000000000000000000000000000000000000000000000000000000000000000000 +T32025110101022891A12020052111223333422222112207398100000000120181110111223333222221222073981000000000000000000000000000000000000000000000000000000000000000 +T12025110101703738A11101402141110332100546000000000000002930260000000000000000000000000000000000202222000000002220020000000000000000000000000000000000000000 +T22025110101703738A22197611011122333341222212222221012212910000003079900000000000000000000000000000000000000000000000000000000000000000000000000000000000000 +T32025110101703738A12009091111223333612222122204310100000000120080401111223333122221222043111000000000000000000000000000000000000000000000000000000000000000 +T12025120101703738A11101402141110332100546000000000000002930270000000000000000000000000000000000202222000000002220020000000000000000000000000000000000000000 +T22025120101703738A22197611011122333341222212222221012212110000003001800000000000000000000000000000000000000000000000000000000000000000000000000000000000000 +T32025120101703738A12009091111223333612222122204310100000000120080401111223333122221222043111000000000000000000000000000000000000000000000000000000000000000 +TRAILER0153861 \ No newline at end of file diff --git a/tdrs-backend/tdpservice/parsers/test/helpers.py b/tdrs-backend/tdpservice/parsers/test/helpers.py new file mode 100644 index 000000000..600336429 --- /dev/null +++ b/tdrs-backend/tdpservice/parsers/test/helpers.py @@ -0,0 +1,17 @@ +"""Shared helpers for parser tests.""" + +from tdpservice.parsers.factory import ParserFactory + + +def parse_datafile(dfs, datafile, **factory_kwargs): + """Parse a datafile using the parser factory with consistent defaults.""" + dfs.datafile = datafile + parser = ParserFactory.get_instance( + datafile=datafile, + dfs=dfs, + section=factory_kwargs.pop("section", datafile.section), + program_type=factory_kwargs.pop("program_type", datafile.program_type), + **factory_kwargs, + ) + parser.parse_and_validate() + return parser diff --git a/tdrs-backend/tdpservice/parsers/test/test_dataclasses.py b/tdrs-backend/tdpservice/parsers/test/test_dataclasses.py index 8a0f12a41..bcda59e60 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_dataclasses.py +++ b/tdrs-backend/tdpservice/parsers/test/test_dataclasses.py @@ -3,47 +3,48 @@ from tdpservice.parsers.dataclasses import Position, RawRow, TupleRow -def test_position(): - """Test the Position class.""" - pos1 = Position(1, 3) - pos2 = Position(1) - - assert pos1.is_range is True - assert len(pos1) == 2 - - assert pos2.end == 2 - assert pos2.is_range is False - assert len(pos2) == 1 - - -def test_raw_row(): - """Test the RawRow class.""" - row = RawRow("test", 4, 4, 1, "test") - - assert row.value_at(Position(0, 2)) == "te" - assert row.value_at_is(Position(0, 2), "te") is True - assert row.raw_length() == 4 - assert len(row) == 4 - assert row[1] == "e" - assert str(row) == "test" - assert hash(row) == hash("test") - assert row == RawRow("test", 4, 4, 1, "test") - assert row != RawRow("test2", 4, 4, 1, "test2") - - -def test_tuple_row(): - """Test the TupleRow class.""" - row = TupleRow(("test", "test2"), 2, 2, 1, "test") - raw_row = RawRow(("test", "test2"), 2, 2, 1, "test") - - assert row.value_at(Position(0)) == "test" - assert row.value_at_is(Position(0), "test") is True - assert row.value_at(Position(0, 2)) == ("test", "test2") - assert row.raw_length() == 2 - assert len(row) == 2 - assert row[1] == "test2" - assert str(row) == "('test', 'test2')" - assert hash(row) == hash(("test", "test2")) - assert row == raw_row - assert row == TupleRow(("test", "test2"), 2, 2, 1, "test") - assert row != RawRow("test", 4, 4, 1, "test") +class TestParserDataclasses: + """Tests for parser dataclasses.""" + + def test_position(self): + """Test the Position class.""" + pos1 = Position(1, 3) + pos2 = Position(1) + + assert pos1.is_range is True + assert len(pos1) == 2 + + assert pos2.end == 2 + assert pos2.is_range is False + assert len(pos2) == 1 + + def test_raw_row(self): + """Test the RawRow class.""" + row = RawRow("test", 4, 4, 1, "test") + + assert row.value_at(Position(0, 2)) == "te" + assert row.value_at_is(Position(0, 2), "te") is True + assert row.raw_length() == 4 + assert len(row) == 4 + assert row[1] == "e" + assert str(row) == "test" + assert hash(row) == hash("test") + assert row == RawRow("test", 4, 4, 1, "test") + assert row != RawRow("test2", 4, 4, 1, "test2") + + def test_tuple_row(self): + """Test the TupleRow class.""" + row = TupleRow(("test", "test2"), 2, 2, 1, "test") + raw_row = RawRow(("test", "test2"), 2, 2, 1, "test") + + assert row.value_at(Position(0)) == "test" + assert row.value_at_is(Position(0), "test") is True + assert row.value_at(Position(0, 2)) == ("test", "test2") + assert row.raw_length() == 2 + assert len(row) == 2 + assert row[1] == "test2" + assert str(row) == "('test', 'test2')" + assert hash(row) == hash(("test", "test2")) + assert row == raw_row + assert row == TupleRow(("test", "test2"), 2, 2, 1, "test") + assert row != RawRow("test", 4, 4, 1, "test") diff --git a/tdrs-backend/tdpservice/parsers/test/test_decoders.py b/tdrs-backend/tdpservice/parsers/test/test_decoders.py index 90291adc8..43c241dc7 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_decoders.py +++ b/tdrs-backend/tdpservice/parsers/test/test_decoders.py @@ -11,86 +11,83 @@ ) -@pytest.mark.django_db -def test_utf8_decoder(small_correct_file): - """Test UTF8 decoder is selected and decodes data.""" - decoder = DecoderFactory.get_instance(small_correct_file.file) - assert isinstance(decoder, Utf8Decoder) - assert decoder.raw_file == small_correct_file.file - header_row = next(decoder.decode()) - assert isinstance(header_row, RawRow) - assert isinstance(header_row.data, str) - assert header_row.data == "HEADER20204A06 TAN1 D" +class TestDecoderFactory: + """Tests for decoder selection and decoding.""" - -@pytest.mark.django_db -def test_csv_decoder(fra_csv): - """Test CSV decoder is selected and decodes data.""" - decoder = DecoderFactory.get_instance(fra_csv.file) - assert isinstance(decoder, CsvDecoder) - assert decoder.raw_file == fra_csv.file - first_row = next(decoder.decode()) - assert isinstance(first_row, TupleRow) - assert isinstance(first_row.data, tuple) - assert first_row.data == ("202401", "946412419") - - -@pytest.mark.django_db -def test_xlsx_decoder(fra_xlsx): - """Test XLSX decoder is selected and decodes data.""" - decoder = DecoderFactory.get_instance(fra_xlsx.file) - assert isinstance(decoder, XlsxDecoder) - assert decoder.raw_file == fra_xlsx.file - first_row = next(decoder.decode()) - assert isinstance(first_row, TupleRow) - assert isinstance(first_row.data, tuple) - assert first_row.data == (202401, 946412419) - - -@pytest.mark.django_db -def test_xlsx_decoder_multisheet(fra_multi_sheet_xlsx): - """Test XLSX decoder is selected and decodes data.""" - decoder = DecoderFactory.get_instance(fra_multi_sheet_xlsx.file) - assert isinstance(decoder, XlsxDecoder) - assert decoder.raw_file == fra_multi_sheet_xlsx.file - first_row = next(decoder.decode()) - assert isinstance(first_row, TupleRow) - assert isinstance(first_row.data, tuple) - assert first_row.data == (202401, 946412419) - - -@pytest.mark.django_db -def test_empty_file_decoder(empty_file): - """Test UTF8 decoder is selected on empty file with no extension.""" - with pytest.raises(StopIteration): - decoder = DecoderFactory.get_instance(empty_file.file) + @pytest.mark.django_db + def test_utf8_decoder(self, small_correct_file): + """Test UTF8 decoder is selected and decodes data.""" + decoder = DecoderFactory.get_instance(small_correct_file.file) assert isinstance(decoder, Utf8Decoder) - assert decoder.raw_file == empty_file.file - - # Shouldn't be able to decode anything since file is empty - next(decoder.decode()) - - -@pytest.mark.django_db -def test_unknown_decoder(unknown_png): - """Test unknown decoder.""" - with pytest.raises(ValueError) as e: - DecoderFactory.get_instance(unknown_png.file) - assert repr(e) == "Could not determine what decoder to use for file." - - -@pytest.mark.django_db -def test_file_offset(small_correct_file): - """Test raw length matches file length.""" - decoder = DecoderFactory.get_instance(small_correct_file.file) - assert isinstance(decoder, Utf8Decoder) - assert decoder.raw_file == small_correct_file.file - raw_length = 0 - decoded_length = 0 - for row in decoder.decode(): - assert "\n" not in row.data - assert "\r" not in row.data - raw_length += row.raw_length() - decoded_length += len(row) - assert raw_length == len(small_correct_file.file) - assert decoded_length != raw_length + assert decoder.raw_file == small_correct_file.file + header_row = next(decoder.decode()) + assert isinstance(header_row, RawRow) + assert isinstance(header_row.data, str) + assert header_row.data == "HEADER20204A06 TAN1 D" + + @pytest.mark.django_db + def test_csv_decoder(self, fra_csv): + """Test CSV decoder is selected and decodes data.""" + decoder = DecoderFactory.get_instance(fra_csv.file) + assert isinstance(decoder, CsvDecoder) + assert decoder.raw_file == fra_csv.file + first_row = next(decoder.decode()) + assert isinstance(first_row, TupleRow) + assert isinstance(first_row.data, tuple) + assert first_row.data == ("202401", "946412419") + + @pytest.mark.django_db + def test_xlsx_decoder(self, fra_xlsx): + """Test XLSX decoder is selected and decodes data.""" + decoder = DecoderFactory.get_instance(fra_xlsx.file) + assert isinstance(decoder, XlsxDecoder) + assert decoder.raw_file == fra_xlsx.file + first_row = next(decoder.decode()) + assert isinstance(first_row, TupleRow) + assert isinstance(first_row.data, tuple) + assert first_row.data == (202401, 946412419) + + @pytest.mark.django_db + def test_xlsx_decoder_multisheet(self, fra_multi_sheet_xlsx): + """Test XLSX decoder is selected and decodes data.""" + decoder = DecoderFactory.get_instance(fra_multi_sheet_xlsx.file) + assert isinstance(decoder, XlsxDecoder) + assert decoder.raw_file == fra_multi_sheet_xlsx.file + first_row = next(decoder.decode()) + assert isinstance(first_row, TupleRow) + assert isinstance(first_row.data, tuple) + assert first_row.data == (202401, 946412419) + + @pytest.mark.django_db + def test_empty_file_decoder(self, empty_file): + """Test UTF8 decoder is selected on empty file with no extension.""" + with pytest.raises(StopIteration): + decoder = DecoderFactory.get_instance(empty_file.file) + assert isinstance(decoder, Utf8Decoder) + assert decoder.raw_file == empty_file.file + + # Shouldn't be able to decode anything since file is empty + next(decoder.decode()) + + @pytest.mark.django_db + def test_unknown_decoder(self, unknown_png): + """Test unknown decoder.""" + with pytest.raises(ValueError) as e: + DecoderFactory.get_instance(unknown_png.file) + assert repr(e) == "Could not determine what decoder to use for file." + + @pytest.mark.django_db + def test_file_offset(self, small_correct_file): + """Test raw length matches file length.""" + decoder = DecoderFactory.get_instance(small_correct_file.file) + assert isinstance(decoder, Utf8Decoder) + assert decoder.raw_file == small_correct_file.file + raw_length = 0 + decoded_length = 0 + for row in decoder.decode(): + assert "\n" not in row.data + assert "\r" not in row.data + raw_length += row.raw_length() + decoded_length += len(row) + assert raw_length == len(small_correct_file.file) + assert decoded_length != raw_length diff --git a/tdrs-backend/tdpservice/parsers/test/test_header.py b/tdrs-backend/tdpservice/parsers/test/test_header.py index fe25ed6a2..66fc75afc 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_header.py +++ b/tdrs-backend/tdpservice/parsers/test/test_header.py @@ -10,122 +10,123 @@ logger = logging.getLogger(__name__) -@pytest.fixture -def test_datafile(stt_user, stt): - """Fixture for small_correct_file.""" - return util.create_test_datafile("small_correct_file.txt", stt_user, stt) +class TestHeaderSchema: + """Tests for header schema parsing.""" + @pytest.fixture + def test_datafile(self, stt_user, stt): + """Fixture for small_correct_file.""" + return util.create_test_datafile("small_correct_file.txt", stt_user, stt) -@pytest.mark.django_db -def test_header_cleanup(test_datafile): - """Test the header parser.""" - YEAR = "2020" - QUARTER = "4" - TYPE = "A" - STATE_FIPS = " " - TRIBE_CODE = " " - PROGRAM_CODE = "TAN" - EDIT_CODE = "1" - ENCRYPTION_CODE = " " - UPDATE_INDICATOR = "D" - header_line = ( - f"HEADER{YEAR}{QUARTER}{TYPE}{STATE_FIPS}{TRIBE_CODE}" - + f"{PROGRAM_CODE}{EDIT_CODE}{ENCRYPTION_CODE}{UPDATE_INDICATOR}" - ) - length = len(header_line) - row = RawRow( - data=header_line, - raw_len=length, - decoded_len=length, - row_num=1, - record_type="HEADER", - ) - header_schema = schema_defs.header - header_schema.prepare(test_datafile) - header, header_is_valid, header_errors = header_schema.parse_and_validate(row) + @pytest.mark.django_db + def test_header_cleanup(self, test_datafile): + """Test the header parser.""" + YEAR = "2020" + QUARTER = "4" + TYPE = "A" + STATE_FIPS = " " + TRIBE_CODE = " " + PROGRAM_CODE = "TAN" + EDIT_CODE = "1" + ENCRYPTION_CODE = " " + UPDATE_INDICATOR = "D" + header_line = ( + f"HEADER{YEAR}{QUARTER}{TYPE}{STATE_FIPS}{TRIBE_CODE}" + + f"{PROGRAM_CODE}{EDIT_CODE}{ENCRYPTION_CODE}{UPDATE_INDICATOR}" + ) + length = len(header_line) + row = RawRow( + data=header_line, + raw_len=length, + decoded_len=length, + row_num=1, + record_type="HEADER", + ) + header_schema = schema_defs.header + header_schema.prepare(test_datafile) + header, header_is_valid, header_errors = header_schema.parse_and_validate(row) - assert header_is_valid - assert header_errors == [] + assert header_is_valid + assert header_errors == [] - -@pytest.mark.parametrize( - "header_line, is_valid, error", - [ - # Title error - ( - " 20204A06 TAN1ED", - False, - "Your file does not begin with a HEADER record.", - ), - # quarter error - ( - "HEADER20205A06 TAN1ED", - False, - "HEADER Item 5 (quarter): 5 is not in [1, 2, 3, 4].", - ), - # Type error - ( - "HEADER20204E06 TAN1ED", - False, - "HEADER Item 6 (type): E is not in [A, C, G, S].", - ), - # Fips error - ( - "HEADER20204A07 TAN1ED", - True, + @pytest.mark.parametrize( + "header_line, is_valid, error", + [ + # Title error + ( + " 20204A06 TAN1ED", + False, + "Your file does not begin with a HEADER record.", + ), + # quarter error + ( + "HEADER20205A06 TAN1ED", + False, + "HEADER Item 5 (quarter): 5 is not in [1, 2, 3, 4].", + ), + # Type error + ( + "HEADER20204E06 TAN1ED", + False, + "HEADER Item 6 (type): E is not in [A, C, G, S].", + ), + # Fips error + ( + "HEADER20204A07 TAN1ED", + True, + ( + "HEADER Item 1 (state fips): 07 is not in [00, 01, 02, 04, 05, 06, 08, 09, " + "10, 11, 12, 13, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, " + "30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 44, 45, 46, 47, 48, 49, " + "50, 51, 53, 54, 55, 56, 66, 72, 78]." + ), + ), + # Tribe error + ( + "HEADER20204A00 -1TAN1ED", + True, + "HEADER Item 3 (tribe code): -1 is not in range [0, 999].", + ), + # Program type error + ( + "HEADER20204A06 BAD1ED", + False, + "HEADER Item 7 (program type): BAD is not in [TAN, SSP].", + ), + # Edit error + ("HEADER20204A06 TAN3ED", False, "HEADER Item 8 (edit): 3 is not in [1, 2]."), + # Encryption error ( - "HEADER Item 1 (state fips): 07 is not in [00, 01, 02, 04, 05, 06, 08, 09, " - "10, 11, 12, 13, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, " - "30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 44, 45, 46, 47, 48, 49, " - "50, 51, 53, 54, 55, 56, 66, 72, 78]." + "HEADER20204A06 TAN1AD", + True, + "HEADER Item 9 (encryption): A is not in [ , E].", ), - ), - # Tribe error - ( - "HEADER20204A00 -1TAN1ED", - True, - "HEADER Item 3 (tribe code): -1 is not in range [0, 999].", - ), - # Program type error - ( - "HEADER20204A06 BAD1ED", - False, - "HEADER Item 7 (program type): BAD is not in [TAN, SSP].", - ), - # Edit error - ("HEADER20204A06 TAN3ED", False, "HEADER Item 8 (edit): 3 is not in [1, 2]."), - # Encryption error - ( - "HEADER20204A06 TAN1AD", - True, - "HEADER Item 9 (encryption): A is not in [ , E].", - ), - # Update error - ( - "HEADER20204A06 TAN1EA", - True, + # Update error ( - "HEADER Update Indicator must be set to D instead of A. Please review " - "Exporting Complete Data Using FTANF in the Knowledge Center." + "HEADER20204A06 TAN1EA", + True, + ( + "HEADER Update Indicator must be set to D instead of A. Please review " + "Exporting Complete Data Using FTANF in the Knowledge Center." + ), ), - ), - ], -) -@pytest.mark.django_db -def test_header_fields(test_datafile, header_line, is_valid, error): - """Test validate all header fields.""" - length = len(header_line) - row = RawRow( - data=header_line, - raw_len=length, - decoded_len=length, - row_num=1, - record_type="HEADER", + ], ) - header_schema = schema_defs.header - header_schema.prepare(test_datafile) - header, header_is_valid, header_errors = header_schema.parse_and_validate(row) + @pytest.mark.django_db + def test_header_fields(self, test_datafile, header_line, is_valid, error): + """Test validate all header fields.""" + length = len(header_line) + row = RawRow( + data=header_line, + raw_len=length, + decoded_len=length, + row_num=1, + record_type="HEADER", + ) + header_schema = schema_defs.header + header_schema.prepare(test_datafile) + header, header_is_valid, header_errors = header_schema.parse_and_validate(row) - assert is_valid == header_is_valid - if header_errors: - assert error == header_errors[0].error_message + assert is_valid == header_is_valid + if header_errors: + assert error == header_errors[0].error_message diff --git a/tdrs-backend/tdpservice/parsers/test/test_models.py b/tdrs-backend/tdpservice/parsers/test/test_models.py index 7dae5dc07..e9ba5a788 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_models.py +++ b/tdrs-backend/tdpservice/parsers/test/test_models.py @@ -6,20 +6,21 @@ from .factories import ParserErrorFactory -@pytest.fixture -def parser_error_instance(): - """Create a parser error instance.""" - return ParserErrorFactory.create() - - -@pytest.mark.django_db -def test_parser_error_instance(parser_error_instance): - """Test that the parser error instance is created.""" - assert isinstance(parser_error_instance, ParserError) - - -@pytest.mark.django_db -def test_parser_error_rpt_month_name(parser_error_instance): - """Test that the parser error instance is created.""" - parser_error_instance.rpt_month_year = 202001 - assert parser_error_instance.rpt_month_name == "January" +class TestParserErrorModel: + """Tests for parser error model.""" + + @pytest.fixture + def parser_error_instance(self): + """Create a parser error instance.""" + return ParserErrorFactory.create() + + @pytest.mark.django_db + def test_parser_error_instance(self, parser_error_instance): + """Test that the parser error instance is created.""" + assert isinstance(parser_error_instance, ParserError) + + @pytest.mark.django_db + def test_parser_error_rpt_month_name(self, parser_error_instance): + """Test that the parser error instance is created.""" + parser_error_instance.rpt_month_year = 202001 + assert parser_error_instance.rpt_month_name == "January" diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 543e7c840..2abb82e25 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -1,26 +1,19 @@ """Test the implementation of the parse_file method with realistic datafiles.""" import logging -import os - from django.conf import settings from django.db.models import Q as Query import pytest from tdpservice.parsers import aggregates, util -from tdpservice.parsers.factory import ParserFactory +from tdpservice.parsers.test.helpers import parse_datafile from tdpservice.parsers.models import ( DataFileSummary, ParserError, ParserErrorCategoryChoices, ) from tdpservice.search_indexes.models.fra import TANF_Exiter1 -from tdpservice.search_indexes.models.program_audit import ( - ProgramAudit_T1, - ProgramAudit_T2, - ProgramAudit_T3, -) from tdpservice.search_indexes.models.ssp import ( SSP_M1, SSP_M2, @@ -52,2622 +45,1768 @@ logger = logging.getLogger(__name__) settings.GENERATE_TRAILER_ERRORS = True - - # TODO: the name of this test doesn't make perfect sense anymore since it will always have errors now. # TODO: parametrize and merge with test_zero_filled_fips_code_file -@pytest.mark.django_db -def test_parse_small_correct_file(small_correct_file, dfs): - """Test parsing of small_correct_file.""" - small_correct_file.year = 2021 - small_correct_file.quarter = "Q1" - small_correct_file.save() - dfs.datafile = small_correct_file - - parser = ParserFactory.get_instance( - datafile=small_correct_file, - dfs=dfs, - section=small_correct_file.section, - program_type=small_correct_file.program_type, - ) - parser.parse_and_validate() - - errors = ParserError.objects.filter(file=small_correct_file).order_by("id") - assert errors.count() == 2 - assert errors.first().error_type == ParserErrorCategoryChoices.CASE_CONSISTENCY - - dfs.status = dfs.get_status() - dfs.case_aggregates = aggregates.case_aggregates_by_month(dfs.datafile, dfs.status) - assert dfs.case_aggregates == { - "rejected": 1, - "months": [ - { - "accepted_without_errors": "N/A", - "accepted_with_errors": "N/A", - "month": "Oct", - }, - { - "accepted_without_errors": "N/A", - "accepted_with_errors": "N/A", - "month": "Nov", - }, - { - "accepted_without_errors": "N/A", - "accepted_with_errors": "N/A", - "month": "Dec", - }, - ], - } - - assert dfs.get_status() == DataFileSummary.Status.REJECTED - assert TANF_T1.objects.count() == 0 - - -@pytest.mark.django_db -def test_parse_section_mismatch(small_correct_file, dfs): - """Test parsing of small_correct_file where the DataFile section doesn't match the rawfile section.""" - small_correct_file.section = "Closed Case Data" - small_correct_file.save() - - dfs.datafile = small_correct_file - - parser = ParserFactory.get_instance( - datafile=small_correct_file, - dfs=dfs, - section=small_correct_file.section, - program_type=small_correct_file.program_type, - ) - parser.parse_and_validate() - - dfs.status = dfs.get_status() - assert dfs.status == DataFileSummary.Status.REJECTED - parser_errors = ParserError.objects.filter(file=small_correct_file) - dfs.case_aggregates = aggregates.case_aggregates_by_month(dfs.datafile, dfs.status) - assert dfs.case_aggregates == { - "rejected": 1, - "months": [ - { - "accepted_without_errors": "N/A", - "accepted_with_errors": "N/A", - "month": "Oct", - }, - { - "accepted_without_errors": "N/A", - "accepted_with_errors": "N/A", - "month": "Nov", - }, - { - "accepted_without_errors": "N/A", - "accepted_with_errors": "N/A", - "month": "Dec", - }, - ], - } - assert parser_errors.count() == 1 - - err = parser_errors.first() - - assert err.row_number == 1 - assert err.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert ( - err.error_message - == "Data does not match the expected layout for Closed Case Data." - ) - assert err.content_type is None - assert err.object_id is None - - -@pytest.mark.django_db -def test_parse_wrong_program_type(small_correct_file, dfs): - """Test parsing of small_correct_file where the DataFile program type doesn't match the rawfile.""" - small_correct_file.section = "SSP Active Case Data" - small_correct_file.save() - - dfs.datafile = small_correct_file - dfs.save() - parser = ParserFactory.get_instance( - datafile=small_correct_file, - dfs=dfs, - section=small_correct_file.section, - program_type=small_correct_file.program_type, - ) - parser.parse_and_validate() - assert dfs.get_status() == DataFileSummary.Status.REJECTED - - parser_errors = ParserError.objects.filter(file=small_correct_file) - assert parser_errors.count() == 1 - - err = parser_errors.first() - - assert err.row_number == 1 - assert err.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert ( - err.error_message - == "Data does not match the expected layout for SSP Active Case Data." - ) - assert err.content_type is None - assert err.object_id is None - - -@pytest.mark.django_db -def test_parse_big_file(big_file, dfs): - """Test parsing of ADS.E2J.FTP1.TS06.""" - expected_t1_record_count = 815 - expected_t2_record_count = 882 - expected_t3_record_count = 1376 - - big_file.year = 2022 - big_file.quarter = "Q1" - - dfs.datafile = big_file - - parser = ParserFactory.get_instance( - datafile=big_file, - dfs=dfs, - section=big_file.section, - program_type=big_file.program_type, - ) - parser.parse_and_validate() - - dfs.status = dfs.get_status() - assert dfs.status == DataFileSummary.Status.ACCEPTED_WITH_ERRORS - - dfs.case_aggregates = aggregates.case_aggregates_by_month(dfs.datafile, dfs.status) - assert dfs.case_aggregates == { - "months": [ - { - "month": "Oct", - "accepted_without_errors": 11, - "accepted_with_errors": 259, - }, - { - "month": "Nov", - "accepted_without_errors": 12, - "accepted_with_errors": 261, - }, - { - "month": "Dec", - "accepted_without_errors": 15, - "accepted_with_errors": 257, - }, - ], - "rejected": 0, - } - - assert TANF_T1.objects.count() == expected_t1_record_count - assert TANF_T2.objects.count() == expected_t2_record_count - assert TANF_T3.objects.count() == expected_t3_record_count - - -@pytest.mark.django_db -def test_parse_bad_test_file(bad_test_file, dfs): - """Test parsing of bad_TANF_S2.""" - parser = ParserFactory.get_instance( - datafile=bad_test_file, - dfs=dfs, - section=bad_test_file.section, - program_type=bad_test_file.program_type, - ) - parser.parse_and_validate() - - parser_errors = ParserError.objects.filter(file=bad_test_file) - assert parser_errors.count() == 1 - - err = parser_errors.first() - - assert err.row_number == 1 - assert err.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert err.error_message == "HEADER: record length is 24 characters but must be 23." - assert err.content_type is None - assert err.object_id is None - - -@pytest.mark.django_db -def test_parse_bad_file_missing_header(bad_file_missing_header, dfs): - """Test parsing of bad_missing_header.""" - parser = ParserFactory.get_instance( - datafile=bad_file_missing_header, - dfs=dfs, - section=bad_file_missing_header.section, - program_type=bad_file_missing_header.program_type, - ) - parser.parse_and_validate() - dfs.datafile = bad_file_missing_header - assert dfs.get_status() == DataFileSummary.Status.REJECTED - - parser_errors = ParserError.objects.filter(file=bad_file_missing_header).order_by( - "created_at" - ) - - assert parser_errors.count() == 2 - err = parser_errors.first() - assert err.row_number == 1 - assert err.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert err.error_message == "HEADER: record length is 14 characters but must be 23." - assert err.content_type is None - assert err.object_id is None - - -@pytest.mark.django_db -def test_parse_bad_file_multiple_headers(bad_file_multiple_headers, dfs): - """Test parsing of bad_two_headers.""" - bad_file_multiple_headers.year = 2024 - bad_file_multiple_headers.quarter = "Q1" - bad_file_multiple_headers.save() - parser = ParserFactory.get_instance( - datafile=bad_file_multiple_headers, - dfs=dfs, - section=bad_file_multiple_headers.section, - program_type=bad_file_multiple_headers.program_type, - ) - parser.parse_and_validate() - dfs.datafile = bad_file_multiple_headers - assert dfs.get_status() == DataFileSummary.Status.REJECTED - - parser_errors = ParserError.objects.filter(file=bad_file_multiple_headers) - assert parser_errors.count() == 1 - - err = parser_errors.first() - - assert err.row_number == 9 - assert err.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert err.error_message == "Multiple headers found." - assert err.content_type is None - assert err.object_id is None - - -@pytest.mark.django_db -def test_parse_big_bad_test_file(big_bad_test_file, dfs): - """Test parsing of bad_TANF_S1.""" - big_bad_test_file.year = 2022 - big_bad_test_file.quarter = "Q1" - parser = ParserFactory.get_instance( - datafile=big_bad_test_file, - dfs=dfs, - section=big_bad_test_file.section, - program_type=big_bad_test_file.program_type, - ) - parser.parse_and_validate() - - parser_errors = ParserError.objects.filter(file=big_bad_test_file) - assert parser_errors.count() == 1 - - err = parser_errors.first() - - assert err.row_number == 3679 - assert err.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert err.error_message == "Multiple headers found." - assert err.content_type is None - assert err.object_id is None - - -@pytest.mark.django_db -def test_parse_bad_trailer_file(bad_trailer_file, dfs): - """Test parsing bad_trailer_1.""" - bad_trailer_file.year = 2021 - bad_trailer_file.quarter = "Q1" - dfs.datafile = bad_trailer_file - - parser = ParserFactory.get_instance( - datafile=bad_trailer_file, - dfs=dfs, - section=bad_trailer_file.section, - program_type=bad_trailer_file.program_type, - ) - parser.parse_and_validate() - - parser_errors = ParserError.objects.filter(file=bad_trailer_file) - assert parser_errors.count() == 5 - - trailer_error = parser_errors.get(row_number=3) - assert trailer_error.error_type == ParserErrorCategoryChoices.RECORD_PRE_CHECK - assert ( - trailer_error.error_message - == "TRAILER: record length is 11 characters but must be 23." - ) - assert trailer_error.content_type is None - assert trailer_error.object_id is None - - # reporting month/year test - row_errors = parser_errors.filter(row_number=2) - row_errors_list = [] - for row_error in row_errors: - row_errors_list.append(row_error) - assert row_error.error_type == ParserErrorCategoryChoices.RECORD_PRE_CHECK - assert trailer_error.error_message in [ - "TRAILER: record length is 11 characters but must be 23.", - "T1: Reporting month year None does not match file reporting year:2021, quarter:Q1.", - ] - assert row_error.content_type is None - assert row_error.object_id is None - - row_errors = list(parser_errors.filter(row_number=2).order_by("id")) - length_error = row_errors[0] - assert length_error.error_type == ParserErrorCategoryChoices.RECORD_PRE_CHECK - assert ( - length_error.error_message - == "T1: record length should be at least 117 characters but it is 7 characters." - ) - assert length_error.content_type is None - assert length_error.object_id is None - - -@pytest.mark.django_db() -def test_parse_bad_trailer_file2(bad_trailer_file_2, dfs): - """Test parsing bad_trailer_2.""" - dfs.datafile = bad_trailer_file_2 - dfs.save() - - bad_trailer_file_2.year = 2021 - bad_trailer_file_2.quarter = "Q1" - parser = ParserFactory.get_instance( - datafile=bad_trailer_file_2, - dfs=dfs, - section=bad_trailer_file_2.section, - program_type=bad_trailer_file_2.program_type, - ) - parser.parse_and_validate() - - parser_errors = ParserError.objects.filter(file=bad_trailer_file_2) - assert parser_errors.count() == 9 - - parser_errors = parser_errors.exclude( - error_type=ParserErrorCategoryChoices.CASE_CONSISTENCY - ) - trailer_errors = list(parser_errors.filter(row_number=3).order_by("id")) - - trailer_error_1 = trailer_errors[0] - assert trailer_error_1.error_type == ParserErrorCategoryChoices.RECORD_PRE_CHECK - assert ( - trailer_error_1.error_message - == "TRAILER: record length is 7 characters but must be 23." - ) - assert trailer_error_1.content_type is None - assert trailer_error_1.object_id is None - - trailer_error_2 = trailer_errors[1] - assert trailer_error_2.error_type == ParserErrorCategoryChoices.RECORD_PRE_CHECK - assert ( - trailer_error_2.error_message == "Your file does not end with a TRAILER record." - ) - assert trailer_error_2.content_type is None - assert trailer_error_2.object_id is None - - row_2_errors = parser_errors.filter(row_number=2).order_by("id") - row_2_error = row_2_errors.first() - assert row_2_error.error_type == ParserErrorCategoryChoices.FIELD_VALUE - assert row_2_error.error_message == ( - "T1 Item 13 (Receives Subsidized Housing): 3 is not in range [1, 2]." - ) - - # catch-rpt-month-year-mismatches - row_3_errors = parser_errors.filter(row_number=3) - row_3_error_list = [] - - for row_3_error in row_3_errors: - row_3_error_list.append(row_3_error) - assert row_3_error.error_type == ParserErrorCategoryChoices.RECORD_PRE_CHECK - assert row_3_error.error_message in { - "T1: record length should be at least 117 characters but it is 7 characters.", - "T1: Reporting month year None does not match file reporting year:2021, quarter:Q1.", - "TRAILER: record length is 7 characters but must be 23.", - "T1: Case number T1trash cannot contain blanks.", - "Your file does not end with a TRAILER record.", +class TestParse: + """Tests for parse and validation flows.""" + + @pytest.fixture + def parsed_small_correct_file(self, small_correct_file, dfs): + """Return parsed small_correct_file and its DataFileSummary.""" + small_correct_file.year = 2021 + small_correct_file.quarter = "Q1" + small_correct_file.save() + + parse_datafile(dfs, small_correct_file) + + return small_correct_file, dfs + + @pytest.fixture + def parsed_bad_trailer_file(self, bad_trailer_file, dfs): + """Return parsed bad_trailer_file and its errors.""" + bad_trailer_file.year = 2021 + bad_trailer_file.quarter = "Q1" + + parse_datafile(dfs, bad_trailer_file) + + parser_errors = ParserError.objects.filter(file=bad_trailer_file) + return bad_trailer_file, dfs, parser_errors + + @pytest.fixture + def parsed_bad_trailer_file_2(self, bad_trailer_file_2, dfs): + """Return parsed bad_trailer_file_2 and its errors.""" + dfs.datafile = bad_trailer_file_2 + dfs.save() + + bad_trailer_file_2.year = 2021 + bad_trailer_file_2.quarter = "Q1" + + parse_datafile(dfs, bad_trailer_file_2) + + parser_errors = ParserError.objects.filter(file=bad_trailer_file_2) + return bad_trailer_file_2, dfs, parser_errors + + @pytest.mark.django_db + def test_small_correct_file_case_consistency_error( + self, parsed_small_correct_file + ): + """Test case consistency errors are recorded for small_correct_file.""" + datafile, _dfs = parsed_small_correct_file + errors = ParserError.objects.filter(file=datafile).order_by("id") + assert errors.count() == 2 + assert errors.first().error_type == ParserErrorCategoryChoices.CASE_CONSISTENCY + + @pytest.mark.django_db + def test_small_correct_file_case_aggregates_rejected( + self, parsed_small_correct_file + ): + """Test case aggregates for rejected small_correct_file.""" + _datafile, dfs = parsed_small_correct_file + dfs.status = dfs.get_status() + dfs.case_aggregates = aggregates.case_aggregates_by_month(dfs.datafile, dfs.status) + assert dfs.case_aggregates == { + "rejected": 1, + "months": [ + { + "accepted_without_errors": "N/A", + "accepted_with_errors": "N/A", + "month": "Oct", + }, + { + "accepted_without_errors": "N/A", + "accepted_with_errors": "N/A", + "month": "Nov", + }, + { + "accepted_without_errors": "N/A", + "accepted_with_errors": "N/A", + "month": "Dec", + }, + ], } - assert row_3_error.content_type is None - assert row_3_error.object_id is None - - # case number validators - row_3_errors = [trailer_errors[2], trailer_errors[3]] - length_error = row_3_errors[0] - assert length_error.error_type == ParserErrorCategoryChoices.RECORD_PRE_CHECK - assert ( - length_error.error_message - == "T1: record length should be at least 117 characters but it is 7 characters." - ) - assert length_error.content_type is None - assert length_error.object_id is None - - trailer_error_3 = trailer_errors[3] - assert trailer_error_3.error_type == ParserErrorCategoryChoices.RECORD_PRE_CHECK - assert ( - trailer_error_3.error_message - == "T1: Case number T1trash cannot contain blanks." - ) - assert trailer_error_3.content_type is None - assert trailer_error_3.object_id is None - - trailer_error_4 = trailer_errors[4] - assert trailer_error_4.error_type == ParserErrorCategoryChoices.RECORD_PRE_CHECK - assert trailer_error_4.error_message == ( - "T1: Reporting month year None does not " - "match file reporting year:2021, quarter:Q1." - ) - assert trailer_error_4.content_type is None - assert trailer_error_4.object_id is None - - -@pytest.mark.django_db -def test_parse_empty_file(empty_file, dfs): - """Test parsing of empty_file.""" - dfs.datafile = empty_file - dfs.save() - parser = ParserFactory.get_instance( - datafile=empty_file, - dfs=dfs, - section=empty_file.section, - program_type=empty_file.program_type, - ) - parser.parse_and_validate() - - dfs.status = dfs.get_status() - dfs.case_aggregates = aggregates.case_aggregates_by_month(empty_file, dfs.status) - - assert dfs.status == DataFileSummary.Status.REJECTED - assert dfs.case_aggregates == { - "rejected": 1, - "months": [ - { - "accepted_without_errors": "N/A", - "accepted_with_errors": "N/A", - "month": "Oct", - }, - { - "accepted_without_errors": "N/A", - "accepted_with_errors": "N/A", - "month": "Nov", - }, - { - "accepted_without_errors": "N/A", - "accepted_with_errors": "N/A", - "month": "Dec", - }, + assert dfs.get_status() == DataFileSummary.Status.REJECTED + + @pytest.mark.django_db + def test_small_correct_file_no_records_created(self, parsed_small_correct_file): + """Test that small_correct_file does not create records when rejected.""" + _datafile, _dfs = parsed_small_correct_file + assert TANF_T1.objects.count() == 0 + + @pytest.mark.django_db + @pytest.mark.parametrize( + "section, expected_message, expected_aggregates, save_dfs", + [ + ( + "Closed Case Data", + "Data does not match the expected layout for Closed Case Data.", + { + "rejected": 1, + "months": [ + { + "accepted_without_errors": "N/A", + "accepted_with_errors": "N/A", + "month": "Oct", + }, + { + "accepted_without_errors": "N/A", + "accepted_with_errors": "N/A", + "month": "Nov", + }, + { + "accepted_without_errors": "N/A", + "accepted_with_errors": "N/A", + "month": "Dec", + }, + ], + }, + False, + ), + ( + "SSP Active Case Data", + "Data does not match the expected layout for " + "SSP Active Case Data.", + None, + True, + ), ], - } - - parser_errors = ParserError.objects.filter(file=empty_file).order_by("id") - - assert parser_errors.count() == 2 - - err = parser_errors.first() - - assert err.row_number == 1 - assert err.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert err.error_message == "HEADER: record length is 0 characters but must be 23." - assert err.content_type is None - assert err.object_id is None - - -@pytest.mark.django_db -def test_parse_small_ssp_section1_datafile(small_ssp_section1_datafile, dfs): - """Test parsing small_ssp_section1_datafile.""" - small_ssp_section1_datafile.year = 2024 - small_ssp_section1_datafile.quarter = "Q1" - - expected_m1_record_count = 5 - expected_m2_record_count = 6 - expected_m3_record_count = 8 - - dfs.datafile = small_ssp_section1_datafile - dfs.save() - parser = ParserFactory.get_instance( - datafile=small_ssp_section1_datafile, - dfs=dfs, - section=small_ssp_section1_datafile.section, - program_type=small_ssp_section1_datafile.program_type, - ) - parser.parse_and_validate() - - parser_errors = ParserError.objects.filter(file=small_ssp_section1_datafile) - dfs.status = dfs.get_status() - assert dfs.status == DataFileSummary.Status.PARTIALLY_ACCEPTED - dfs.case_aggregates = aggregates.case_aggregates_by_month(dfs.datafile, dfs.status) - - assert dfs.case_aggregates["rejected"] == 1 - for month in dfs.case_aggregates["months"]: - if month["month"] == "Oct": - assert month["accepted_without_errors"] == 0 - assert month["accepted_with_errors"] == 5 - else: - assert month["accepted_without_errors"] == 0 - assert month["accepted_with_errors"] == 0 - - parser_errors = ParserError.objects.filter(file=small_ssp_section1_datafile) - assert parser_errors.count() == 9 - assert SSP_M1.objects.count() == expected_m1_record_count - assert SSP_M2.objects.count() == expected_m2_record_count - assert SSP_M3.objects.count() == expected_m3_record_count - - -@pytest.mark.django_db() -def test_parse_ssp_section1_datafile(ssp_section1_datafile, dfs): - """Test parsing ssp_section1_datafile.""" - ssp_section1_datafile.year = 2019 - ssp_section1_datafile.quarter = "Q1" - - expected_m1_record_count = 818 - expected_m2_record_count = 989 - expected_m3_record_count = 1748 - - dfs.datafile = ssp_section1_datafile - dfs.save() - - parser = ParserFactory.get_instance( - datafile=ssp_section1_datafile, - dfs=dfs, - section=ssp_section1_datafile.section, - program_type=ssp_section1_datafile.program_type, - ) - parser.parse_and_validate() - - parser_errors = ParserError.objects.filter(file=ssp_section1_datafile).order_by( - "row_number" - ) - - err = parser_errors.first() - - assert err.row_number == 2 - assert err.error_type == ParserErrorCategoryChoices.FIELD_VALUE - assert err.error_message == ( - "M1 Item 11 (Receives Subsidized Housing): 3 is not in range [1, 2]." - ) - assert err.content_type is not None - assert err.object_id is not None - - cat4_errors = parser_errors.filter( - error_type=ParserErrorCategoryChoices.CASE_CONSISTENCY - ).order_by("id") - - assert cat4_errors.count() == 3 - assert ( - cat4_errors[0].error_message - == "Duplicate record detected with record type M3 at line 453. " - + "Record is a duplicate of the record at line number 452." - ) - assert ( - cat4_errors[1].error_message - == "Duplicate record detected with record type M3 at line 3273. " - + "Record is a duplicate of the record at line number 3272." - ) - assert ( - cat4_errors[2].error_message - == "Partial duplicate record detected with record type M3 at line 3275. " - + "Record is a partial duplicate of the record at line number 3274. Duplicated fields " - + "causing error: Item 0 (Record Type), Item 3 (Reporting Year and Month), Item 5 (Case Number), " - + "Item 60 (Family Affiliation), Item 61 (Date of Birth), and Item 62 (Social Security Number)." ) - - assert parser_errors.count() == 31726 - - assert SSP_M1.objects.count() == expected_m1_record_count - assert SSP_M2.objects.count() == expected_m2_record_count - assert SSP_M3.objects.count() == expected_m3_record_count - - -@pytest.mark.django_db -def test_parse_tanf_section1_datafile(small_tanf_section1_datafile, dfs): - """Test parsing of small_tanf_section1_datafile and validate T2 model data.""" - small_tanf_section1_datafile.year = 2021 - small_tanf_section1_datafile.quarter = "Q1" - dfs.datafile = small_tanf_section1_datafile - - parser = ParserFactory.get_instance( - datafile=small_tanf_section1_datafile, - dfs=dfs, - section=small_tanf_section1_datafile.section, - program_type=small_tanf_section1_datafile.program_type, - ) - parser.parse_and_validate() - - dfs.status = dfs.get_status() - assert dfs.status == DataFileSummary.Status.ACCEPTED_WITH_ERRORS - dfs.case_aggregates = aggregates.case_aggregates_by_month(dfs.datafile, dfs.status) - assert dfs.case_aggregates == { - "months": [ - {"month": "Oct", "accepted_without_errors": 1, "accepted_with_errors": 4}, - {"month": "Nov", "accepted_without_errors": 0, "accepted_with_errors": 0}, - {"month": "Dec", "accepted_without_errors": 0, "accepted_with_errors": 0}, + def test_parse_section_mismatch_variants( + self, + small_correct_file, + dfs, + section, + expected_message, + expected_aggregates, + save_dfs, + ): + """Test parsing when file metadata does not match the raw data layout.""" + small_correct_file.section = section + small_correct_file.save() + + dfs.datafile = small_correct_file + if save_dfs: + dfs.save() + + parse_datafile(dfs, small_correct_file) + + dfs.status = dfs.get_status() + assert dfs.status == DataFileSummary.Status.REJECTED + parser_errors = ParserError.objects.filter(file=small_correct_file) + assert parser_errors.count() == 1 + + if expected_aggregates is not None: + dfs.case_aggregates = aggregates.case_aggregates_by_month( + dfs.datafile, dfs.status + ) + assert dfs.case_aggregates == expected_aggregates + + err = parser_errors.first() + assert err.row_number == 1 + assert err.error_type == ParserErrorCategoryChoices.PRE_CHECK + assert err.error_message == expected_message + assert err.content_type is None + assert err.object_id is None + + @pytest.mark.django_db + @pytest.mark.parametrize( + "fixture_name, updates, expected", + [ + ( + "bad_test_file", + {}, + { + "count": 1, + "row_number": 1, + "error_message": ( + "HEADER: record length is 24 characters but must be 23." + ), + }, + ), + ( + "bad_file_missing_header", + {}, + { + "count": 2, + "row_number": 1, + "error_message": ( + "HEADER: record length is 14 characters but must be 23." + ), + "status": DataFileSummary.Status.REJECTED, + }, + ), + ( + "bad_file_multiple_headers", + {"year": 2024, "quarter": "Q1"}, + { + "count": 1, + "row_number": 9, + "error_message": "Multiple headers found.", + "status": DataFileSummary.Status.REJECTED, + }, + ), + ( + "big_bad_test_file", + {"year": 2022, "quarter": "Q1"}, + { + "count": 1, + "row_number": 3679, + "error_message": "Multiple headers found.", + }, + ), ], - "rejected": 0, - } - - assert TANF_T2.objects.count() == 5 - - t2_models = TANF_T2.objects.all() - - t2 = t2_models[0] - assert t2.RPT_MONTH_YEAR == 202010 - assert t2.CASE_NUMBER == "11111111112" - assert t2.FAMILY_AFFILIATION == 1 - assert t2.OTHER_UNEARNED_INCOME == "0291" - - t2_2 = t2_models[1] - assert t2_2.RPT_MONTH_YEAR == 202010 - assert t2_2.CASE_NUMBER == "11111111115" - assert t2_2.FAMILY_AFFILIATION == 2 - assert t2_2.OTHER_UNEARNED_INCOME == "0000" - - -@pytest.mark.django_db() -def test_parse_tanf_section1_datafile_obj_counts(small_tanf_section1_datafile, dfs): - """Test parsing of small_tanf_section1_datafile in general.""" - small_tanf_section1_datafile.year = 2021 - small_tanf_section1_datafile.quarter = "Q1" - - dfs.datafile = small_tanf_section1_datafile - dfs.save() - - parser = ParserFactory.get_instance( - datafile=small_tanf_section1_datafile, - dfs=dfs, - section=small_tanf_section1_datafile.section, - program_type=small_tanf_section1_datafile.program_type, ) - parser.parse_and_validate() - - assert TANF_T1.objects.count() == 5 - assert TANF_T2.objects.count() == 5 - assert TANF_T3.objects.count() == 6 - - -@pytest.mark.django_db() -def test_parse_tanf_section1_datafile_t3s(small_tanf_section1_datafile, dfs): - """Test parsing of small_tanf_section1_datafile and validate T3 model data.""" - small_tanf_section1_datafile.year = 2021 - small_tanf_section1_datafile.quarter = "Q1" - - dfs.datafile = small_tanf_section1_datafile - dfs.save() - - parser = ParserFactory.get_instance( - datafile=small_tanf_section1_datafile, - dfs=dfs, - section=small_tanf_section1_datafile.section, - program_type=small_tanf_section1_datafile.program_type, - ) - parser.parse_and_validate() - - assert TANF_T3.objects.count() == 6 - - t3_models = TANF_T3.objects.all() - t3_1 = t3_models[0] - assert t3_1.RPT_MONTH_YEAR == 202010 - assert t3_1.CASE_NUMBER == "11111111112" - assert t3_1.FAMILY_AFFILIATION == 1 - assert t3_1.SEX == 2 - assert t3_1.EDUCATION_LEVEL == "98" - - t3_5 = t3_models[4] - assert t3_5.RPT_MONTH_YEAR == 202010 - assert t3_5.CASE_NUMBER == "11111111151" - assert t3_5.FAMILY_AFFILIATION == 1 - assert t3_5.SEX == 1 - assert t3_5.EDUCATION_LEVEL == "98" - - -@pytest.mark.django_db() -@pytest.mark.skip(reason="long runtime") # big_files -def test_parse_super_big_s1_file(super_big_s1_file, dfs): - """Test parsing of super_big_s1_file and validate all T1/T2/T3 records are created.""" - super_big_s1_file.year = 2023 - super_big_s1_file.quarter = "Q2" - super_big_s1_file.save() - - dfs.datafile = super_big_s1_file - dfs.save() - - parser = ParserFactory.get_instance( - datafile=super_big_s1_file, - dfs=dfs, - section=super_big_s1_file.section, - program_type=super_big_s1_file.program_type, - ) - parser.parse_and_validate() - expected_t1_record_count = 96607 - expected_t2_record_count = 112753 - expected_t3_record_count = 172525 - - assert TANF_T1.objects.count() == expected_t1_record_count - assert TANF_T2.objects.count() == expected_t2_record_count - assert TANF_T3.objects.count() == expected_t3_record_count - - -@pytest.mark.django_db() -def test_parse_big_s1_file_with_rollback(big_s1_rollback_file, dfs): - """Test parsing of big_s1_rollback_file. - - Validate all T1/T2/T3 records are not created due to multiple headers. - """ - big_s1_rollback_file.year = 2023 - big_s1_rollback_file.quarter = "Q2" - big_s1_rollback_file.save() - - dfs.datafile = big_s1_rollback_file - dfs.save() - - parser = ParserFactory.get_instance( - datafile=big_s1_rollback_file, - dfs=dfs, - section=big_s1_rollback_file.section, - program_type=big_s1_rollback_file.program_type, - ) - parser.parse_and_validate() - - parser_errors = ParserError.objects.filter(file=big_s1_rollback_file) - assert parser_errors.count() == 1 - - err = parser_errors.first() - - assert err.row_number == 13609 - assert err.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert err.error_message == "Multiple headers found." - assert err.content_type is None - assert err.object_id is None - - assert TANF_T1.objects.count() == 0 - assert TANF_T2.objects.count() == 0 - assert TANF_T3.objects.count() == 0 - - -@pytest.mark.django_db -def test_parse_bad_tfs1_missing_required(bad_tanf_s1__row_missing_required_field, dfs): - """Test parsing a bad TANF Section 1 submission where a row is missing required data.""" - bad_tanf_s1__row_missing_required_field.year = 2021 - bad_tanf_s1__row_missing_required_field.quarter = "Q1" + def test_parse_precheck_header_errors(self, request, fixture_name, updates, expected, dfs): + """Test parsing failures triggered by header/pre-check validation.""" + datafile = request.getfixturevalue(fixture_name) + for field, value in updates.items(): + setattr(datafile, field, value) + if updates: + datafile.save() + + parse_datafile(dfs, datafile) + + if expected.get("status"): + assert dfs.get_status() == expected["status"] + + parser_errors = ParserError.objects.filter(file=datafile).order_by("id") + assert parser_errors.count() == expected["count"] + + err = parser_errors.first() + assert err.row_number == expected["row_number"] + assert err.error_type == ParserErrorCategoryChoices.PRE_CHECK + assert err.error_message == expected["error_message"] + assert err.content_type is None + assert err.object_id is None + + @pytest.mark.django_db + def test_bad_trailer_file_trailer_error(self, parsed_bad_trailer_file): + """Test trailer errors for bad_trailer_1.""" + _datafile, _dfs, parser_errors = parsed_bad_trailer_file + assert parser_errors.count() == 5 + + trailer_error = parser_errors.get(row_number=3) + assert trailer_error.error_type == ParserErrorCategoryChoices.RECORD_PRE_CHECK + assert ( + trailer_error.error_message + == "TRAILER: record length is 11 characters but must be 23." + ) + assert trailer_error.content_type is None + assert trailer_error.object_id is None + + @pytest.mark.django_db + def test_bad_trailer_file_row_errors(self, parsed_bad_trailer_file): + """Test row-level errors for bad_trailer_1.""" + _datafile, _dfs, parser_errors = parsed_bad_trailer_file + + # reporting month/year test + row_errors = parser_errors.filter(row_number=2) + row_errors_list = [] + for row_error in row_errors: + row_errors_list.append(row_error) + assert row_error.error_type == ParserErrorCategoryChoices.RECORD_PRE_CHECK + assert row_error.error_message in [ + "TRAILER: record length is 11 characters but must be 23.", + "T1: Case number T1trash cannot contain blanks.", + "T1: record length should be at least 117 characters but it is 7 characters.", + "T1: Reporting month year None does not match file reporting year:2021, quarter:Q1.", + ] + assert row_error.content_type is None + assert row_error.object_id is None + + row_errors = list(parser_errors.filter(row_number=2).order_by("id")) + length_error = row_errors[0] + assert length_error.error_type == ParserErrorCategoryChoices.RECORD_PRE_CHECK + assert ( + length_error.error_message + == "T1: record length should be at least 117 characters but it is 7 characters." + ) + assert length_error.content_type is None + assert length_error.object_id is None - dfs.datafile = bad_tanf_s1__row_missing_required_field - dfs.save() + @pytest.mark.django_db + def test_bad_trailer_file2_trailer_errors(self, parsed_bad_trailer_file_2): + """Test trailer errors for bad_trailer_2.""" + _datafile, _dfs, parser_errors = parsed_bad_trailer_file_2 + assert parser_errors.count() == 9 - parser = ParserFactory.get_instance( - datafile=bad_tanf_s1__row_missing_required_field, - dfs=dfs, - section=bad_tanf_s1__row_missing_required_field.section, - program_type=bad_tanf_s1__row_missing_required_field.program_type, - ) - parser.parse_and_validate() - - assert dfs.get_status() == DataFileSummary.Status.REJECTED + parser_errors = parser_errors.exclude( + error_type=ParserErrorCategoryChoices.CASE_CONSISTENCY + ) + trailer_errors = list(parser_errors.filter(row_number=3).order_by("id")) - parser_errors = ParserError.objects.filter( - file=bad_tanf_s1__row_missing_required_field - ) + trailer_error_1 = trailer_errors[0] + assert trailer_error_1.error_type == ParserErrorCategoryChoices.RECORD_PRE_CHECK + assert ( + trailer_error_1.error_message + == "TRAILER: record length is 7 characters but must be 23." + ) + assert trailer_error_1.content_type is None + assert trailer_error_1.object_id is None + + trailer_error_2 = trailer_errors[1] + assert trailer_error_2.error_type == ParserErrorCategoryChoices.RECORD_PRE_CHECK + assert ( + trailer_error_2.error_message + == "Your file does not end with a TRAILER record." + ) + assert trailer_error_2.content_type is None + assert trailer_error_2.object_id is None - assert parser_errors.count() == 5 - - error_message = "T1: Reporting month year None does not match file reporting year:2021, quarter:Q1." - row_2_error = parser_errors.get(row_number=2, error_message=error_message) - assert row_2_error.error_type == ParserErrorCategoryChoices.RECORD_PRE_CHECK - assert row_2_error.error_message == error_message - - error_message = "T2: Reporting month year None does not match file reporting year:2021, quarter:Q1." - row_3_error = parser_errors.get(row_number=3, error_message=error_message) - assert row_3_error.error_type == ParserErrorCategoryChoices.RECORD_PRE_CHECK - assert row_3_error.error_message == error_message - - error_message = "T3: Reporting month year None does not match file reporting year:2021, quarter:Q1." - row_4_error = parser_errors.get(row_number=4, error_message=error_message) - assert row_4_error.error_type == ParserErrorCategoryChoices.RECORD_PRE_CHECK - assert row_4_error.error_message == error_message - - error_message = "Unknown Record_Type was found." - row_5_error = parser_errors.get(row_number=5, error_message=error_message) - assert row_5_error.error_type == ParserErrorCategoryChoices.RECORD_PRE_CHECK - assert row_5_error.error_message == error_message - assert row_5_error.content_type is None - assert row_5_error.object_id is None - - -@pytest.mark.django_db() -def test_parse_bad_ssp_s1_missing_required(bad_ssp_s1__row_missing_required_field, dfs): - """Test parsing a bad TANF Section 1 submission where a row is missing required data.""" - bad_ssp_s1__row_missing_required_field.year = 2019 - bad_ssp_s1__row_missing_required_field.quarter = "Q1" - - dfs.datafile = bad_ssp_s1__row_missing_required_field - dfs.save() - - parser = ParserFactory.get_instance( - datafile=bad_ssp_s1__row_missing_required_field, - dfs=dfs, - section=bad_ssp_s1__row_missing_required_field.section, - program_type=bad_ssp_s1__row_missing_required_field.program_type, - ) - parser.parse_and_validate() + @pytest.mark.django_db + def test_bad_trailer_file2_row_2_error(self, parsed_bad_trailer_file_2): + """Test row 2 validation error for bad_trailer_2.""" + _datafile, _dfs, parser_errors = parsed_bad_trailer_file_2 - parser_errors = ParserError.objects.filter( - file=bad_ssp_s1__row_missing_required_field - ) - assert parser_errors.count() == 6 + parser_errors = parser_errors.exclude( + error_type=ParserErrorCategoryChoices.CASE_CONSISTENCY + ) + row_2_errors = parser_errors.filter(row_number=2).order_by("id") + row_2_error = row_2_errors.first() + assert row_2_error.error_type == ParserErrorCategoryChoices.FIELD_VALUE + assert row_2_error.error_message == ( + "T1 Item 13 (Receives Subsidized Housing): 3 is not in range [1, 2]." + ) - row_2_error = parser_errors.get( - row_number=2, - error_message__contains="Reporting month year None does not match file reporting year:2019, quarter:Q1.", - ) - assert row_2_error.error_type == ParserErrorCategoryChoices.RECORD_PRE_CHECK + @pytest.mark.django_db + def test_bad_trailer_file2_row_3_errors(self, parsed_bad_trailer_file_2): + """Test row 3 errors and case number validation for bad_trailer_2.""" + _datafile, _dfs, parser_errors = parsed_bad_trailer_file_2 - row_3_error = parser_errors.get( - row_number=3, - error_message__contains="Reporting month year None does not match file reporting year:2019, quarter:Q1.", - ) - assert row_3_error.error_type == ParserErrorCategoryChoices.RECORD_PRE_CHECK + parser_errors = parser_errors.exclude( + error_type=ParserErrorCategoryChoices.CASE_CONSISTENCY + ) + trailer_errors = list(parser_errors.filter(row_number=3).order_by("id")) + + # catch-rpt-month-year-mismatches + row_3_errors = parser_errors.filter(row_number=3) + row_3_error_list = [] + + for row_3_error in row_3_errors: + row_3_error_list.append(row_3_error) + assert row_3_error.error_type == ParserErrorCategoryChoices.RECORD_PRE_CHECK + assert row_3_error.error_message in { + "T1: record length should be at least 117 characters but it is 7 characters.", + "T1: Reporting month year None does not match file reporting year:2021, quarter:Q1.", + "TRAILER: record length is 7 characters but must be 23.", + "T1: Case number T1trash cannot contain blanks.", + "Your file does not end with a TRAILER record.", + } + assert row_3_error.content_type is None + assert row_3_error.object_id is None + + # case number validators + row_3_errors = [trailer_errors[2], trailer_errors[3]] + length_error = row_3_errors[0] + assert length_error.error_type == ParserErrorCategoryChoices.RECORD_PRE_CHECK + assert ( + length_error.error_message + == "T1: record length should be at least 117 characters but it is 7 characters." + ) + assert length_error.content_type is None + assert length_error.object_id is None + + trailer_error_3 = trailer_errors[3] + assert trailer_error_3.error_type == ParserErrorCategoryChoices.RECORD_PRE_CHECK + assert ( + trailer_error_3.error_message + == "T1: Case number T1trash cannot contain blanks." + ) + assert trailer_error_3.content_type is None + assert trailer_error_3.object_id is None + + trailer_error_4 = trailer_errors[4] + assert trailer_error_4.error_type == ParserErrorCategoryChoices.RECORD_PRE_CHECK + assert trailer_error_4.error_message == ( + "T1: Reporting month year None does not " + "match file reporting year:2021, quarter:Q1." + ) + assert trailer_error_4.content_type is None + assert trailer_error_4.object_id is None + + @pytest.mark.django_db + def test_parse_empty_file(self, empty_file, dfs): + """Test parsing of empty_file.""" + dfs.datafile = empty_file + dfs.save() + parse_datafile(dfs, empty_file) + + dfs.status = dfs.get_status() + dfs.case_aggregates = aggregates.case_aggregates_by_month(empty_file, dfs.status) + + assert dfs.status == DataFileSummary.Status.REJECTED + assert dfs.case_aggregates == { + "rejected": 1, + "months": [ + { + "accepted_without_errors": "N/A", + "accepted_with_errors": "N/A", + "month": "Oct", + }, + { + "accepted_without_errors": "N/A", + "accepted_with_errors": "N/A", + "month": "Nov", + }, + { + "accepted_without_errors": "N/A", + "accepted_with_errors": "N/A", + "month": "Dec", + }, + ], + } - row_4_error = parser_errors.get( - row_number=4, - error_message__contains="Reporting month year None does not match file reporting year:2019, quarter:Q1.", - ) - assert row_4_error.error_type == ParserErrorCategoryChoices.RECORD_PRE_CHECK + parser_errors = ParserError.objects.filter(file=empty_file).order_by("id") + + assert parser_errors.count() == 2 + + err = parser_errors.first() + + assert err.row_number == 1 + assert err.error_type == ParserErrorCategoryChoices.PRE_CHECK + assert err.error_message == "HEADER: record length is 0 characters but must be 23." + assert err.content_type is None + assert err.object_id is None + + @pytest.mark.django_db + def test_parse_small_ssp_section1_datafile(self, small_ssp_section1_datafile, dfs): + """Test parsing small_ssp_section1_datafile.""" + small_ssp_section1_datafile.year = 2024 + small_ssp_section1_datafile.quarter = "Q1" + + expected_m1_record_count = 5 + expected_m2_record_count = 6 + expected_m3_record_count = 8 + + dfs.datafile = small_ssp_section1_datafile + dfs.save() + parse_datafile(dfs, small_ssp_section1_datafile) + + parser_errors = ParserError.objects.filter(file=small_ssp_section1_datafile) + dfs.status = dfs.get_status() + assert dfs.status == DataFileSummary.Status.PARTIALLY_ACCEPTED + dfs.case_aggregates = aggregates.case_aggregates_by_month(dfs.datafile, dfs.status) + + assert dfs.case_aggregates["rejected"] == 1 + for month in dfs.case_aggregates["months"]: + if month["month"] == "Oct": + assert month["accepted_without_errors"] == 0 + assert month["accepted_with_errors"] == 5 + else: + assert month["accepted_without_errors"] == 0 + assert month["accepted_with_errors"] == 0 + + parser_errors = ParserError.objects.filter(file=small_ssp_section1_datafile) + assert parser_errors.count() == 9 + assert SSP_M1.objects.count() == expected_m1_record_count + assert SSP_M2.objects.count() == expected_m2_record_count + assert SSP_M3.objects.count() == expected_m3_record_count + + @pytest.mark.django_db() + def test_parse_ssp_section1_datafile(self, ssp_section1_datafile, dfs): + """Test parsing ssp_section1_datafile.""" + ssp_section1_datafile.year = 2019 + ssp_section1_datafile.quarter = "Q1" + + expected_m1_record_count = 818 + expected_m2_record_count = 989 + expected_m3_record_count = 1748 + + dfs.datafile = ssp_section1_datafile + dfs.save() + + parse_datafile(dfs, ssp_section1_datafile) + + parser_errors = ParserError.objects.filter(file=ssp_section1_datafile).order_by( + "row_number" + ) - error_message = ( - "Reporting month year None does not match file reporting year:2019, quarter:Q1." - ) - rpt_month_errors = parser_errors.filter(error_message__contains=error_message) - assert len(rpt_month_errors) == 3 - for i, e in enumerate(rpt_month_errors): - assert e.error_type == ParserErrorCategoryChoices.RECORD_PRE_CHECK - assert error_message.format(i + 1) in e.error_message - assert e.object_id is None - - row_5_error = parser_errors.get( - row_number=5, error_message="Unknown Record_Type was found." - ) - assert row_5_error.error_type == ParserErrorCategoryChoices.RECORD_PRE_CHECK - assert row_5_error.content_type is None - assert row_5_error.object_id is None + err = parser_errors.first() - trailer_error = parser_errors.get( - row_number=6, - error_message="TRAILER: record length is 15 characters but must be 23.", - ) - assert trailer_error.error_type == ParserErrorCategoryChoices.RECORD_PRE_CHECK - assert trailer_error.content_type is None - assert trailer_error.object_id is None - - -@pytest.mark.django_db -def test_dfs_set_case_aggregates(small_correct_file, dfs): - """Test that the case aggregates are set correctly.""" - small_correct_file.year = 2020 - small_correct_file.quarter = "Q3" - small_correct_file.section = "Active Case Data" - small_correct_file.save() - # this still needs to execute to create db objects to be queried - parser = ParserFactory.get_instance( - datafile=small_correct_file, - dfs=dfs, - section=small_correct_file.section, - program_type=small_correct_file.program_type, - ) - parser.parse_and_validate() - dfs.file = small_correct_file - dfs.status = dfs.get_status() - dfs.case_aggregates = aggregates.case_aggregates_by_month( - small_correct_file, dfs.status - ) + assert err.row_number == 2 + assert err.error_type == ParserErrorCategoryChoices.FIELD_VALUE + assert err.error_message == ( + "M1 Item 11 (Receives Subsidized Housing): 3 is not in range [1, 2]." + ) + assert err.content_type is not None + assert err.object_id is not None - for month in dfs.case_aggregates["months"]: - if month["month"] == "Oct": - assert month["accepted_without_errors"] == 1 - assert month["accepted_with_errors"] == 0 + cat4_errors = parser_errors.filter( + error_type=ParserErrorCategoryChoices.CASE_CONSISTENCY + ).order_by("id") + assert cat4_errors.count() == 3 + assert ( + cat4_errors[0].error_message + == "Duplicate record detected with record type M3 at line 453. " + + "Record is a duplicate of the record at line number 452." + ) + assert ( + cat4_errors[1].error_message + == "Duplicate record detected with record type M3 at line 3273. " + + "Record is a duplicate of the record at line number 3272." + ) + assert ( + cat4_errors[2].error_message + == "Partial duplicate record detected with record type M3 at line 3275. " + + "Record is a partial duplicate of the record at line number 3274. Duplicated fields " + + "causing error: Item 0 (Record Type), Item 3 (Reporting Year and Month), Item 5 (Case Number), " + + "Item 60 (Family Affiliation), Item 61 (Date of Birth), and Item 62 (Social Security Number)." + ) -@pytest.mark.django_db() -def test_parse_small_tanf_section2_file(small_tanf_section2_file, dfs): - """Test parsing a small TANF Section 2 submission.""" - small_tanf_section2_file.year = 2021 - small_tanf_section2_file.quarter = "Q1" + assert parser_errors.count() == 31726 + + assert SSP_M1.objects.count() == expected_m1_record_count + assert SSP_M2.objects.count() == expected_m2_record_count + assert SSP_M3.objects.count() == expected_m3_record_count + + @pytest.mark.django_db + def test_parse_tanf_section1_datafile(self, small_tanf_section1_datafile, dfs): + """Test parsing of small_tanf_section1_datafile and validate T2 model data.""" + small_tanf_section1_datafile.year = 2021 + small_tanf_section1_datafile.quarter = "Q1" + dfs.datafile = small_tanf_section1_datafile + + parse_datafile(dfs, small_tanf_section1_datafile) + + dfs.status = dfs.get_status() + assert dfs.status == DataFileSummary.Status.ACCEPTED_WITH_ERRORS + dfs.case_aggregates = aggregates.case_aggregates_by_month(dfs.datafile, dfs.status) + assert dfs.case_aggregates == { + "months": [ + {"month": "Oct", "accepted_without_errors": 1, "accepted_with_errors": 4}, + {"month": "Nov", "accepted_without_errors": 0, "accepted_with_errors": 0}, + {"month": "Dec", "accepted_without_errors": 0, "accepted_with_errors": 0}, + ], + "rejected": 0, + } - dfs.datafile = small_tanf_section2_file - dfs.save() + assert TANF_T2.objects.count() == 5 - parser = ParserFactory.get_instance( - datafile=small_tanf_section2_file, - dfs=dfs, - section=small_tanf_section2_file.section, - program_type=small_tanf_section2_file.program_type, - ) - parser.parse_and_validate() + t2_models = TANF_T2.objects.all() - assert TANF_T4.objects.all().count() == 1 - assert TANF_T5.objects.all().count() == 1 + t2 = t2_models[0] + assert t2.RPT_MONTH_YEAR == 202010 + assert t2.CASE_NUMBER == "11111111112" + assert t2.FAMILY_AFFILIATION == 1 + assert t2.OTHER_UNEARNED_INCOME == "0291" - parser_errors = ParserError.objects.filter(file=small_tanf_section2_file) + t2_2 = t2_models[1] + assert t2_2.RPT_MONTH_YEAR == 202010 + assert t2_2.CASE_NUMBER == "11111111115" + assert t2_2.FAMILY_AFFILIATION == 2 + assert t2_2.OTHER_UNEARNED_INCOME == "0000" - assert parser_errors.count() == 0 + @pytest.mark.django_db() + def test_parse_tanf_section1_datafile_obj_counts(self, small_tanf_section1_datafile, dfs): + """Test parsing of small_tanf_section1_datafile in general.""" + small_tanf_section1_datafile.year = 2021 + small_tanf_section1_datafile.quarter = "Q1" - t4 = TANF_T4.objects.first() - t5 = TANF_T5.objects.first() + dfs.datafile = small_tanf_section1_datafile + dfs.save() - assert t4.DISPOSITION == 1 - assert t4.REC_SUB_CC == 3 + parse_datafile(dfs, small_tanf_section1_datafile) - assert t5.SEX == 2 - assert t5.AMOUNT_UNEARNED_INCOME == "0000" + assert TANF_T1.objects.count() == 5 + assert TANF_T2.objects.count() == 5 + assert TANF_T3.objects.count() == 6 + @pytest.mark.django_db() + def test_parse_tanf_section1_datafile_t3s(self, small_tanf_section1_datafile, dfs): + """Test parsing of small_tanf_section1_datafile and validate T3 model data.""" + small_tanf_section1_datafile.year = 2021 + small_tanf_section1_datafile.quarter = "Q1" -@pytest.mark.django_db() -def test_parse_tanf_section2_file(tanf_section2_file, dfs): - """Test parsing TANF Section 2 submission.""" - tanf_section2_file.year = 2022 - tanf_section2_file.quarter = "Q1" + dfs.datafile = small_tanf_section1_datafile + dfs.save() - dfs.datafile = tanf_section2_file - dfs.save() + parse_datafile(dfs, small_tanf_section1_datafile) - parser = ParserFactory.get_instance( - datafile=tanf_section2_file, - dfs=dfs, - section=tanf_section2_file.section, - program_type=tanf_section2_file.program_type, - ) - parser.parse_and_validate() - parser_errors = ParserError.objects.filter(file=tanf_section2_file) - err = parser_errors.first() - print("\n\n", err.error_message, "\n\n") + assert TANF_T3.objects.count() == 6 - assert TANF_T4.objects.all().count() == 223 - assert TANF_T5.objects.all().count() == 605 + t3_models = TANF_T3.objects.all() + t3_1 = t3_models[0] + assert t3_1.RPT_MONTH_YEAR == 202010 + assert t3_1.CASE_NUMBER == "11111111112" + assert t3_1.FAMILY_AFFILIATION == 1 + assert t3_1.SEX == 2 + assert t3_1.EDUCATION_LEVEL == "98" - parser_errors = ParserError.objects.filter(file=tanf_section2_file) + t3_5 = t3_models[4] + assert t3_5.RPT_MONTH_YEAR == 202010 + assert t3_5.CASE_NUMBER == "11111111151" + assert t3_5.FAMILY_AFFILIATION == 1 + assert t3_5.SEX == 1 + assert t3_5.EDUCATION_LEVEL == "98" - err = parser_errors.first() - assert err.error_type == ParserErrorCategoryChoices.FIELD_VALUE - assert err.error_message == ( - "T4 Item 10 (Received Subsidized Housing): 3 is not in range [1, 2]." - ) - assert err.content_type.model == "tanf_t4" - assert err.object_id is not None + @pytest.mark.django_db + def test_parse_bad_tfs1_missing_required(self, bad_tanf_s1__row_missing_required_field, dfs): + """Test parsing a bad TANF Section 1 submission where a row is missing required data.""" + bad_tanf_s1__row_missing_required_field.year = 2021 + bad_tanf_s1__row_missing_required_field.quarter = "Q1" + dfs.datafile = bad_tanf_s1__row_missing_required_field + dfs.save() -@pytest.mark.django_db() -def test_parse_tanf_section3_file(tanf_section3_file, dfs): - """Test parsing TANF Section 3 submission.""" - tanf_section3_file.year = 2022 - tanf_section3_file.quarter = "Q1" + parse_datafile(dfs, bad_tanf_s1__row_missing_required_field) - dfs.datafile = tanf_section3_file + assert dfs.get_status() == DataFileSummary.Status.REJECTED - parser = ParserFactory.get_instance( - datafile=tanf_section3_file, - dfs=dfs, - section=tanf_section3_file.section, - program_type=tanf_section3_file.program_type, - ) - parser.parse_and_validate() + parser_errors = ParserError.objects.filter( + file=bad_tanf_s1__row_missing_required_field + ) - dfs.status = dfs.get_status() - dfs.case_aggregates = aggregates.total_errors_by_month(dfs.datafile, dfs.status) - assert dfs.case_aggregates == { - "months": [ - {"month": "Oct", "total_errors": 0}, - {"month": "Nov", "total_errors": 0}, - {"month": "Dec", "total_errors": 0}, - ] - } + assert parser_errors.count() == 5 - assert dfs.get_status() == DataFileSummary.Status.ACCEPTED + error_message = "T1: Reporting month year None does not match file reporting year:2021, quarter:Q1." + row_2_error = parser_errors.get(row_number=2, error_message=error_message) + assert row_2_error.error_type == ParserErrorCategoryChoices.RECORD_PRE_CHECK + assert row_2_error.error_message == error_message - assert TANF_T6.objects.all().count() == 3 + error_message = "T2: Reporting month year None does not match file reporting year:2021, quarter:Q1." + row_3_error = parser_errors.get(row_number=3, error_message=error_message) + assert row_3_error.error_type == ParserErrorCategoryChoices.RECORD_PRE_CHECK + assert row_3_error.error_message == error_message - parser_errors = ParserError.objects.filter(file=tanf_section3_file) - assert parser_errors.count() == 0 + error_message = "T3: Reporting month year None does not match file reporting year:2021, quarter:Q1." + row_4_error = parser_errors.get(row_number=4, error_message=error_message) + assert row_4_error.error_type == ParserErrorCategoryChoices.RECORD_PRE_CHECK + assert row_4_error.error_message == error_message - t6_objs = TANF_T6.objects.all().order_by("NUM_APPROVED") + error_message = "Unknown Record_Type was found." + row_5_error = parser_errors.get(row_number=5, error_message=error_message) + assert row_5_error.error_type == ParserErrorCategoryChoices.RECORD_PRE_CHECK + assert row_5_error.error_message == error_message + assert row_5_error.content_type is None + assert row_5_error.object_id is None - first = t6_objs.first() - second = t6_objs[1] - third = t6_objs[2] + @pytest.mark.django_db() + def test_parse_bad_ssp_s1_missing_required(self, bad_ssp_s1__row_missing_required_field, dfs): + """Test parsing a bad TANF Section 1 submission where a row is missing required data.""" + bad_ssp_s1__row_missing_required_field.year = 2019 + bad_ssp_s1__row_missing_required_field.quarter = "Q1" - assert first.RPT_MONTH_YEAR == 202112 - assert second.RPT_MONTH_YEAR == 202111 - assert third.RPT_MONTH_YEAR == 202110 + dfs.datafile = bad_ssp_s1__row_missing_required_field + dfs.save() - assert first.NUM_APPROVED == 3924 - assert second.NUM_APPROVED == 3977 - assert third.NUM_APPROVED == 4301 + parse_datafile(dfs, bad_ssp_s1__row_missing_required_field) - assert first.NUM_CLOSED_CASES == 3884 - assert second.NUM_CLOSED_CASES == 3881 - assert third.NUM_CLOSED_CASES == 5453 + parser_errors = ParserError.objects.filter( + file=bad_ssp_s1__row_missing_required_field + ) + assert parser_errors.count() == 6 + row_2_error = parser_errors.get( + row_number=2, + error_message__contains="Reporting month year None does not match file reporting year:2019, quarter:Q1.", + ) + assert row_2_error.error_type == ParserErrorCategoryChoices.RECORD_PRE_CHECK -@pytest.mark.django_db() -def test_parse_tanf_section1_blanks_file(tanf_section1_file_with_blanks, dfs): - """Test section 1 fields that are allowed to have blanks.""" - tanf_section1_file_with_blanks.year = 2021 - tanf_section1_file_with_blanks.quarter = "Q1" + row_3_error = parser_errors.get( + row_number=3, + error_message__contains="Reporting month year None does not match file reporting year:2019, quarter:Q1.", + ) + assert row_3_error.error_type == ParserErrorCategoryChoices.RECORD_PRE_CHECK - dfs.datafile = tanf_section1_file_with_blanks - dfs.save() + row_4_error = parser_errors.get( + row_number=4, + error_message__contains="Reporting month year None does not match file reporting year:2019, quarter:Q1.", + ) + assert row_4_error.error_type == ParserErrorCategoryChoices.RECORD_PRE_CHECK - parser = ParserFactory.get_instance( - datafile=tanf_section1_file_with_blanks, - dfs=dfs, - section=tanf_section1_file_with_blanks.section, - program_type=tanf_section1_file_with_blanks.program_type, - ) - parser.parse_and_validate() + error_message = ( + "Reporting month year None does not match file reporting year:2019, quarter:Q1." + ) + rpt_month_errors = parser_errors.filter(error_message__contains=error_message) + assert len(rpt_month_errors) == 3 + for i, e in enumerate(rpt_month_errors): + assert e.error_type == ParserErrorCategoryChoices.RECORD_PRE_CHECK + assert error_message.format(i + 1) in e.error_message + assert e.object_id is None + + row_5_error = parser_errors.get( + row_number=5, error_message="Unknown Record_Type was found." + ) + assert row_5_error.error_type == ParserErrorCategoryChoices.RECORD_PRE_CHECK + assert row_5_error.content_type is None + assert row_5_error.object_id is None - parser_errors = ParserError.objects.filter(file=tanf_section1_file_with_blanks) + trailer_error = parser_errors.get( + row_number=6, + error_message="TRAILER: record length is 15 characters but must be 23.", + ) + assert trailer_error.error_type == ParserErrorCategoryChoices.RECORD_PRE_CHECK + assert trailer_error.content_type is None + assert trailer_error.object_id is None + + @pytest.mark.django_db + def test_dfs_set_case_aggregates(self, small_correct_file, dfs): + """Test that the case aggregates are set correctly.""" + small_correct_file.year = 2020 + small_correct_file.quarter = "Q3" + small_correct_file.section = "Active Case Data" + small_correct_file.save() + # this still needs to execute to create db objects to be queried + parse_datafile(dfs, small_correct_file) + dfs.file = small_correct_file + dfs.status = dfs.get_status() + dfs.case_aggregates = aggregates.case_aggregates_by_month( + small_correct_file, dfs.status + ) - assert parser_errors.count() == 23 + for month in dfs.case_aggregates["months"]: + if month["month"] == "Oct": + assert month["accepted_without_errors"] == 1 + assert month["accepted_with_errors"] == 0 - # Should only be cat3 validator errors - for error in parser_errors: - assert error.error_type == ParserErrorCategoryChoices.VALUE_CONSISTENCY + @pytest.mark.django_db() + def test_parse_small_tanf_section2_file(self, small_tanf_section2_file, dfs): + """Test parsing a small TANF Section 2 submission.""" + small_tanf_section2_file.year = 2021 + small_tanf_section2_file.quarter = "Q1" - t1 = TANF_T1.objects.first() - t2 = TANF_T2.objects.first() - t3 = TANF_T3.objects.first() + dfs.datafile = small_tanf_section2_file + dfs.save() - assert t1.FAMILY_SANC_ADULT is None - assert t2.MARITAL_STATUS is None - assert t3.CITIZENSHIP_STATUS is None + parse_datafile(dfs, small_tanf_section2_file) + assert TANF_T4.objects.all().count() == 1 + assert TANF_T5.objects.all().count() == 1 -@pytest.mark.django_db() -def test_parse_tanf_section4_file(tanf_section4_file, dfs): - """Test parsing TANF Section 4 submission.""" - tanf_section4_file.year = 2022 - tanf_section4_file.quarter = "Q1" + parser_errors = ParserError.objects.filter(file=small_tanf_section2_file) - dfs.datafile = tanf_section4_file + assert parser_errors.count() == 0 - parser = ParserFactory.get_instance( - datafile=tanf_section4_file, - dfs=dfs, - section=tanf_section4_file.section, - program_type=tanf_section4_file.program_type, - ) - parser.parse_and_validate() + t4 = TANF_T4.objects.first() + t5 = TANF_T5.objects.first() - dfs.status = dfs.get_status() - dfs.case_aggregates = aggregates.total_errors_by_month(dfs.datafile, dfs.status) - assert dfs.case_aggregates == { - "months": [ - {"month": "Oct", "total_errors": 0}, - {"month": "Nov", "total_errors": 0}, - {"month": "Dec", "total_errors": 0}, - ] - } + assert t4.DISPOSITION == 1 + assert t4.REC_SUB_CC == 3 - assert dfs.get_status() == DataFileSummary.Status.ACCEPTED + assert t5.SEX == 2 + assert t5.AMOUNT_UNEARNED_INCOME == "0000" - assert TANF_T7.objects.all().count() == 18 + @pytest.mark.django_db() + def test_parse_tanf_section2_file(self, tanf_section2_file, dfs): + """Test parsing TANF Section 2 submission.""" + tanf_section2_file.year = 2022 + tanf_section2_file.quarter = "Q1" - parser_errors = ParserError.objects.filter(file=tanf_section4_file) - assert parser_errors.count() == 0 + dfs.datafile = tanf_section2_file + dfs.save() - t7_objs = TANF_T7.objects.all().order_by("FAMILIES_MONTH") + parse_datafile(dfs, tanf_section2_file) - first = t7_objs.first() - sixth = t7_objs[5] + assert TANF_T4.objects.all().count() == 223 + assert TANF_T5.objects.all().count() == 605 - assert first.RPT_MONTH_YEAR == 202111 - assert sixth.RPT_MONTH_YEAR == 202112 + parser_errors = ParserError.objects.filter(file=tanf_section2_file) - assert first.TDRS_SECTION_IND == "2" - assert sixth.TDRS_SECTION_IND == "2" + err = parser_errors.first() + assert err.error_type == ParserErrorCategoryChoices.FIELD_VALUE + assert err.error_message == ( + "T4 Item 10 (Received Subsidized Housing): 3 is not in range [1, 2]." + ) + assert err.content_type.model == "tanf_t4" + assert err.object_id is not None + + @pytest.mark.django_db() + def test_parse_tanf_section3_file(self, tanf_section3_file, dfs): + """Test parsing TANF Section 3 submission.""" + tanf_section3_file.year = 2022 + tanf_section3_file.quarter = "Q1" + + dfs.datafile = tanf_section3_file + + parse_datafile(dfs, tanf_section3_file) + + dfs.status = dfs.get_status() + dfs.case_aggregates = aggregates.total_errors_by_month(dfs.datafile, dfs.status) + assert dfs.case_aggregates == { + "months": [ + {"month": "Oct", "total_errors": 0}, + {"month": "Nov", "total_errors": 0}, + {"month": "Dec", "total_errors": 0}, + ] + } - assert first.FAMILIES_MONTH == 274 - assert sixth.FAMILIES_MONTH == 499 + assert dfs.get_status() == DataFileSummary.Status.ACCEPTED + assert TANF_T6.objects.all().count() == 3 -@pytest.mark.django_db() -def test_parse_bad_tanf_section4_file(bad_tanf_section4_file, dfs): - """Test parsing TANF Section 4 submission when no records are created.""" - bad_tanf_section4_file.year = 2021 - bad_tanf_section4_file.quarter = "Q1" + parser_errors = ParserError.objects.filter(file=tanf_section3_file) + assert parser_errors.count() == 0 - dfs.datafile = bad_tanf_section4_file + t6_objs = TANF_T6.objects.all().order_by("NUM_APPROVED") - parser = ParserFactory.get_instance( - datafile=bad_tanf_section4_file, - dfs=dfs, - section=bad_tanf_section4_file.section, - program_type=bad_tanf_section4_file.program_type, - ) - parser.parse_and_validate() + first = t6_objs.first() + second = t6_objs[1] + third = t6_objs[2] - dfs.status = dfs.get_status() - dfs.case_aggregates = aggregates.total_errors_by_month(dfs.datafile, dfs.status) + assert first.RPT_MONTH_YEAR == 202112 + assert second.RPT_MONTH_YEAR == 202111 + assert third.RPT_MONTH_YEAR == 202110 - assert dfs.case_aggregates == { - "months": [ - {"month": "Oct", "total_errors": "N/A"}, - {"month": "Nov", "total_errors": "N/A"}, - {"month": "Dec", "total_errors": "N/A"}, - ] - } + assert first.NUM_APPROVED == 3924 + assert second.NUM_APPROVED == 3977 + assert third.NUM_APPROVED == 4301 - assert dfs.get_status() == DataFileSummary.Status.REJECTED + assert first.NUM_CLOSED_CASES == 3884 + assert second.NUM_CLOSED_CASES == 3881 + assert third.NUM_CLOSED_CASES == 5453 - assert TANF_T7.objects.all().count() == 0 + @pytest.mark.django_db() + def test_parse_tanf_section1_blanks_file(self, tanf_section1_file_with_blanks, dfs): + """Test section 1 fields that are allowed to have blanks.""" + tanf_section1_file_with_blanks.year = 2021 + tanf_section1_file_with_blanks.quarter = "Q1" - parser_errors = ParserError.objects.filter(file=bad_tanf_section4_file).order_by( - "id" - ) - assert parser_errors.count() == 2 + dfs.datafile = tanf_section1_file_with_blanks + dfs.save() - error = parser_errors.first() - error.error_message == "Value length 151 does not match 247." - error.error_type == ParserErrorCategoryChoices.PRE_CHECK + parse_datafile(dfs, tanf_section1_file_with_blanks) - error = parser_errors[1] - error.error_message == "No records created." - error.error_type == ParserErrorCategoryChoices.PRE_CHECK + parser_errors = ParserError.objects.filter(file=tanf_section1_file_with_blanks) + assert parser_errors.count() == 23 -@pytest.mark.django_db() -def test_parse_ssp_section4_file(ssp_section4_file, dfs): - """Test parsing SSP Section 4 submission.""" - ssp_section4_file.year = 2022 - ssp_section4_file.quarter = "Q1" + # Should only be cat3 validator errors + for error in parser_errors: + assert error.error_type == ParserErrorCategoryChoices.VALUE_CONSISTENCY - dfs.datafile = ssp_section4_file + t1 = TANF_T1.objects.first() + t2 = TANF_T2.objects.first() + t3 = TANF_T3.objects.first() - parser = ParserFactory.get_instance( - datafile=ssp_section4_file, - dfs=dfs, - section=ssp_section4_file.section, - program_type=ssp_section4_file.program_type, - ) - parser.parse_and_validate() + assert t1.FAMILY_SANC_ADULT is None + assert t2.MARITAL_STATUS is None + assert t3.CITIZENSHIP_STATUS is None - m7_objs = SSP_M7.objects.all().order_by("FAMILIES_MONTH") + @pytest.mark.django_db() + def test_parse_tanf_section4_file(self, tanf_section4_file, dfs): + """Test parsing TANF Section 4 submission.""" + tanf_section4_file.year = 2022 + tanf_section4_file.quarter = "Q1" - dfs.status = dfs.get_status() - dfs.case_aggregates = aggregates.total_errors_by_month(dfs.datafile, dfs.status) + dfs.datafile = tanf_section4_file - assert dfs.case_aggregates == { - "months": [ - {"month": "Oct", "total_errors": 0}, - {"month": "Nov", "total_errors": 0}, - {"month": "Dec", "total_errors": 0}, - ] - } + parse_datafile(dfs, tanf_section4_file) - assert m7_objs.count() == 12 + dfs.status = dfs.get_status() + dfs.case_aggregates = aggregates.total_errors_by_month(dfs.datafile, dfs.status) + assert dfs.case_aggregates == { + "months": [ + {"month": "Oct", "total_errors": 0}, + {"month": "Nov", "total_errors": 0}, + {"month": "Dec", "total_errors": 0}, + ] + } - first = m7_objs.first() - assert first.RPT_MONTH_YEAR == 202110 - assert first.FAMILIES_MONTH == 748 + assert dfs.get_status() == DataFileSummary.Status.ACCEPTED + assert TANF_T7.objects.all().count() == 18 -@pytest.mark.django_db() -def test_parse_ssp_section2_rec_oadsi_file(ssp_section2_rec_oadsi_file, dfs): - """Test parsing SSP Section 2 REC_OADSI.""" - ssp_section2_rec_oadsi_file.year = 2019 - ssp_section2_rec_oadsi_file.quarter = "Q1" + parser_errors = ParserError.objects.filter(file=tanf_section4_file) + assert parser_errors.count() == 0 - dfs.datafile = ssp_section2_rec_oadsi_file + t7_objs = TANF_T7.objects.all().order_by("FAMILIES_MONTH") - parser = ParserFactory.get_instance( - datafile=ssp_section2_rec_oadsi_file, - dfs=dfs, - section=ssp_section2_rec_oadsi_file.section, - program_type=ssp_section2_rec_oadsi_file.program_type, - ) - parser.parse_and_validate() - parser_errors = ParserError.objects.filter(file=ssp_section2_rec_oadsi_file) + first = t7_objs.first() + sixth = t7_objs[5] - assert parser_errors.count() == 0 + assert first.RPT_MONTH_YEAR == 202111 + assert sixth.RPT_MONTH_YEAR == 202112 + assert first.TDRS_SECTION_IND == "2" + assert sixth.TDRS_SECTION_IND == "2" -@pytest.mark.django_db() -def test_parse_ssp_section2_file(ssp_section2_file, dfs): - """Test parsing SSP Section 2 submission.""" - ssp_section2_file.year = 2019 - ssp_section2_file.quarter = "Q1" + assert first.FAMILIES_MONTH == 274 + assert sixth.FAMILIES_MONTH == 499 - dfs.datafile = ssp_section2_file + @pytest.mark.django_db() + def test_parse_bad_tanf_section4_file(self, bad_tanf_section4_file, dfs): + """Test parsing TANF Section 4 submission when no records are created.""" + bad_tanf_section4_file.year = 2021 + bad_tanf_section4_file.quarter = "Q1" - parser = ParserFactory.get_instance( - datafile=ssp_section2_file, - dfs=dfs, - section=ssp_section2_file.section, - program_type=ssp_section2_file.program_type, - ) - parser.parse_and_validate() + dfs.datafile = bad_tanf_section4_file - dfs.status = dfs.get_status() - dfs.case_aggregates = aggregates.case_aggregates_by_month(dfs.datafile, dfs.status) - for dfs_case_aggregate in dfs.case_aggregates["months"]: - assert dfs_case_aggregate["accepted_without_errors"] == 0 - assert dfs_case_aggregate["accepted_with_errors"] in [75, 78] - assert dfs_case_aggregate["month"] in ["Oct", "Nov", "Dec"] - assert dfs.get_status() == DataFileSummary.Status.PARTIALLY_ACCEPTED + parse_datafile(dfs, bad_tanf_section4_file) - m4_objs = SSP_M4.objects.all().order_by("id") - m5_objs = SSP_M5.objects.all().order_by("AMOUNT_EARNED_INCOME") + dfs.status = dfs.get_status() + dfs.case_aggregates = aggregates.total_errors_by_month(dfs.datafile, dfs.status) - expected_m4_count = 231 - expected_m5_count = 703 + assert dfs.case_aggregates == { + "months": [ + {"month": "Oct", "total_errors": "N/A"}, + {"month": "Nov", "total_errors": "N/A"}, + {"month": "Dec", "total_errors": "N/A"}, + ] + } - assert SSP_M4.objects.count() == expected_m4_count - assert SSP_M5.objects.count() == expected_m5_count + assert dfs.get_status() == DataFileSummary.Status.REJECTED - m4 = m4_objs.first() - assert m4.DISPOSITION == 1 - assert m4.REC_SUB_CC == 3 + assert TANF_T7.objects.all().count() == 0 - m5 = m5_objs.first() - assert m5.FAMILY_AFFILIATION == 1 - assert m5.AMOUNT_EARNED_INCOME == "0000" - assert m5.AMOUNT_UNEARNED_INCOME == "0000" + parser_errors = ParserError.objects.filter(file=bad_tanf_section4_file).order_by( + "id" + ) + assert parser_errors.count() == 2 + error = parser_errors.first() + error.error_message == "Value length 151 does not match 247." + error.error_type == ParserErrorCategoryChoices.PRE_CHECK -@pytest.mark.django_db() -def test_parse_ssp_section3_file(ssp_section3_file, dfs): - """Test parsing TANF Section 3 submission.""" - ssp_section3_file.year = 2022 - ssp_section3_file.quarter = "Q1" + error = parser_errors[1] + error.error_message == "No records created." + error.error_type == ParserErrorCategoryChoices.PRE_CHECK - dfs.datafile = ssp_section3_file + @pytest.mark.django_db() + def test_parse_ssp_section4_file(self, ssp_section4_file, dfs): + """Test parsing SSP Section 4 submission.""" + ssp_section4_file.year = 2022 + ssp_section4_file.quarter = "Q1" - parser = ParserFactory.get_instance( - datafile=ssp_section3_file, - dfs=dfs, - section=ssp_section3_file.section, - program_type=ssp_section3_file.program_type, - ) - parser.parse_and_validate() - - dfs.status = dfs.get_status() - dfs.case_aggregates = aggregates.total_errors_by_month(dfs.datafile, dfs.status) - assert dfs.case_aggregates == { - "months": [ - {"month": "Oct", "total_errors": 0}, - {"month": "Nov", "total_errors": 0}, - {"month": "Dec", "total_errors": 0}, - ] - } - - assert dfs.get_status() == DataFileSummary.Status.ACCEPTED - - m6_objs = SSP_M6.objects.all().order_by("RPT_MONTH_YEAR") - assert m6_objs.count() == 3 - - parser_errors = ParserError.objects.filter(file=ssp_section3_file) - assert parser_errors.count() == 0 - - first = m6_objs.first() - second = m6_objs[1] - third = m6_objs[2] - - assert first.RPT_MONTH_YEAR == 202110 - assert second.RPT_MONTH_YEAR == 202111 - assert third.RPT_MONTH_YEAR == 202112 - - assert first.SSPMOE_FAMILIES == 15869 - assert second.SSPMOE_FAMILIES == 16008 - assert third.SSPMOE_FAMILIES == 15956 - - assert first.NUM_RECIPIENTS == 51355 - assert second.NUM_RECIPIENTS == 51696 - assert third.NUM_RECIPIENTS == 51348 - - -@pytest.mark.django_db -def test_rpt_month_year_mismatch(header_datafile, dfs): - """Test that the rpt_month_year mismatch error is raised.""" - datafile = header_datafile - - datafile.section = "Active Case Data" - # test_datafile fixture uses create_test_data_file which assigns - # a default year / quarter of 2021 / Q1 - datafile.year = 2021 - datafile.quarter = "Q1" - datafile.save() - - dfs.datafile = header_datafile - dfs.save() - - parser = ParserFactory.get_instance( - datafile=datafile, - dfs=dfs, - section=datafile.section, - program_type=datafile.program_type, - ) - parser.parse_and_validate() + dfs.datafile = ssp_section4_file - parser_errors = ParserError.objects.filter(file=datafile) - assert parser_errors.count() == 2 - assert ( - parser_errors.first().error_type == ParserErrorCategoryChoices.CASE_CONSISTENCY - ) + parse_datafile(dfs, ssp_section4_file) - datafile.year = 2023 - datafile.save() + m7_objs = SSP_M7.objects.all().order_by("FAMILIES_MONTH") - parser = ParserFactory.get_instance( - datafile=datafile, - dfs=dfs, - section=datafile.section, - program_type=datafile.program_type, - ) - parser.parse_and_validate() + dfs.status = dfs.get_status() + dfs.case_aggregates = aggregates.total_errors_by_month(dfs.datafile, dfs.status) - parser_errors = ParserError.objects.filter(file=datafile).order_by("-id") - assert parser_errors.count() == 3 + assert dfs.case_aggregates == { + "months": [ + {"month": "Oct", "total_errors": 0}, + {"month": "Nov", "total_errors": 0}, + {"month": "Dec", "total_errors": 0}, + ] + } - err = parser_errors.first() - assert err.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert ( - err.error_message - == "Submitted reporting year:2020, quarter:Q4 doesn't" - + " match file reporting year:2023, quarter:Q1." - ) + assert m7_objs.count() == 12 + first = m7_objs.first() + assert first.RPT_MONTH_YEAR == 202110 + assert first.FAMILIES_MONTH == 748 -@pytest.mark.django_db() -def test_parse_tribal_section_1_file(tribal_section_1_file, dfs): - """Test parsing Tribal TANF Section 1 submission.""" - tribal_section_1_file.year = 2022 - tribal_section_1_file.quarter = "Q1" - tribal_section_1_file.save() + @pytest.mark.django_db() + def test_parse_ssp_section2_rec_oadsi_file(self, ssp_section2_rec_oadsi_file, dfs): + """Test parsing SSP Section 2 REC_OADSI.""" + ssp_section2_rec_oadsi_file.year = 2019 + ssp_section2_rec_oadsi_file.quarter = "Q1" - dfs.datafile = tribal_section_1_file + dfs.datafile = ssp_section2_rec_oadsi_file - parser = ParserFactory.get_instance( - datafile=tribal_section_1_file, - dfs=dfs, - section=tribal_section_1_file.section, - program_type=tribal_section_1_file.program_type, - ) - parser.parse_and_validate() - - dfs.status = dfs.get_status() - assert dfs.status == DataFileSummary.Status.ACCEPTED - dfs.case_aggregates = aggregates.case_aggregates_by_month(dfs.datafile, dfs.status) - assert dfs.case_aggregates == { - "rejected": 0, - "months": [ - {"month": "Oct", "accepted_without_errors": 1, "accepted_with_errors": 0}, - {"month": "Nov", "accepted_without_errors": 0, "accepted_with_errors": 0}, - {"month": "Dec", "accepted_without_errors": 0, "accepted_with_errors": 0}, - ], - } - - assert Tribal_TANF_T1.objects.all().count() == 1 - assert Tribal_TANF_T2.objects.all().count() == 1 - assert Tribal_TANF_T3.objects.all().count() == 2 - - t1_objs = Tribal_TANF_T1.objects.all().order_by("CASH_AMOUNT") - t2_objs = Tribal_TANF_T2.objects.all().order_by("MONTHS_FED_TIME_LIMIT") - t3_objs = Tribal_TANF_T3.objects.all().order_by("EDUCATION_LEVEL") - - t1 = t1_objs.first() - t2 = t2_objs.first() - t3 = t3_objs.last() - - assert t1.CASH_AMOUNT == 502 - assert t2.MONTHS_FED_TIME_LIMIT == " 0" - assert t3.EDUCATION_LEVEL == "98" - - -@pytest.mark.django_db() -def test_parse_tribal_section_1_inconsistency_file( - tribal_section_1_inconsistency_file, dfs -): - """Test parsing inconsistent Tribal TANF Section 1 submission.""" - parser = ParserFactory.get_instance( - datafile=tribal_section_1_inconsistency_file, - dfs=dfs, - section=tribal_section_1_inconsistency_file.section, - program_type=tribal_section_1_inconsistency_file.program_type, - ) - parser.parse_and_validate() + parse_datafile(dfs, ssp_section2_rec_oadsi_file) + parser_errors = ParserError.objects.filter(file=ssp_section2_rec_oadsi_file) - assert Tribal_TANF_T1.objects.all().count() == 0 + assert parser_errors.count() == 0 - parser_errors = ParserError.objects.filter(file=tribal_section_1_inconsistency_file) - assert parser_errors.count() == 1 + @pytest.mark.django_db() + def test_parse_ssp_section2_file(self, ssp_section2_file, dfs): + """Test parsing SSP Section 2 submission.""" + ssp_section2_file.year = 2019 + ssp_section2_file.quarter = "Q1" - assert ( - parser_errors.first().error_message - == "Tribe Code (142) inconsistency with Program Type (TAN) " - + "and FIPS Code (01)." - ) + dfs.datafile = ssp_section2_file + parse_datafile(dfs, ssp_section2_file) -@pytest.mark.django_db() -def test_parse_tribal_section_2_file(tribal_section_2_file, dfs): - """Test parsing Tribal TANF Section 2 submission.""" - tribal_section_2_file.year = 2020 - tribal_section_2_file.quarter = "Q1" + dfs.status = dfs.get_status() + dfs.case_aggregates = aggregates.case_aggregates_by_month(dfs.datafile, dfs.status) + for dfs_case_aggregate in dfs.case_aggregates["months"]: + assert dfs_case_aggregate["accepted_without_errors"] == 0 + assert dfs_case_aggregate["accepted_with_errors"] in [75, 78] + assert dfs_case_aggregate["month"] in ["Oct", "Nov", "Dec"] + assert dfs.get_status() == DataFileSummary.Status.PARTIALLY_ACCEPTED - dfs.datafile = tribal_section_2_file + m4_objs = SSP_M4.objects.all().order_by("id") + m5_objs = SSP_M5.objects.all().order_by("AMOUNT_EARNED_INCOME") - parser = ParserFactory.get_instance( - datafile=tribal_section_2_file, - dfs=dfs, - section=tribal_section_2_file.section, - program_type=tribal_section_2_file.program_type, - ) - parser.parse_and_validate() - - dfs.status = dfs.get_status() - dfs.case_aggregates = aggregates.case_aggregates_by_month(dfs.datafile, dfs.status) - assert dfs.case_aggregates == { - "rejected": 0, - "months": [ - {"accepted_without_errors": 3, "accepted_with_errors": 0, "month": "Oct"}, - {"accepted_without_errors": 3, "accepted_with_errors": 0, "month": "Nov"}, - {"accepted_without_errors": 0, "accepted_with_errors": 0, "month": "Dec"}, - ], - } + expected_m4_count = 231 + expected_m5_count = 703 - assert dfs.get_status() == DataFileSummary.Status.ACCEPTED + assert SSP_M4.objects.count() == expected_m4_count + assert SSP_M5.objects.count() == expected_m5_count - assert Tribal_TANF_T4.objects.all().count() == 6 - assert Tribal_TANF_T5.objects.all().count() == 13 + m4 = m4_objs.first() + assert m4.DISPOSITION == 1 + assert m4.REC_SUB_CC == 3 - t4_objs = Tribal_TANF_T4.objects.all().order_by("CLOSURE_REASON") - t5_objs = Tribal_TANF_T5.objects.all().order_by("COUNTABLE_MONTH_FED_TIME") + m5 = m5_objs.first() + assert m5.FAMILY_AFFILIATION == 1 + assert m5.AMOUNT_EARNED_INCOME == "0000" + assert m5.AMOUNT_UNEARNED_INCOME == "0000" - t4 = t4_objs.first() - t5 = t5_objs.last() + @pytest.mark.django_db() + def test_parse_ssp_section3_file(self, ssp_section3_file, dfs): + """Test parsing TANF Section 3 submission.""" + ssp_section3_file.year = 2022 + ssp_section3_file.quarter = "Q1" - assert t4.CLOSURE_REASON == "15" - assert t5.COUNTABLE_MONTH_FED_TIME == " 8" + dfs.datafile = ssp_section3_file + parse_datafile(dfs, ssp_section3_file) -@pytest.mark.django_db() -def test_parse_tribal_section_3_file(tribal_section_3_file, dfs): - """Test parsing Tribal TANF Section 3 submission.""" - tribal_section_3_file.year = 2022 - tribal_section_3_file.quarter = "Q1" + dfs.status = dfs.get_status() + dfs.case_aggregates = aggregates.total_errors_by_month(dfs.datafile, dfs.status) + assert dfs.case_aggregates == { + "months": [ + {"month": "Oct", "total_errors": 0}, + {"month": "Nov", "total_errors": 0}, + {"month": "Dec", "total_errors": 0}, + ] + } - dfs.datafile = tribal_section_3_file + assert dfs.get_status() == DataFileSummary.Status.ACCEPTED - parser = ParserFactory.get_instance( - datafile=tribal_section_3_file, - dfs=dfs, - section=tribal_section_3_file.section, - program_type=tribal_section_3_file.program_type, - ) - parser.parse_and_validate() + m6_objs = SSP_M6.objects.all().order_by("RPT_MONTH_YEAR") + assert m6_objs.count() == 3 - dfs.status = dfs.get_status() - dfs.case_aggregates = aggregates.total_errors_by_month(dfs.datafile, dfs.status) - assert dfs.case_aggregates == { - "months": [ - {"month": "Oct", "total_errors": 0}, - {"month": "Nov", "total_errors": 0}, - {"month": "Dec", "total_errors": 0}, - ] - } + parser_errors = ParserError.objects.filter(file=ssp_section3_file) + assert parser_errors.count() == 0 - assert dfs.get_status() == DataFileSummary.Status.ACCEPTED + first = m6_objs.first() + second = m6_objs[1] + third = m6_objs[2] - assert Tribal_TANF_T6.objects.all().count() == 3 + assert first.RPT_MONTH_YEAR == 202110 + assert second.RPT_MONTH_YEAR == 202111 + assert third.RPT_MONTH_YEAR == 202112 - t6_objs = Tribal_TANF_T6.objects.all().order_by("NUM_APPLICATIONS") + assert first.SSPMOE_FAMILIES == 15869 + assert second.SSPMOE_FAMILIES == 16008 + assert third.SSPMOE_FAMILIES == 15956 - t6 = t6_objs.first() + assert first.NUM_RECIPIENTS == 51355 + assert second.NUM_RECIPIENTS == 51696 + assert third.NUM_RECIPIENTS == 51348 - assert t6.NUM_APPLICATIONS == 1 - assert t6.NUM_FAMILIES == 41 - assert t6.NUM_CLOSED_CASES == 3 + @pytest.mark.django_db + def test_rpt_month_year_mismatch(self, header_datafile, dfs): + """Test that the rpt_month_year mismatch error is raised.""" + datafile = header_datafile + datafile.section = "Active Case Data" + # test_datafile fixture uses create_test_data_file which assigns + # a default year / quarter of 2021 / Q1 + datafile.year = 2021 + datafile.quarter = "Q1" + datafile.save() -@pytest.mark.django_db() -def test_parse_tribal_section_4_file(tribal_section_4_file, dfs): - """Test parsing Tribal TANF Section 4 submission.""" - tribal_section_4_file.year = 2022 - tribal_section_4_file.quarter = "Q1" + dfs.datafile = header_datafile + dfs.save() - dfs.datafile = tribal_section_4_file + parse_datafile(dfs, datafile) - parser = ParserFactory.get_instance( - datafile=tribal_section_4_file, - dfs=dfs, - section=tribal_section_4_file.section, - program_type=tribal_section_4_file.program_type, - ) - parser.parse_and_validate() - - dfs.status = dfs.get_status() - dfs.case_aggregates = aggregates.total_errors_by_month(dfs.datafile, dfs.status) - assert dfs.case_aggregates == { - "months": [ - {"month": "Oct", "total_errors": 0}, - {"month": "Nov", "total_errors": 0}, - {"month": "Dec", "total_errors": 0}, - ] - } - - assert Tribal_TANF_T7.objects.all().count() == 18 - - t7_objs = Tribal_TANF_T7.objects.all().order_by("FAMILIES_MONTH") - - first = t7_objs.first() - sixth = t7_objs[5] - - assert first.RPT_MONTH_YEAR == 202111 - assert sixth.RPT_MONTH_YEAR == 202112 - - assert first.TDRS_SECTION_IND == "2" - assert sixth.TDRS_SECTION_IND == "2" - - assert first.FAMILIES_MONTH == 274 - assert sixth.FAMILIES_MONTH == 499 - - -@pytest.mark.parametrize( - "file_fixture, result, number_of_errors, error_message", - [ - ("second_child_only_space_t3_file", 1, 0, ""), - ("one_child_t3_file", 1, 0, ""), - ("t3_file", 1, 0, ""), - ( - "t3_file_two_child", - 1, - 1, - "The second child record is too short at 97 characters" - + " and must be at least 101 characters.", - ), - ("t3_file_two_child_with_space_filled", 2, 0, ""), - ( - "two_child_second_filled", - 2, - 8, - "T3 Item 68 (Date of Birth): Year 6 must be larger than 1900.", - ), - ("t3_file_zero_filled_second", 1, 0, ""), - ], -) -@pytest.mark.django_db() -def test_misformatted_multi_records( - file_fixture, result, number_of_errors, error_message, request, dfs -): - """Test that (not space filled) multi-records are caught.""" - file_fixture = request.getfixturevalue(file_fixture) - dfs.datafile = file_fixture - parser = ParserFactory.get_instance( - datafile=file_fixture, - dfs=dfs, - section=file_fixture.section, - program_type=file_fixture.program_type, - ) - parser.parse_and_validate() - parser_errors = ParserError.objects.filter(file=file_fixture) - t3 = TANF_T3.objects.all() - assert t3.count() == result - - parser_errors = ParserError.objects.all() - assert parser_errors.count() == number_of_errors - if number_of_errors > 0: - error_messages = [parser_error.error_message for parser_error in parser_errors] - assert error_message in error_messages - - parser_errors = ( - ParserError.objects.all() - .exclude( - # exclude extraneous cat 4 errors - error_type=ParserErrorCategoryChoices.CASE_CONSISTENCY + parser_errors = ParserError.objects.filter(file=datafile) + assert parser_errors.count() == 2 + assert ( + parser_errors.first().error_type == ParserErrorCategoryChoices.CASE_CONSISTENCY ) - .exclude(error_message="No records created.") - ) - assert parser_errors.count() == number_of_errors + datafile.year = 2023 + datafile.save() + parse_datafile(dfs, datafile) -@pytest.mark.django_db() -def test_empty_t4_t5_values(t4_t5_empty_values, dfs): - """Test that empty field values for un-required fields parse.""" - dfs.datafile = t4_t5_empty_values - parser = ParserFactory.get_instance( - datafile=t4_t5_empty_values, - dfs=dfs, - section=t4_t5_empty_values.section, - program_type=t4_t5_empty_values.program_type, - ) - parser.parse_and_validate() - parser_errors = ParserError.objects.filter(file=t4_t5_empty_values) - t4 = TANF_T4.objects.all() - t5 = TANF_T5.objects.all() - assert t4.count() == 1 - assert t4[0].STRATUM is None - logger.info(t4[0].__dict__) - assert t5.count() == 1 - assert parser_errors[0].error_message == ( - "T4 Item 10 (Received Subsidized Housing): 3 is not in range [1, 2]." - ) + parser_errors = ParserError.objects.filter(file=datafile).order_by("-id") + assert parser_errors.count() == 3 + err = parser_errors.first() + assert err.error_type == ParserErrorCategoryChoices.PRE_CHECK + assert ( + err.error_message + == "Submitted reporting year:2020, quarter:Q4 doesn't" + + " match file reporting year:2023, quarter:Q1." + ) -@pytest.mark.django_db() -def test_parse_t2_invalid_dob(t2_invalid_dob_file, dfs): - """Test parsing a TANF T2 record with an invalid DOB.""" - dfs.datafile = t2_invalid_dob_file - t2_invalid_dob_file.year = 2021 - t2_invalid_dob_file.quarter = "Q1" - dfs.save() + @pytest.mark.django_db() + def test_parse_tribal_section_1_file(self, tribal_section_1_file, dfs): + """Test parsing Tribal TANF Section 1 submission.""" + tribal_section_1_file.year = 2022 + tribal_section_1_file.quarter = "Q1" + tribal_section_1_file.save() + + dfs.datafile = tribal_section_1_file + + parse_datafile(dfs, tribal_section_1_file) + + dfs.status = dfs.get_status() + assert dfs.status == DataFileSummary.Status.ACCEPTED + dfs.case_aggregates = aggregates.case_aggregates_by_month(dfs.datafile, dfs.status) + assert dfs.case_aggregates == { + "rejected": 0, + "months": [ + {"month": "Oct", "accepted_without_errors": 1, "accepted_with_errors": 0}, + {"month": "Nov", "accepted_without_errors": 0, "accepted_with_errors": 0}, + {"month": "Dec", "accepted_without_errors": 0, "accepted_with_errors": 0}, + ], + } - parser = ParserFactory.get_instance( - datafile=t2_invalid_dob_file, - dfs=dfs, - section=t2_invalid_dob_file.section, - program_type=t2_invalid_dob_file.program_type, - ) - parser.parse_and_validate() + assert Tribal_TANF_T1.objects.all().count() == 1 + assert Tribal_TANF_T2.objects.all().count() == 1 + assert Tribal_TANF_T3.objects.all().count() == 2 - parser_errors = ParserError.objects.filter(file=t2_invalid_dob_file).order_by("pk") + t1_objs = Tribal_TANF_T1.objects.all().order_by("CASH_AMOUNT") + t2_objs = Tribal_TANF_T2.objects.all().order_by("MONTHS_FED_TIME_LIMIT") + t3_objs = Tribal_TANF_T3.objects.all().order_by("EDUCATION_LEVEL") - month_error = parser_errors[2] - year_error = parser_errors[1] - digits_error = parser_errors[0] + t1 = t1_objs.first() + t2 = t2_objs.first() + t3 = t3_objs.last() - assert ( - month_error.error_message - == "T2 Item 32 (Date of Birth): $9 is not a valid month." - ) - assert ( - year_error.error_message - == "T2 Item 32 (Date of Birth): Year Q897 must be larger than 1900." - ) - assert ( - digits_error.error_message - == "T2 Item 32 (Date of Birth): Q897$9 3 does not have exactly 8 digits." - ) + assert t1.CASH_AMOUNT == 502 + assert t2.MONTHS_FED_TIME_LIMIT == " 0" + assert t3.EDUCATION_LEVEL == "98" + @pytest.mark.django_db() + def test_parse_tribal_section_1_inconsistency_file( + self, + tribal_section_1_inconsistency_file, dfs + ): + """Test parsing inconsistent Tribal TANF Section 1 submission.""" + parse_datafile(dfs, tribal_section_1_inconsistency_file) -@pytest.mark.django_db() -def test_parse_tanf_section4_file_with_errors(tanf_section_4_file_with_errors, dfs): - """Test parsing TANF Section 4 submission.""" - tanf_section_4_file_with_errors.year = 2022 - tanf_section_4_file_with_errors.quarter = "Q1" - dfs.datafile = tanf_section_4_file_with_errors + assert Tribal_TANF_T1.objects.all().count() == 0 - parser = ParserFactory.get_instance( - datafile=tanf_section_4_file_with_errors, - dfs=dfs, - section=tanf_section_4_file_with_errors.section, - program_type=tanf_section_4_file_with_errors.program_type, - ) - parser.parse_and_validate() + parser_errors = ParserError.objects.filter(file=tribal_section_1_inconsistency_file) + assert parser_errors.count() == 1 - dfs.status = dfs.get_status() - dfs.case_aggregates = aggregates.total_errors_by_month(dfs.datafile, dfs.status) - assert dfs.case_aggregates == { - "months": [ - {"month": "Oct", "total_errors": 2}, - {"month": "Nov", "total_errors": 3}, - {"month": "Dec", "total_errors": 2}, - ] - } + assert ( + parser_errors.first().error_message + == "Tribe Code (142) inconsistency with Program Type (TAN) " + + "and FIPS Code (01)." + ) - assert dfs.get_status() == DataFileSummary.Status.ACCEPTED_WITH_ERRORS + @pytest.mark.django_db() + def test_parse_tribal_section_2_file(self, tribal_section_2_file, dfs): + """Test parsing Tribal TANF Section 2 submission.""" + tribal_section_2_file.year = 2020 + tribal_section_2_file.quarter = "Q1" + + dfs.datafile = tribal_section_2_file + + parse_datafile(dfs, tribal_section_2_file) + + dfs.status = dfs.get_status() + dfs.case_aggregates = aggregates.case_aggregates_by_month(dfs.datafile, dfs.status) + assert dfs.case_aggregates == { + "rejected": 0, + "months": [ + {"accepted_without_errors": 3, "accepted_with_errors": 0, "month": "Oct"}, + {"accepted_without_errors": 3, "accepted_with_errors": 0, "month": "Nov"}, + {"accepted_without_errors": 0, "accepted_with_errors": 0, "month": "Dec"}, + ], + } - assert TANF_T7.objects.all().count() == 18 + assert dfs.get_status() == DataFileSummary.Status.ACCEPTED - parser_errors = ParserError.objects.filter(file=tanf_section_4_file_with_errors) + assert Tribal_TANF_T4.objects.all().count() == 6 + assert Tribal_TANF_T5.objects.all().count() == 13 - assert parser_errors.count() == 7 + t4_objs = Tribal_TANF_T4.objects.all().order_by("CLOSURE_REASON") + t5_objs = Tribal_TANF_T5.objects.all().order_by("COUNTABLE_MONTH_FED_TIME") - t7_objs = TANF_T7.objects.all().order_by("FAMILIES_MONTH") + t4 = t4_objs.first() + t5 = t5_objs.last() - first = t7_objs.first() - sixth = t7_objs[5] + assert t4.CLOSURE_REASON == "15" + assert t5.COUNTABLE_MONTH_FED_TIME == " 8" - assert first.RPT_MONTH_YEAR == 202111 - assert sixth.RPT_MONTH_YEAR == 202110 + @pytest.mark.django_db() + def test_parse_tribal_section_3_file(self, tribal_section_3_file, dfs): + """Test parsing Tribal TANF Section 3 submission.""" + tribal_section_3_file.year = 2022 + tribal_section_3_file.quarter = "Q1" - assert first.TDRS_SECTION_IND == "1" - assert sixth.TDRS_SECTION_IND == "1" + dfs.datafile = tribal_section_3_file - assert first.FAMILIES_MONTH == 0 - assert sixth.FAMILIES_MONTH == 446 + parse_datafile(dfs, tribal_section_3_file) + dfs.status = dfs.get_status() + dfs.case_aggregates = aggregates.total_errors_by_month(dfs.datafile, dfs.status) + assert dfs.case_aggregates == { + "months": [ + {"month": "Oct", "total_errors": 0}, + {"month": "Nov", "total_errors": 0}, + {"month": "Dec", "total_errors": 0}, + ] + } -@pytest.mark.django_db() -def test_parse_no_records_file(no_records_file, dfs): - """Test parsing TANF Section 4 submission.""" - dfs.datafile = no_records_file - parser = ParserFactory.get_instance( - datafile=no_records_file, - dfs=dfs, - section=no_records_file.section, - program_type=no_records_file.program_type, - ) - parser.parse_and_validate() + assert dfs.get_status() == DataFileSummary.Status.ACCEPTED - dfs.status = dfs.get_status() - assert dfs.status == DataFileSummary.Status.REJECTED + assert Tribal_TANF_T6.objects.all().count() == 3 - errors = ParserError.objects.filter(file=no_records_file) + t6_objs = Tribal_TANF_T6.objects.all().order_by("NUM_APPLICATIONS") - assert errors.count() == 1 + t6 = t6_objs.first() - error = errors.first() - assert error.error_message == "No records created." - assert error.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert error.content_type is None - assert error.object_id is None + assert t6.NUM_APPLICATIONS == 1 + assert t6.NUM_FAMILIES == 41 + assert t6.NUM_CLOSED_CASES == 3 + @pytest.mark.django_db() + def test_parse_tribal_section_4_file(self, tribal_section_4_file, dfs): + """Test parsing Tribal TANF Section 4 submission.""" + tribal_section_4_file.year = 2022 + tribal_section_4_file.quarter = "Q1" -@pytest.mark.django_db -def test_parse_aggregates_rejected_datafile(aggregates_rejected_datafile, dfs): - """Test record rejection counting when record has more than one preparsing error.""" - aggregates_rejected_datafile.year = 2021 - aggregates_rejected_datafile.quarter = "Q1" + dfs.datafile = tribal_section_4_file - print(aggregates_rejected_datafile) + parse_datafile(dfs, tribal_section_4_file) - dfs.datafile = aggregates_rejected_datafile + dfs.status = dfs.get_status() + dfs.case_aggregates = aggregates.total_errors_by_month(dfs.datafile, dfs.status) + assert dfs.case_aggregates == { + "months": [ + {"month": "Oct", "total_errors": 0}, + {"month": "Nov", "total_errors": 0}, + {"month": "Dec", "total_errors": 0}, + ] + } - parser = ParserFactory.get_instance( - datafile=aggregates_rejected_datafile, - dfs=dfs, - section=aggregates_rejected_datafile.section, - program_type=aggregates_rejected_datafile.program_type, - ) - parser.parse_and_validate() - - dfs.status = dfs.get_status() - assert dfs.status == DataFileSummary.Status.REJECTED - dfs.case_aggregates = aggregates.case_aggregates_by_month(dfs.datafile, dfs.status) - assert dfs.case_aggregates == { - "months": [ - { - "month": "Oct", - "accepted_without_errors": "N/A", - "accepted_with_errors": "N/A", - }, - { - "month": "Nov", - "accepted_without_errors": "N/A", - "accepted_with_errors": "N/A", - }, - { - "month": "Dec", - "accepted_without_errors": "N/A", - "accepted_with_errors": "N/A", - }, + assert Tribal_TANF_T7.objects.all().count() == 18 + + t7_objs = Tribal_TANF_T7.objects.all().order_by("FAMILIES_MONTH") + + first = t7_objs.first() + sixth = t7_objs[5] + + assert first.RPT_MONTH_YEAR == 202111 + assert sixth.RPT_MONTH_YEAR == 202112 + + assert first.TDRS_SECTION_IND == "2" + assert sixth.TDRS_SECTION_IND == "2" + + assert first.FAMILIES_MONTH == 274 + assert sixth.FAMILIES_MONTH == 499 + + @pytest.mark.parametrize( + "file_fixture, result, number_of_errors, error_message", + [ + ("second_child_only_space_t3_file", 1, 0, ""), + ("one_child_t3_file", 1, 0, ""), + ("t3_file", 1, 0, ""), + ( + "t3_file_two_child", + 1, + 1, + "The second child record is too short at 97 characters" + + " and must be at least 101 characters.", + ), + ("t3_file_two_child_with_space_filled", 2, 0, ""), + ( + "two_child_second_filled", + 2, + 8, + "T3 Item 68 (Date of Birth): Year 6 must be larger than 1900.", + ), + ("t3_file_zero_filled_second", 1, 0, ""), ], - "rejected": 1, - } - - errors = ParserError.objects.filter(file=aggregates_rejected_datafile).order_by( - "id" - ) - - assert errors.count() == 3 - record_precheck_errors = errors.filter(row_number=2) - assert record_precheck_errors.count() == 2 - for error in record_precheck_errors: - assert error.error_type == ParserErrorCategoryChoices.RECORD_PRE_CHECK - - assert errors.last().error_type == ParserErrorCategoryChoices.PRE_CHECK - - assert TANF_T2.objects.count() == 0 - - -@pytest.mark.django_db() -def test_parse_tanf_section_1_file_with_bad_update_indicator( - tanf_section_1_file_with_bad_update_indicator, dfs -): - """Test parsing TANF Section 1 submission update indicator.""" - dfs.datafile = tanf_section_1_file_with_bad_update_indicator - - parser = ParserFactory.get_instance( - datafile=tanf_section_1_file_with_bad_update_indicator, - dfs=dfs, - section=tanf_section_1_file_with_bad_update_indicator.section, - program_type=tanf_section_1_file_with_bad_update_indicator.program_type, ) - parser.parse_and_validate() - - parser_errors = ParserError.objects.filter( - file=tanf_section_1_file_with_bad_update_indicator, - ) - - assert dfs.get_status() == DataFileSummary.Status.ACCEPTED_WITH_ERRORS - - assert parser_errors.count() == 5 - - error_messages = [error.error_message for error in parser_errors] - - assert ( - "HEADER Update Indicator must be set to D instead of U. Please review" - + " Exporting Complete Data Using FTANF in the Knowledge Center." - in error_messages - ) - - -@pytest.mark.django_db() -def test_parse_tribal_section_4_bad_quarter(tribal_section_4_bad_quarter, dfs): - """Test handling invalid quarter value that raises a ValueError exception.""" - tribal_section_4_bad_quarter.year = 2021 - tribal_section_4_bad_quarter.quarter = "Q1" - dfs.datafile = tribal_section_4_bad_quarter - - parser = ParserFactory.get_instance( - datafile=tribal_section_4_bad_quarter, - dfs=dfs, - section=tribal_section_4_bad_quarter.section, - program_type=tribal_section_4_bad_quarter.program_type, - ) - parser.parse_and_validate() - parser_errors = ParserError.objects.filter( - file=tribal_section_4_bad_quarter - ).order_by("id") - - assert parser_errors.count() == 3 - - parser_errors.first().error_message == ( - "T7: 2020 is invalid. Calendar Quarter must be a numeric " - "representing the Calendar Year and Quarter formatted as YYYYQ" - ) - - Tribal_TANF_T7.objects.count() == 0 - - -@pytest.mark.django_db() -def test_parse_t3_cat2_invalid_citizenship(t3_cat2_invalid_citizenship_file, dfs): - """Test parsing a TANF T3 record with an invalid CITIZENSHIP_STATUS.""" - dfs.datafile = t3_cat2_invalid_citizenship_file - t3_cat2_invalid_citizenship_file.year = 2021 - t3_cat2_invalid_citizenship_file.quarter = "Q1" - dfs.save() - - parser = ParserFactory.get_instance( - datafile=t3_cat2_invalid_citizenship_file, - dfs=dfs, - section=t3_cat2_invalid_citizenship_file.section, - program_type=t3_cat2_invalid_citizenship_file.program_type, - ) - parser.parse_and_validate() - - exclusion = Query(error_type=ParserErrorCategoryChoices.CASE_CONSISTENCY) | Query( - error_type=ParserErrorCategoryChoices.PRE_CHECK - ) - - parser_errors = ( - ParserError.objects.filter(file=t3_cat2_invalid_citizenship_file) - .exclude(exclusion) - .order_by("pk") - ) - - # no errors expected as fields are not required - assert parser_errors.count() == 0 + @pytest.mark.django_db() + def test_misformatted_multi_records( + self, + file_fixture, result, number_of_errors, error_message, request, dfs + ): + """Test that (not space filled) multi-records are caught.""" + file_fixture = request.getfixturevalue(file_fixture) + dfs.datafile = file_fixture + parse_datafile(dfs, file_fixture) + parser_errors = ParserError.objects.filter(file=file_fixture) + t3 = TANF_T3.objects.all() + assert t3.count() == result + + parser_errors = ParserError.objects.all() + assert parser_errors.count() == number_of_errors + if number_of_errors > 0: + error_messages = [parser_error.error_message for parser_error in parser_errors] + assert error_message in error_messages + + parser_errors = ( + ParserError.objects.all() + .exclude( + # exclude extraneous cat 4 errors + error_type=ParserErrorCategoryChoices.CASE_CONSISTENCY + ) + .exclude(error_message="No records created.") + ) + assert parser_errors.count() == number_of_errors + + @pytest.mark.django_db() + def test_empty_t4_t5_values(self, t4_t5_empty_values, dfs): + """Test that empty field values for un-required fields parse.""" + dfs.datafile = t4_t5_empty_values + parse_datafile(dfs, t4_t5_empty_values) + parser_errors = ParserError.objects.filter(file=t4_t5_empty_values) + t4 = TANF_T4.objects.all() + t5 = TANF_T5.objects.all() + assert t4.count() == 1 + assert t4[0].STRATUM is None + logger.info(t4[0].__dict__) + assert t5.count() == 1 + assert parser_errors[0].error_message == ( + "T4 Item 10 (Received Subsidized Housing): 3 is not in range [1, 2]." + ) -@pytest.mark.django_db() -def test_parse_m2_cat2_invalid_37_38_39_file(m2_cat2_invalid_37_38_39_file, dfs): - """Test parsing an SSP M2 file with an invalid EDUCATION_LEVEL, CITIZENSHIP_STATUS, COOPERATION_CHILD_SUPPORT.""" - dfs.datafile = m2_cat2_invalid_37_38_39_file - m2_cat2_invalid_37_38_39_file.year = 2024 - m2_cat2_invalid_37_38_39_file.quarter = "Q1" - dfs.save() + @pytest.mark.django_db() + def test_parse_t2_invalid_dob(self, t2_invalid_dob_file, dfs): + """Test parsing a TANF T2 record with an invalid DOB.""" + dfs.datafile = t2_invalid_dob_file + t2_invalid_dob_file.year = 2021 + t2_invalid_dob_file.quarter = "Q1" + dfs.save() - parser = ParserFactory.get_instance( - datafile=m2_cat2_invalid_37_38_39_file, - dfs=dfs, - section=m2_cat2_invalid_37_38_39_file.section, - program_type=m2_cat2_invalid_37_38_39_file.program_type, - ) - parser.parse_and_validate() + parse_datafile(dfs, t2_invalid_dob_file) - exclusion = Query(error_type=ParserErrorCategoryChoices.CASE_CONSISTENCY) | Query( - error_type=ParserErrorCategoryChoices.PRE_CHECK - ) + parser_errors = ParserError.objects.filter(file=t2_invalid_dob_file).order_by("pk") - parser_errors = ( - ParserError.objects.filter(file=m2_cat2_invalid_37_38_39_file) - .exclude(exclusion) - .order_by("pk") - ) + month_error = parser_errors[2] + year_error = parser_errors[1] + digits_error = parser_errors[0] - # no errors expected as fields are not required - assert parser_errors.count() == 0 + assert ( + month_error.error_message + == "T2 Item 32 (Date of Birth): $9 is not a valid month." + ) + assert ( + year_error.error_message + == "T2 Item 32 (Date of Birth): Year Q897 must be larger than 1900." + ) + assert ( + digits_error.error_message + == "T2 Item 32 (Date of Birth): Q897$9 3 does not have exactly 8 digits." + ) + @pytest.mark.django_db() + def test_parse_tanf_section4_file_with_errors(self, tanf_section_4_file_with_errors, dfs): + """Test parsing TANF Section 4 submission.""" + tanf_section_4_file_with_errors.year = 2022 + tanf_section_4_file_with_errors.quarter = "Q1" + dfs.datafile = tanf_section_4_file_with_errors + + parse_datafile(dfs, tanf_section_4_file_with_errors) + + dfs.status = dfs.get_status() + dfs.case_aggregates = aggregates.total_errors_by_month(dfs.datafile, dfs.status) + assert dfs.case_aggregates == { + "months": [ + {"month": "Oct", "total_errors": 2}, + {"month": "Nov", "total_errors": 3}, + {"month": "Dec", "total_errors": 2}, + ] + } -@pytest.mark.django_db() -def test_parse_m3_cat2_invalid_68_69_file(m3_cat2_invalid_68_69_file, dfs): - """Test parsing an SSP M3 file with an invalid EDUCATION_LEVEL and CITIZENSHIP_STATUS.""" - dfs.datafile = m3_cat2_invalid_68_69_file - m3_cat2_invalid_68_69_file.year = 2024 - m3_cat2_invalid_68_69_file.quarter = "Q1" - dfs.save() + assert dfs.get_status() == DataFileSummary.Status.ACCEPTED_WITH_ERRORS - parser = ParserFactory.get_instance( - datafile=m3_cat2_invalid_68_69_file, - dfs=dfs, - section=m3_cat2_invalid_68_69_file.section, - program_type=m3_cat2_invalid_68_69_file.program_type, - ) - parser.parse_and_validate() + assert TANF_T7.objects.all().count() == 18 - exclusion = Query(error_type=ParserErrorCategoryChoices.CASE_CONSISTENCY) | Query( - error_type=ParserErrorCategoryChoices.PRE_CHECK - ) + parser_errors = ParserError.objects.filter(file=tanf_section_4_file_with_errors) - parser_errors = ( - ParserError.objects.filter(file=m3_cat2_invalid_68_69_file) - .exclude(exclusion) - .order_by("pk") - ) + assert parser_errors.count() == 7 - assert parser_errors.count() == 2 + t7_objs = TANF_T7.objects.all().order_by("FAMILIES_MONTH") - error_msgs = { - "Item 68 (Educational Level) 00 must be between 1 and 16 or must be between 98 and 99.", - "Item 68 (Educational Level) 00 must be between 1 and 16 or must be between 98 and 99.", - } + first = t7_objs.first() + sixth = t7_objs[5] - for e in parser_errors: - assert e.error_message in error_msgs + assert first.RPT_MONTH_YEAR == 202111 + assert sixth.RPT_MONTH_YEAR == 202110 + assert first.TDRS_SECTION_IND == "1" + assert sixth.TDRS_SECTION_IND == "1" -@pytest.mark.django_db() -def test_parse_m5_cat2_invalid_23_24_file(m5_cat2_invalid_23_24_file, dfs): - """Test parsing an SSP M5 file with an invalid EDUCATION_LEVEL and CITIZENSHIP_STATUS.""" - dfs.datafile = m5_cat2_invalid_23_24_file - m5_cat2_invalid_23_24_file.year = 2019 - m5_cat2_invalid_23_24_file.quarter = "Q1" - dfs.save() + assert first.FAMILIES_MONTH == 0 + assert sixth.FAMILIES_MONTH == 446 - parser = ParserFactory.get_instance( - datafile=m5_cat2_invalid_23_24_file, - dfs=dfs, - section=m5_cat2_invalid_23_24_file.section, - program_type=m5_cat2_invalid_23_24_file.program_type, - ) - parser.parse_and_validate() + @pytest.mark.django_db() + def test_parse_no_records_file(self, no_records_file, dfs): + """Test parsing TANF Section 4 submission.""" + dfs.datafile = no_records_file + parse_datafile(dfs, no_records_file) - exclusion = Query(error_type=ParserErrorCategoryChoices.CASE_CONSISTENCY) | Query( - error_type=ParserErrorCategoryChoices.PRE_CHECK - ) + dfs.status = dfs.get_status() + assert dfs.status == DataFileSummary.Status.REJECTED - parser_errors = ( - ParserError.objects.filter(file=m5_cat2_invalid_23_24_file) - .exclude(exclusion) - .order_by("pk") - ) + errors = ParserError.objects.filter(file=no_records_file) - assert parser_errors.count() == 0 + assert errors.count() == 1 + error = errors.first() + assert error.error_message == "No records created." + assert error.error_type == ParserErrorCategoryChoices.PRE_CHECK + assert error.content_type is None + assert error.object_id is None -@pytest.mark.django_db() -def test_zero_filled_fips_code_file(test_file_zero_filled_fips_code, dfs): - """Test parsing a file with zero filled FIPS code.""" - # TODO: this test can be merged as parametrized test with "test_parse_small_correct_file" - dfs.datafile = test_file_zero_filled_fips_code - test_file_zero_filled_fips_code.year = 2024 - test_file_zero_filled_fips_code.quarter = "Q2" - test_file_zero_filled_fips_code.save() + @pytest.mark.django_db + def test_parse_aggregates_rejected_datafile(self, aggregates_rejected_datafile, dfs): + """Test record rejection counting when record has more than one preparsing error.""" + aggregates_rejected_datafile.year = 2021 + aggregates_rejected_datafile.quarter = "Q1" - parser = ParserFactory.get_instance( - datafile=test_file_zero_filled_fips_code, - dfs=dfs, - section=test_file_zero_filled_fips_code.section, - program_type=test_file_zero_filled_fips_code.program_type, - ) - parser.parse_and_validate() + print(aggregates_rejected_datafile) - parser_errors = ParserError.objects.filter(file=test_file_zero_filled_fips_code) - assert ( - "T1 Item 2 (County FIPS Code): field is required but a value was not" - + " provided." - in [i.error_message for i in parser_errors] - ) + dfs.datafile = aggregates_rejected_datafile + parse_datafile(dfs, aggregates_rejected_datafile) -@pytest.mark.parametrize( - "file, batch_size, model, record_type, num_errors", - [ - ("tanf_s1_exact_dup_file", 10000, TANF_T1, "T1", 3), - ( - "tanf_s1_exact_dup_file", - 1, - TANF_T1, - "T1", - 3, - ), # This forces an in memory and database deletion of records. - ("tanf_s2_exact_dup_file", 10000, TANF_T4, "T4", 3), - ( - "tanf_s2_exact_dup_file", - 1, - TANF_T4, - "T4", - 3, - ), # This forces an in memory and database deletion of records. - ("tanf_s3_exact_dup_file", 10000, TANF_T6, "T6", 3), - ( - "tanf_s3_exact_dup_file", - 1, - TANF_T6, - "T6", - 3, - ), # This forces an in memory and database deletion of records. - ("tanf_s4_exact_dup_file", 10000, TANF_T7, "T7", 18), - ( - "tanf_s4_exact_dup_file", - 1, - TANF_T7, - "T7", - 18, - ), # This forces an in memory and database deletion of records. - ("ssp_s1_exact_dup_file", 10000, SSP_M1, "M1", 3), - ( - "ssp_s1_exact_dup_file", - 1, - SSP_M1, - "M1", - 3, - ), # This forces an in memory and database deletion of records. - ("ssp_s2_exact_dup_file", 10000, SSP_M4, "M4", 3), - ( - "ssp_s2_exact_dup_file", - 1, - SSP_M4, - "M4", - 3, - ), # This forces an in memory and database deletion of records. - ("ssp_s3_exact_dup_file", 10000, SSP_M6, "M6", 3), - ( - "ssp_s3_exact_dup_file", - 1, - SSP_M6, - "M6", - 3, - ), # This forces an in memory and database deletion of records. - ("ssp_s4_exact_dup_file", 10000, SSP_M7, "M7", 12), - ( - "ssp_s4_exact_dup_file", - 1, - SSP_M7, - "M7", - 12, - ), # This forces an in memory and database deletion of records. - ], -) -@pytest.mark.django_db() -def test_parse_duplicate( - file, batch_size, model, record_type, num_errors, dfs, request -): - """Test cases for datafiles that have exact duplicate records.""" - datafile = request.getfixturevalue(file) - dfs.datafile = datafile - - settings.BULK_CREATE_BATCH_SIZE = batch_size - - parser = ParserFactory.get_instance( - datafile=datafile, - dfs=dfs, - section=datafile.section, - program_type=datafile.program_type, - ) - parser.parse_and_validate() - - settings.BULK_CREATE_BATCH_SIZE = os.getenv("BULK_CREATE_BATCH_SIZE", 10000) - - parser_errors = ParserError.objects.filter( - file=datafile, error_type=ParserErrorCategoryChoices.CASE_CONSISTENCY - ).order_by("id") - for e in parser_errors: - assert e.error_type == ParserErrorCategoryChoices.CASE_CONSISTENCY - assert parser_errors.count() == num_errors - - dup_error = parser_errors.first() - assert ( - dup_error.error_message - == f"Duplicate record detected with record type {record_type} at line 3. " - + "Record is a duplicate of the record at line number 2." - ) + dfs.status = dfs.get_status() + assert dfs.status == DataFileSummary.Status.REJECTED + dfs.case_aggregates = aggregates.case_aggregates_by_month(dfs.datafile, dfs.status) + assert dfs.case_aggregates == { + "months": [ + { + "month": "Oct", + "accepted_without_errors": "N/A", + "accepted_with_errors": "N/A", + }, + { + "month": "Nov", + "accepted_without_errors": "N/A", + "accepted_with_errors": "N/A", + }, + { + "month": "Dec", + "accepted_without_errors": "N/A", + "accepted_with_errors": "N/A", + }, + ], + "rejected": 1, + } - model.objects.count() == 0 - - -@pytest.mark.parametrize( - "file, batch_size, model, record_type, num_errors, err_msg", - [ - ("tanf_s1_partial_dup_file", 10000, TANF_T1, "T1", 3, "partial_dup_t1_err_msg"), - # This forces an in memory and database deletion of records. - ("tanf_s1_partial_dup_file", 1, TANF_T1, "T1", 3, "partial_dup_t1_err_msg"), - ("tanf_s2_partial_dup_file", 10000, TANF_T5, "T5", 3, "partial_dup_t5_err_msg"), - # This forces an in memory and database deletion of records. - ("tanf_s2_partial_dup_file", 1, TANF_T5, "T5", 3, "partial_dup_t5_err_msg"), - ("ssp_s1_partial_dup_file", 10000, SSP_M1, "M1", 3, "partial_dup_t1_err_msg"), - # This forces an in memory and database deletion of records. - ("ssp_s1_partial_dup_file", 1, SSP_M1, "M1", 3, "partial_dup_t1_err_msg"), - ("ssp_s2_partial_dup_file", 10000, SSP_M5, "M5", 3, "partial_dup_t5_err_msg"), - # This forces an in memory and database deletion of records. - ("ssp_s2_partial_dup_file", 1, SSP_M5, "M5", 3, "partial_dup_t5_err_msg"), - ], -) -@pytest.mark.django_db() -def test_parse_partial_duplicate( - file, batch_size, model, record_type, num_errors, err_msg, dfs, request -): - """Test cases for datafiles that have partial duplicate records.""" - datafile = request.getfixturevalue(file) - expected_error_msg = request.getfixturevalue(err_msg) - - dfs.datafile = datafile - - settings.BULK_CREATE_BATCH_SIZE = batch_size - - print("test duplicates") - print(datafile.file) - - parser = ParserFactory.get_instance( - datafile=datafile, - dfs=dfs, - section=datafile.section, - program_type=datafile.program_type, - ) - parser.parse_and_validate() + errors = ParserError.objects.filter(file=aggregates_rejected_datafile).order_by( + "id" + ) - settings.BULK_CREATE_BATCH_SIZE = os.getenv("BULK_CREATE_BATCH_SIZE", 10000) + assert errors.count() == 3 + record_precheck_errors = errors.filter(row_number=2) + assert record_precheck_errors.count() == 2 + for error in record_precheck_errors: + assert error.error_type == ParserErrorCategoryChoices.RECORD_PRE_CHECK - parser_errors = ParserError.objects.filter( - file=datafile, error_type=ParserErrorCategoryChoices.CASE_CONSISTENCY - ).order_by("id") - for e in parser_errors: - assert e.error_type == ParserErrorCategoryChoices.CASE_CONSISTENCY - assert parser_errors.count() == num_errors + assert errors.last().error_type == ParserErrorCategoryChoices.PRE_CHECK - dup_error = parser_errors.first() - print("Generated Error: ", dup_error.error_message) - assert expected_error_msg.format(record_type=record_type) in dup_error.error_message + assert TANF_T2.objects.count() == 0 - model.objects.count() == 0 + @pytest.mark.django_db() + def test_parse_tanf_section_1_file_with_bad_update_indicator( + self, + tanf_section_1_file_with_bad_update_indicator, dfs + ): + """Test parsing TANF Section 1 submission update indicator.""" + dfs.datafile = tanf_section_1_file_with_bad_update_indicator + parse_datafile(dfs, tanf_section_1_file_with_bad_update_indicator) -@pytest.mark.django_db() -def test_parse_cat_4_edge_case_file(cat4_edge_case_file, dfs): - """Test parsing file with a cat4 error edge case submission.""" - cat4_edge_case_file.year = 2024 - cat4_edge_case_file.quarter = "Q1" + parser_errors = ParserError.objects.filter( + file=tanf_section_1_file_with_bad_update_indicator, + ) - dfs.datafile = cat4_edge_case_file - dfs.save() + assert dfs.get_status() == DataFileSummary.Status.ACCEPTED_WITH_ERRORS - settings.BULK_CREATE_BATCH_SIZE = 1 + assert parser_errors.count() == 5 - parser = ParserFactory.get_instance( - datafile=cat4_edge_case_file, - dfs=dfs, - section=cat4_edge_case_file.section, - program_type=cat4_edge_case_file.program_type, - ) - parser.parse_and_validate() + error_messages = [error.error_message for error in parser_errors] - settings.BULK_CREATE_BATCH_SIZE = os.getenv("BULK_CREATE_BATCH_SIZE", 10000) + assert ( + "HEADER Update Indicator must be set to D instead of U. Please review" + + " Exporting Complete Data Using FTANF in the Knowledge Center." + in error_messages + ) - parser_errors = ParserError.objects.filter(file=cat4_edge_case_file).filter( - error_type=ParserErrorCategoryChoices.CASE_CONSISTENCY - ) + @pytest.mark.django_db() + def test_parse_tribal_section_4_bad_quarter(self, tribal_section_4_bad_quarter, dfs): + """Test handling invalid quarter value that raises a ValueError exception.""" + tribal_section_4_bad_quarter.year = 2021 + tribal_section_4_bad_quarter.quarter = "Q1" + dfs.datafile = tribal_section_4_bad_quarter - assert TANF_T1.objects.all().count() == 2 - assert TANF_T2.objects.all().count() == 2 - assert TANF_T3.objects.all().count() == 4 + parse_datafile(dfs, tribal_section_4_bad_quarter) + parser_errors = ParserError.objects.filter( + file=tribal_section_4_bad_quarter + ).order_by("id") - assert dfs.total_number_of_records_in_file == 17 - assert dfs.total_number_of_records_created == 8 + assert parser_errors.count() == 3 - err = parser_errors.first() - assert err.error_message == ( - "Every T1 record should have at least one corresponding T2 or T3 record with the " - "same Item 4 (Reporting Year and Month) and Item 6 (Case Number)." - ) - assert dfs.get_status() == DataFileSummary.Status.PARTIALLY_ACCEPTED + parser_errors.first().error_message == ( + "T7: 2020 is invalid. Calendar Quarter must be a numeric " + "representing the Calendar Year and Quarter formatted as YYYYQ" + ) + Tribal_TANF_T7.objects.count() == 0 -@pytest.mark.parametrize( - "file", - [ - ("fra_work_outcome_exiter_csv_file"), - ("fra_work_outcome_exiter_xlsx_file"), - ], -) -@pytest.mark.django_db() -def test_parse_fra_work_outcome_exiters(request, file, dfs): - """Test parsing FRA Work Outcome Exiters files.""" - datafile = request.getfixturevalue(file) - datafile.year = 2024 - datafile.quarter = "Q2" - - dfs.datafile = datafile - dfs.save() - - parser = ParserFactory.get_instance( - datafile=datafile, - dfs=dfs, - section=datafile.section, - program_type=datafile.program_type, - ) - parser.parse_and_validate() + @pytest.mark.django_db() + def test_parse_t3_cat2_invalid_citizenship(self, t3_cat2_invalid_citizenship_file, dfs): + """Test parsing a TANF T3 record with an invalid CITIZENSHIP_STATUS.""" + dfs.datafile = t3_cat2_invalid_citizenship_file + t3_cat2_invalid_citizenship_file.year = 2021 + t3_cat2_invalid_citizenship_file.quarter = "Q1" + dfs.save() - assert TANF_Exiter1.objects.all().count() == 5 + parse_datafile(dfs, t3_cat2_invalid_citizenship_file) - errors = ParserError.objects.filter(file=datafile).order_by("id") - assert errors.count() == 8 - for e in errors: - assert e.error_type == ParserErrorCategoryChoices.CASE_CONSISTENCY - assert dfs.total_number_of_records_in_file == 11 - assert dfs.total_number_of_records_created == 5 - assert dfs.get_status() == DataFileSummary.Status.PARTIALLY_ACCEPTED + exclusion = Query(error_type=ParserErrorCategoryChoices.CASE_CONSISTENCY) | Query( + error_type=ParserErrorCategoryChoices.PRE_CHECK + ) + parser_errors = ( + ParserError.objects.filter(file=t3_cat2_invalid_citizenship_file) + .exclude(exclusion) + .order_by("pk") + ) -@pytest.mark.parametrize( - "file", - [ - ("fra_bad_header_csv"), - ("fra_bad_header_xlsx"), - ], -) -@pytest.mark.django_db() -def test_parse_fra_bad_header(request, file, dfs): - """Test parsing FRA files with bad header data.""" - datafile = request.getfixturevalue(file) - datafile.year = 2024 - datafile.quarter = "Q1" - - dfs.datafile = datafile - dfs.save() - - parser = ParserFactory.get_instance( - datafile=datafile, - dfs=dfs, - section=datafile.section, - program_type=datafile.program_type, - ) - parser.parse_and_validate() + # no errors expected as fields are not required + assert parser_errors.count() == 0 - assert TANF_Exiter1.objects.all().count() == 0 + @pytest.mark.django_db() + def test_parse_m2_cat2_invalid_37_38_39_file(self, m2_cat2_invalid_37_38_39_file, dfs): + """Test parsing an SSP M2 file with an invalid EDUCATION_LEVEL, CITIZENSHIP_STATUS, COOPERATION_CHILD_SUPPORT.""" + dfs.datafile = m2_cat2_invalid_37_38_39_file + m2_cat2_invalid_37_38_39_file.year = 2024 + m2_cat2_invalid_37_38_39_file.quarter = "Q1" + dfs.save() - errors = ParserError.objects.filter(file=datafile).order_by("id") - assert len(errors) == 1 - for e in errors: - assert e.error_message == "File does not begin with FRA data." - assert e.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert dfs.get_status() == DataFileSummary.Status.REJECTED + parse_datafile(dfs, m2_cat2_invalid_37_38_39_file) + exclusion = Query(error_type=ParserErrorCategoryChoices.CASE_CONSISTENCY) | Query( + error_type=ParserErrorCategoryChoices.PRE_CHECK + ) -@pytest.mark.parametrize( - "file", - [ - ("fra_empty_first_row_csv"), - ("fra_empty_first_row_xlsx"), - ], -) -@pytest.mark.django_db() -def test_parse_fra_empty_first_row(request, file, dfs): - """Test parsing FRA files with an empty first row/no header data.""" - datafile = request.getfixturevalue(file) - datafile.year = 2024 - datafile.quarter = "Q1" - - dfs.datafile = datafile - dfs.save() - - parser = ParserFactory.get_instance( - datafile=datafile, - dfs=dfs, - section=datafile.section, - program_type=datafile.program_type, - ) - parser.parse_and_validate() + parser_errors = ( + ParserError.objects.filter(file=m2_cat2_invalid_37_38_39_file) + .exclude(exclusion) + .order_by("pk") + ) - assert TANF_Exiter1.objects.all().count() == 0 + # no errors expected as fields are not required + assert parser_errors.count() == 0 - errors = ParserError.objects.filter(file=datafile).order_by("id") - assert len(errors) == 1 - for e in errors: - assert e.error_message == "File does not begin with FRA data." - assert e.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert dfs.get_status() == DataFileSummary.Status.REJECTED + @pytest.mark.django_db() + def test_parse_m3_cat2_invalid_68_69_file(self, m3_cat2_invalid_68_69_file, dfs): + """Test parsing an SSP M3 file with an invalid EDUCATION_LEVEL and CITIZENSHIP_STATUS.""" + dfs.datafile = m3_cat2_invalid_68_69_file + m3_cat2_invalid_68_69_file.year = 2024 + m3_cat2_invalid_68_69_file.quarter = "Q1" + dfs.save() + parse_datafile(dfs, m3_cat2_invalid_68_69_file) -@pytest.mark.parametrize( - "file", - [ - ("fra_ofa_test_csv"), - ("fra_ofa_test_xlsx"), - ], -) -@pytest.mark.django_db() -def test_parse_fra_ofa_test_cases(request, file, dfs): - """Test parsing OFA FRA files.""" - datafile = request.getfixturevalue(file) - datafile.year = 2025 - datafile.quarter = "Q3" - - dfs.datafile = datafile - dfs.save() - - settings.BULK_CREATE_BATCH_SIZE = 1 - - parser = ParserFactory.get_instance( - datafile=datafile, - dfs=dfs, - section=datafile.section, - program_type=datafile.program_type, - ) - parser.parse_and_validate() + exclusion = Query(error_type=ParserErrorCategoryChoices.CASE_CONSISTENCY) | Query( + error_type=ParserErrorCategoryChoices.PRE_CHECK + ) - settings.BULK_CREATE_BATCH_SIZE = os.getenv("BULK_CREATE_BATCH_SIZE", 10000) + parser_errors = ( + ParserError.objects.filter(file=m3_cat2_invalid_68_69_file) + .exclude(exclusion) + .order_by("pk") + ) - errors = ParserError.objects.filter(file=datafile).order_by("id") - for e in errors: - assert e.error_type == ParserErrorCategoryChoices.CASE_CONSISTENCY + assert parser_errors.count() == 2 - assert errors.count() == 23 - assert TANF_Exiter1.objects.all().count() == 10 - assert dfs.total_number_of_records_in_file == 28 - assert dfs.total_number_of_records_created == 10 - assert dfs.get_status() == DataFileSummary.Status.PARTIALLY_ACCEPTED + error_msgs = { + "Item 68 (Educational Level) 00 must be between 1 and 16 or must be between 98 and 99.", + "Item 68 (Educational Level) 00 must be between 1 and 16 or must be between 98 and 99.", + } + for e in parser_errors: + assert e.error_message in error_msgs -@pytest.mark.django_db() -def test_parse_fra_formula_fields(fra_formula_fields_test_xlsx, dfs): - """Test parsing a correct FRA file with formula fields.""" - datafile = fra_formula_fields_test_xlsx - datafile.year = 2025 - datafile.quarter = "Q3" + @pytest.mark.django_db() + def test_parse_m5_cat2_invalid_23_24_file(self, m5_cat2_invalid_23_24_file, dfs): + """Test parsing an SSP M5 file with an invalid EDUCATION_LEVEL and CITIZENSHIP_STATUS.""" + dfs.datafile = m5_cat2_invalid_23_24_file + m5_cat2_invalid_23_24_file.year = 2019 + m5_cat2_invalid_23_24_file.quarter = "Q1" + dfs.save() - dfs.datafile = datafile - dfs.save() + parse_datafile(dfs, m5_cat2_invalid_23_24_file) - parser = ParserFactory.get_instance( - datafile=datafile, - dfs=dfs, - section=datafile.section, - program_type=datafile.program_type, - ) - parser.parse_and_validate() - - errors = ParserError.objects.filter(file=datafile).order_by("id") - assert errors.count() == 0 - assert TANF_Exiter1.objects.all().count() == 8 - assert dfs.total_number_of_records_in_file == 8 - assert dfs.total_number_of_records_created == 8 - assert dfs.get_status() == DataFileSummary.Status.ACCEPTED - - -@pytest.mark.django_db() -def test_parse_fra_decoder_unknown(fra_decoder_unknown, dfs): - """Test parsing a FRA file with bad encoding.""" - datafile = fra_decoder_unknown - datafile.year = 2025 - datafile.quarter = "Q3" - - dfs.datafile = datafile - dfs.save() - - try: - parser = ParserFactory.get_instance( - datafile=datafile, - dfs=dfs, - section=datafile.section, - program_type=datafile.program_type, + exclusion = Query(error_type=ParserErrorCategoryChoices.CASE_CONSISTENCY) | Query( + error_type=ParserErrorCategoryChoices.PRE_CHECK ) - parser.parse_and_validate() - except util.DecoderUnknownException: - pass - - errors = ParserError.objects.filter(file=datafile).order_by("id") - assert errors.count() == 1 - assert errors.first().error_type == ParserErrorCategoryChoices.PRE_CHECK - assert errors.first().error_message == ( - "Could not determine encoding of FRA file. If the file is an XLSX file, " - "ensure it can be opened in Excel. If the file is a CSV, ensure it can be " - "opened in a text editor and is UTF-8 encoded." - ) - assert dfs.get_status() == DataFileSummary.Status.REJECTED - -@pytest.mark.django_db() -def test_parse_section2_no_records(section2_no_records, dfs): - """Test parsing valid section 2 file with no records.""" - datafile = section2_no_records - dfs.datafile = datafile - dfs.save() - - parser = ParserFactory.get_instance( - datafile=datafile, - dfs=dfs, - section=datafile.section, - program_type=datafile.program_type, - ) - parser.parse_and_validate() + parser_errors = ( + ParserError.objects.filter(file=m5_cat2_invalid_23_24_file) + .exclude(exclusion) + .order_by("pk") + ) - errors = ParserError.objects.filter(file=datafile).order_by("id") - assert errors.count() == 0 - assert dfs.get_status() == DataFileSummary.Status.ACCEPTED + assert parser_errors.count() == 0 + @pytest.mark.django_db() + def test_zero_filled_fips_code_file(self, test_file_zero_filled_fips_code, dfs): + """Test parsing a file with zero filled FIPS code.""" + # TODO: this test can be merged as parametrized test with "test_parse_small_correct_file" + dfs.datafile = test_file_zero_filled_fips_code + test_file_zero_filled_fips_code.year = 2024 + test_file_zero_filled_fips_code.quarter = "Q2" + test_file_zero_filled_fips_code.save() -@pytest.mark.django_db() -def test_parse_program_audit_ftanf(request, program_audit_ftanf, dfs): - """Test parsing Program Audit files.""" - datafile = program_audit_ftanf - datafile.year = 2024 - datafile.quarter = "Q2" + parse_datafile(dfs, test_file_zero_filled_fips_code) - dfs.datafile = datafile - dfs.save() + parser_errors = ParserError.objects.filter(file=test_file_zero_filled_fips_code) + assert ( + "T1 Item 2 (County FIPS Code): field is required but a value was not" + + " provided." + in [i.error_message for i in parser_errors] + ) - parser = ParserFactory.get_instance( - datafile=datafile, - dfs=dfs, - section=datafile.section, - program_type=datafile.program_type, - is_program_audit=datafile.is_program_audit, - ) - parser.parse_and_validate() - - assert ProgramAudit_T1.objects.all().count() == 1 - assert ProgramAudit_T2.objects.all().count() == 2 - assert ProgramAudit_T3.objects.all().count() == 1 - - errors = ParserError.objects.filter(file=datafile).order_by("id") - assert len(errors) == 2 - for e in errors: - assert e.error_type == ParserErrorCategoryChoices.FIELD_VALUE - assert dfs.get_status() == DataFileSummary.Status.ACCEPTED_WITH_ERRORS - dfs.case_aggregates = aggregates.case_aggregates_by_month(dfs.datafile, dfs.status) - assert dfs.case_aggregates == { - "months": [ - { - "month": "Jan", - "accepted_without_errors": 0, - "accepted_with_errors": 0, - }, - { - "month": "Feb", - "accepted_without_errors": 0, - "accepted_with_errors": 0, - }, - { - "month": "Mar", - "accepted_without_errors": 0, - "accepted_with_errors": 1, - }, + @pytest.mark.parametrize( + "file", + [ + ("fra_bad_header_csv"), + ("fra_bad_header_xlsx"), ], - "rejected": 0, - } - - -@pytest.mark.django_db() -def test_parse_program_audit_duplicates(request, program_audit_duplicates, dfs): - """Test parsing Program Audit files with duplicate rows.""" - datafile = program_audit_duplicates - datafile.year = 2024 - datafile.quarter = "Q2" - - dfs.datafile = datafile - dfs.save() - - parser = ParserFactory.get_instance( - datafile=datafile, - dfs=dfs, - section=datafile.section, - program_type=datafile.program_type, - is_program_audit=datafile.is_program_audit, ) - parser.parse_and_validate() + @pytest.mark.django_db() + def test_parse_fra_bad_header(self, request, file, dfs): + """Test parsing FRA files with bad header data.""" + datafile = request.getfixturevalue(file) + datafile.year = 2024 + datafile.quarter = "Q1" - assert ProgramAudit_T1.objects.all().count() == 1 - assert ProgramAudit_T2.objects.all().count() == 3 - assert ProgramAudit_T3.objects.all().count() == 2 + dfs.datafile = datafile + dfs.save() - errors = ParserError.objects.filter(file=datafile).order_by("id") - assert len(errors) == 7 + parse_datafile(dfs, datafile) - duplicate_errors = errors.filter( - error_message__contains="Duplicate record detected" - ) - assert duplicate_errors.count() == 2 - - assert dfs.get_status() == DataFileSummary.Status.PARTIALLY_ACCEPTED - dfs.case_aggregates = aggregates.case_aggregates_by_month(dfs.datafile, dfs.status) - assert dfs.case_aggregates == { - "months": [ - { - "month": "Jan", - "accepted_without_errors": 0, - "accepted_with_errors": 0, - }, - { - "month": "Feb", - "accepted_without_errors": 0, - "accepted_with_errors": 0, - }, - { - "month": "Mar", - "accepted_without_errors": 0, - "accepted_with_errors": 1, - }, - ], - "rejected": 2, - } + assert TANF_Exiter1.objects.all().count() == 0 + errors = ParserError.objects.filter(file=datafile).order_by("id") + assert len(errors) == 1 + for e in errors: + assert e.error_message == "File does not begin with FRA data." + assert e.error_type == ParserErrorCategoryChoices.PRE_CHECK + assert dfs.get_status() == DataFileSummary.Status.REJECTED -@pytest.mark.parametrize( - "file", - [ - ("program_audit_space_fill"), - ("program_audit_zero_fill"), - ], -) -@pytest.mark.django_db() -def test_parse_program_audit_space_zero_fill(request, file, dfs): - """Test parsing Program Audit files.""" - datafile = request.getfixturevalue(file) - datafile.year = 2024 - datafile.quarter = "Q1" - - dfs.datafile = datafile - dfs.save() - - parser = ParserFactory.get_instance( - datafile=datafile, - dfs=dfs, - section=datafile.section, - program_type=datafile.program_type, - is_program_audit=datafile.is_program_audit, - ) - parser.parse_and_validate() - - assert ProgramAudit_T1.objects.all().count() == 1 - assert ProgramAudit_T2.objects.all().count() == 1 - assert ProgramAudit_T3.objects.all().count() == 3 - - errors = ParserError.objects.filter(file=datafile).order_by("id") - for e in errors: - print(e) - # assert e.error_type == ParserErrorCategoryChoices.FIELD_VALUE - assert len(errors) == 13 - assert dfs.get_status() == DataFileSummary.Status.ACCEPTED_WITH_ERRORS - dfs.case_aggregates = aggregates.case_aggregates_by_month(dfs.datafile, dfs.status) - assert dfs.case_aggregates == { - "months": [ - { - "month": "Oct", - "accepted_without_errors": 0, - "accepted_with_errors": 1, - }, - { - "month": "Nov", - "accepted_without_errors": 0, - "accepted_with_errors": 0, - }, - { - "month": "Dec", - "accepted_without_errors": 0, - "accepted_with_errors": 0, - }, + @pytest.mark.parametrize( + "file", + [ + ("fra_empty_first_row_csv"), + ("fra_empty_first_row_xlsx"), ], - "rejected": 0, - } - - -@pytest.mark.django_db -def test_parse_tanf_s1_federally_funded_recipients( - tanf_s1_federally_funded_recipients, dfs -): - """Test parsing file that generates the tanf_s1_federally_funded_recipients error.""" - dfs.datafile = tanf_s1_federally_funded_recipients - - parser = ParserFactory.get_instance( - datafile=tanf_s1_federally_funded_recipients, - dfs=dfs, - section=tanf_s1_federally_funded_recipients.section, - program_type=tanf_s1_federally_funded_recipients.program_type, ) - parser.parse_and_validate() + @pytest.mark.django_db() + def test_parse_fra_empty_first_row(self, request, file, dfs): + """Test parsing FRA files with an empty first row/no header data.""" + datafile = request.getfixturevalue(file) + datafile.year = 2024 + datafile.quarter = "Q1" + + dfs.datafile = datafile + dfs.save() + + parse_datafile(dfs, datafile) + + assert TANF_Exiter1.objects.all().count() == 0 + + errors = ParserError.objects.filter(file=datafile).order_by("id") + assert len(errors) == 1 + for e in errors: + assert e.error_message == "File does not begin with FRA data." + assert e.error_type == ParserErrorCategoryChoices.PRE_CHECK + assert dfs.get_status() == DataFileSummary.Status.REJECTED + + @pytest.mark.django_db() + def test_parse_fra_decoder_unknown(self, fra_decoder_unknown, dfs): + """Test parsing a FRA file with bad encoding.""" + datafile = fra_decoder_unknown + datafile.year = 2025 + datafile.quarter = "Q3" + + dfs.datafile = datafile + dfs.save() + + try: + parse_datafile(dfs, datafile) + except util.DecoderUnknownException: + pass + + errors = ParserError.objects.filter(file=datafile).order_by("id") + assert errors.count() == 1 + assert errors.first().error_type == ParserErrorCategoryChoices.PRE_CHECK + assert errors.first().error_message == ( + "Could not determine encoding of FRA file. If the file is an XLSX file, " + "ensure it can be opened in Excel. If the file is a CSV, ensure it can be " + "opened in a text editor and is UTF-8 encoded." + ) + assert dfs.get_status() == DataFileSummary.Status.REJECTED + + @pytest.mark.django_db() + def test_parse_section2_no_records(self, section2_no_records, dfs): + """Test parsing valid section 2 file with no records.""" + datafile = section2_no_records + dfs.datafile = datafile + dfs.save() + + parse_datafile(dfs, datafile) + + errors = ParserError.objects.filter(file=datafile).order_by("id") + assert errors.count() == 0 + assert dfs.get_status() == DataFileSummary.Status.ACCEPTED + + @pytest.mark.django_db + def test_parse_tanf_s1_federally_funded_recipients( + self, + tanf_s1_federally_funded_recipients, dfs + ): + """Test parsing file that generates the tanf_s1_federally_funded_recipients error.""" + dfs.datafile = tanf_s1_federally_funded_recipients + + parse_datafile(dfs, tanf_s1_federally_funded_recipients) + + errors = ParserError.objects.filter( + file=tanf_s1_federally_funded_recipients + ).order_by("id") + + dfs.status = dfs.get_status() + assert dfs.status == DataFileSummary.Status.ACCEPTED_WITH_ERRORS + assert errors.last().error_message == ( + "Federally funded recipients must have a valid Social Security number." + ) - errors = ParserError.objects.filter( - file=tanf_s1_federally_funded_recipients - ).order_by("id") + @pytest.mark.django_db + def test_parse_case_aggregates_edge_case(self, case_aggregates_edge_case, dfs): + """Test parsing of cases_across_months_with_error.txt.""" + case_aggregates_edge_case.year = 2026 + case_aggregates_edge_case.quarter = "Q1" + case_aggregates_edge_case.save() + + dfs.datafile = case_aggregates_edge_case + + parse_datafile(dfs, case_aggregates_edge_case) + + dfs.status = dfs.get_status() + assert dfs.status == DataFileSummary.Status.PARTIALLY_ACCEPTED + + dfs.case_aggregates = aggregates.case_aggregates_by_month(dfs.datafile, dfs.status) + print(dfs.case_aggregates) + assert dfs.case_aggregates == { + "months": [ + { + "month": "Oct", + "accepted_without_errors": 1, + "accepted_with_errors": 0, + }, + { + "month": "Nov", + "accepted_without_errors": 1, + "accepted_with_errors": 0, + }, + { + "month": "Dec", + "accepted_without_errors": 0, + "accepted_with_errors": 1, + }, + ], + "rejected": 2, # Rejected is 2 locally because of the trailer errors. We only generate trailer erros locally. + } - dfs.status = dfs.get_status() - assert dfs.status == DataFileSummary.Status.ACCEPTED_WITH_ERRORS - assert errors.last().error_message == ( - "Federally funded recipients must have a valid Social Security number." - ) + assert TANF_T1.objects.count() == 3 + assert TANF_T2.objects.count() == 3 + assert TANF_T3.objects.count() == 6 diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse_bulk_creation.py b/tdrs-backend/tdpservice/parsers/test/test_parse_bulk_creation.py new file mode 100644 index 000000000..f1b68ca11 --- /dev/null +++ b/tdrs-backend/tdpservice/parsers/test/test_parse_bulk_creation.py @@ -0,0 +1,152 @@ +"""Integration tests covering bulk creation and duplicate handling.""" + +import os + +import pytest +from django.conf import settings + +from tdpservice.parsers.models import ( + DataFileSummary, + ParserError, + ParserErrorCategoryChoices, +) +from tdpservice.parsers.test.helpers import parse_datafile +from tdpservice.search_indexes.models.ssp import SSP_M1, SSP_M4, SSP_M5, SSP_M6, SSP_M7 +from tdpservice.search_indexes.models.tanf import ( + TANF_T1, + TANF_T2, + TANF_T3, + TANF_T4, + TANF_T5, + TANF_T6, + TANF_T7, +) + + +class TestParseBulkCreation: + """Tests for bulk create behavior, duplicates, and edge-case consistency.""" + + @pytest.mark.parametrize( + "file, batch_size, model, record_type, num_errors", + [ + ("tanf_s1_exact_dup_file", 10000, TANF_T1, "T1", 3), + ("tanf_s1_exact_dup_file", 1, TANF_T1, "T1", 3), + ("tanf_s2_exact_dup_file", 10000, TANF_T4, "T4", 3), + ("tanf_s2_exact_dup_file", 1, TANF_T4, "T4", 3), + ("tanf_s3_exact_dup_file", 10000, TANF_T6, "T6", 3), + ("tanf_s3_exact_dup_file", 1, TANF_T6, "T6", 3), + ("tanf_s4_exact_dup_file", 10000, TANF_T7, "T7", 18), + ("tanf_s4_exact_dup_file", 1, TANF_T7, "T7", 18), + ("ssp_s1_exact_dup_file", 10000, SSP_M1, "M1", 3), + ("ssp_s1_exact_dup_file", 1, SSP_M1, "M1", 3), + ("ssp_s2_exact_dup_file", 10000, SSP_M4, "M4", 3), + ("ssp_s2_exact_dup_file", 1, SSP_M4, "M4", 3), + ("ssp_s3_exact_dup_file", 10000, SSP_M6, "M6", 3), + ("ssp_s3_exact_dup_file", 1, SSP_M6, "M6", 3), + ("ssp_s4_exact_dup_file", 10000, SSP_M7, "M7", 12), + ("ssp_s4_exact_dup_file", 1, SSP_M7, "M7", 12), + ], + ) + @pytest.mark.django_db + def test_parse_duplicate( + self, file, batch_size, model, record_type, num_errors, dfs, request + ): + """Test cases for datafiles that have exact duplicate records.""" + datafile = request.getfixturevalue(file) + dfs.datafile = datafile + + settings.BULK_CREATE_BATCH_SIZE = batch_size + + parse_datafile(dfs, datafile) + + settings.BULK_CREATE_BATCH_SIZE = os.getenv("BULK_CREATE_BATCH_SIZE", 10000) + + parser_errors = ParserError.objects.filter( + file=datafile, error_type=ParserErrorCategoryChoices.CASE_CONSISTENCY + ).order_by("id") + for e in parser_errors: + assert e.error_type == ParserErrorCategoryChoices.CASE_CONSISTENCY + assert parser_errors.count() == num_errors + + dup_error = parser_errors.first() + assert ( + dup_error.error_message + == f"Duplicate record detected with record type {record_type} at line 3. " + + "Record is a duplicate of the record at line number 2." + ) + + model.objects.count() == 0 + + @pytest.mark.parametrize( + "file, batch_size, model, record_type, num_errors, err_msg", + [ + ("tanf_s1_partial_dup_file", 10000, TANF_T1, "T1", 3, "partial_dup_t1_err_msg"), + ("tanf_s1_partial_dup_file", 1, TANF_T1, "T1", 3, "partial_dup_t1_err_msg"), + ("tanf_s2_partial_dup_file", 10000, TANF_T5, "T5", 3, "partial_dup_t5_err_msg"), + ("tanf_s2_partial_dup_file", 1, TANF_T5, "T5", 3, "partial_dup_t5_err_msg"), + ("ssp_s1_partial_dup_file", 10000, SSP_M1, "M1", 3, "partial_dup_t1_err_msg"), + ("ssp_s1_partial_dup_file", 1, SSP_M1, "M1", 3, "partial_dup_t1_err_msg"), + ("ssp_s2_partial_dup_file", 10000, SSP_M5, "M5", 3, "partial_dup_t5_err_msg"), + ("ssp_s2_partial_dup_file", 1, SSP_M5, "M5", 3, "partial_dup_t5_err_msg"), + ], + ) + @pytest.mark.django_db + def test_parse_partial_duplicate( + self, file, batch_size, model, record_type, num_errors, err_msg, dfs, request + ): + """Test cases for datafiles that have partial duplicate records.""" + datafile = request.getfixturevalue(file) + expected_error_msg = request.getfixturevalue(err_msg) + + dfs.datafile = datafile + + settings.BULK_CREATE_BATCH_SIZE = batch_size + + parse_datafile(dfs, datafile) + + settings.BULK_CREATE_BATCH_SIZE = os.getenv("BULK_CREATE_BATCH_SIZE", 10000) + + parser_errors = ParserError.objects.filter( + file=datafile, error_type=ParserErrorCategoryChoices.CASE_CONSISTENCY + ).order_by("id") + for e in parser_errors: + assert e.error_type == ParserErrorCategoryChoices.CASE_CONSISTENCY + assert parser_errors.count() == num_errors + + dup_error = parser_errors.first() + assert expected_error_msg.format(record_type=record_type) in dup_error.error_message + + model.objects.count() == 0 + + @pytest.mark.django_db + def test_parse_cat_4_edge_case_file(self, cat4_edge_case_file, dfs): + """Test parsing file with a cat4 error edge case submission.""" + cat4_edge_case_file.year = 2024 + cat4_edge_case_file.quarter = "Q1" + + dfs.datafile = cat4_edge_case_file + dfs.save() + + settings.BULK_CREATE_BATCH_SIZE = 1 + + parse_datafile(dfs, cat4_edge_case_file) + + settings.BULK_CREATE_BATCH_SIZE = os.getenv("BULK_CREATE_BATCH_SIZE", 10000) + + parser_errors = ParserError.objects.filter(file=cat4_edge_case_file).filter( + error_type=ParserErrorCategoryChoices.CASE_CONSISTENCY + ) + + assert TANF_T1.objects.all().count() == 2 + assert TANF_T2.objects.all().count() == 2 + assert TANF_T3.objects.all().count() == 4 + + assert dfs.total_number_of_records_in_file == 17 + assert dfs.total_number_of_records_created == 8 + + err = parser_errors.first() + assert err.error_message == ( + "Every T1 record should have at least one corresponding T2 or T3 record with the " + "same Item 4 (Reporting Year and Month) and Item 6 (Case Number)." + ) + assert dfs.get_status() == DataFileSummary.Status.PARTIALLY_ACCEPTED diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse_fra_integration.py b/tdrs-backend/tdpservice/parsers/test/test_parse_fra_integration.py new file mode 100644 index 000000000..6aab23d12 --- /dev/null +++ b/tdrs-backend/tdpservice/parsers/test/test_parse_fra_integration.py @@ -0,0 +1,99 @@ +"""Integration tests for FRA parsing scenarios.""" + +import os + +import pytest +from django.conf import settings + +from tdpservice.parsers.models import ( + DataFileSummary, + ParserError, + ParserErrorCategoryChoices, +) +from tdpservice.parsers.test.helpers import parse_datafile +from tdpservice.search_indexes.models.fra import TANF_Exiter1 + + +class TestParseFraIntegration: + """Tests for FRA parsing with real fixtures and record creation.""" + + @pytest.mark.parametrize( + "file", + [ + ("fra_work_outcome_exiter_csv_file"), + ("fra_work_outcome_exiter_xlsx_file"), + ], + ) + @pytest.mark.django_db + def test_parse_fra_work_outcome_exiters(self, request, file, dfs): + """Test parsing FRA Work Outcome Exiters files.""" + datafile = request.getfixturevalue(file) + datafile.year = 2024 + datafile.quarter = "Q2" + + dfs.datafile = datafile + dfs.save() + + parse_datafile(dfs, datafile) + + assert TANF_Exiter1.objects.all().count() == 5 + + errors = ParserError.objects.filter(file=datafile).order_by("id") + assert errors.count() == 8 + for e in errors: + assert e.error_type == ParserErrorCategoryChoices.CASE_CONSISTENCY + assert dfs.total_number_of_records_in_file == 11 + assert dfs.total_number_of_records_created == 5 + assert dfs.get_status() == DataFileSummary.Status.PARTIALLY_ACCEPTED + + @pytest.mark.parametrize( + "file", + [ + ("fra_ofa_test_csv"), + ("fra_ofa_test_xlsx"), + ], + ) + @pytest.mark.django_db + def test_parse_fra_ofa_test_cases(self, request, file, dfs): + """Test parsing OFA FRA files.""" + datafile = request.getfixturevalue(file) + datafile.year = 2025 + datafile.quarter = "Q3" + + dfs.datafile = datafile + dfs.save() + + settings.BULK_CREATE_BATCH_SIZE = 1 + + parse_datafile(dfs, datafile) + + settings.BULK_CREATE_BATCH_SIZE = os.getenv("BULK_CREATE_BATCH_SIZE", 10000) + + errors = ParserError.objects.filter(file=datafile).order_by("id") + for e in errors: + assert e.error_type == ParserErrorCategoryChoices.CASE_CONSISTENCY + + assert errors.count() == 23 + assert TANF_Exiter1.objects.all().count() == 10 + assert dfs.total_number_of_records_in_file == 28 + assert dfs.total_number_of_records_created == 10 + assert dfs.get_status() == DataFileSummary.Status.PARTIALLY_ACCEPTED + + @pytest.mark.django_db + def test_parse_fra_formula_fields(self, fra_formula_fields_test_xlsx, dfs): + """Test parsing a correct FRA file with formula fields.""" + datafile = fra_formula_fields_test_xlsx + datafile.year = 2025 + datafile.quarter = "Q3" + + dfs.datafile = datafile + dfs.save() + + parse_datafile(dfs, datafile) + + errors = ParserError.objects.filter(file=datafile).order_by("id") + assert errors.count() == 0 + assert TANF_Exiter1.objects.all().count() == 8 + assert dfs.total_number_of_records_in_file == 8 + assert dfs.total_number_of_records_created == 8 + assert dfs.get_status() == DataFileSummary.Status.ACCEPTED diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse_large_files.py b/tdrs-backend/tdpservice/parsers/test/test_parse_large_files.py new file mode 100644 index 000000000..a29fcc9b0 --- /dev/null +++ b/tdrs-backend/tdpservice/parsers/test/test_parse_large_files.py @@ -0,0 +1,111 @@ +"""Integration tests for large TANF datafile parsing.""" + +import pytest + +from tdpservice.parsers import aggregates +from tdpservice.parsers.models import ( + DataFileSummary, + ParserError, + ParserErrorCategoryChoices, +) +from tdpservice.parsers.test.helpers import parse_datafile +from tdpservice.search_indexes.models.tanf import TANF_T1, TANF_T2, TANF_T3 + + +class TestParseLargeFiles: + """Tests for large and long-running parse scenarios.""" + + @pytest.fixture + def parsed_big_file(self, big_file, dfs): + """Return parsed big_file and its DataFileSummary.""" + big_file.year = 2022 + big_file.quarter = "Q1" + big_file.save() + parse_datafile(dfs, big_file) + return big_file, dfs + + @pytest.mark.django_db + def test_big_file_status_and_case_aggregates(self, parsed_big_file): + """Test status and case aggregates for ADS.E2J.FTP1.TS06.""" + _datafile, dfs = parsed_big_file + dfs.status = dfs.get_status() + assert dfs.status == DataFileSummary.Status.ACCEPTED_WITH_ERRORS + + dfs.case_aggregates = aggregates.case_aggregates_by_month( + dfs.datafile, dfs.status + ) + assert dfs.case_aggregates == { + "months": [ + { + "month": "Oct", + "accepted_without_errors": 11, + "accepted_with_errors": 259, + }, + { + "month": "Nov", + "accepted_without_errors": 12, + "accepted_with_errors": 261, + }, + { + "month": "Dec", + "accepted_without_errors": 15, + "accepted_with_errors": 257, + }, + ], + "rejected": 0, + } + + @pytest.mark.django_db + def test_big_file_record_counts(self, parsed_big_file): + """Test record counts for ADS.E2J.FTP1.TS06.""" + _datafile, _dfs = parsed_big_file + assert TANF_T1.objects.count() == 815 + assert TANF_T2.objects.count() == 882 + assert TANF_T3.objects.count() == 1376 + + @pytest.mark.django_db + @pytest.mark.skip(reason="long runtime") + def test_parse_super_big_s1_file(self, super_big_s1_file, dfs): + """Test parsing super_big_s1_file and validate all records are created.""" + super_big_s1_file.year = 2023 + super_big_s1_file.quarter = "Q2" + super_big_s1_file.save() + + dfs.datafile = super_big_s1_file + dfs.save() + + parse_datafile(dfs, super_big_s1_file) + expected_t1_record_count = 96607 + expected_t2_record_count = 112753 + expected_t3_record_count = 172525 + + assert TANF_T1.objects.count() == expected_t1_record_count + assert TANF_T2.objects.count() == expected_t2_record_count + assert TANF_T3.objects.count() == expected_t3_record_count + + @pytest.mark.django_db + def test_parse_big_s1_file_with_rollback(self, big_s1_rollback_file, dfs): + """Test parsing big_s1_rollback_file with rollback on error.""" + big_s1_rollback_file.year = 2023 + big_s1_rollback_file.quarter = "Q2" + big_s1_rollback_file.save() + + dfs.datafile = big_s1_rollback_file + dfs.save() + + parse_datafile(dfs, big_s1_rollback_file) + + parser_errors = ParserError.objects.filter(file=big_s1_rollback_file) + assert parser_errors.count() == 1 + + err = parser_errors.first() + + assert err.row_number == 13609 + assert err.error_type == ParserErrorCategoryChoices.PRE_CHECK + assert err.error_message == "Multiple headers found." + assert err.content_type is None + assert err.object_id is None + + assert TANF_T1.objects.count() == 0 + assert TANF_T2.objects.count() == 0 + assert TANF_T3.objects.count() == 0 diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse_program_audit.py b/tdrs-backend/tdpservice/parsers/test/test_parse_program_audit.py new file mode 100644 index 000000000..dbfac02c1 --- /dev/null +++ b/tdrs-backend/tdpservice/parsers/test/test_parse_program_audit.py @@ -0,0 +1,240 @@ +"""Integration tests for Program Audit parsing.""" + +import pytest + +from tdpservice.parsers import aggregates +from tdpservice.parsers.models import ( + DataFileSummary, + ParserError, + ParserErrorCategoryChoices, +) +from tdpservice.parsers.test.helpers import parse_datafile +from tdpservice.search_indexes.models.program_audit import ( + ProgramAudit_T1, + ProgramAudit_T2, + ProgramAudit_T3, +) + + +class TestParseProgramAudit: + """Tests for Program Audit parsing scenarios.""" + + @pytest.fixture + def parsed_program_audit_ftanf(self, program_audit_ftanf, dfs): + """Return parsed program audit FTANF file and its DataFileSummary.""" + datafile = program_audit_ftanf + datafile.year = 2024 + datafile.quarter = "Q2" + + dfs.datafile = datafile + dfs.save() + + parse_datafile(dfs, datafile, is_program_audit=datafile.is_program_audit) + + return datafile, dfs + + @pytest.fixture + def parsed_program_audit_duplicates(self, program_audit_duplicates, dfs): + """Return parsed program audit duplicates file and its DataFileSummary.""" + datafile = program_audit_duplicates + datafile.year = 2024 + datafile.quarter = "Q2" + + dfs.datafile = datafile + dfs.save() + + parse_datafile(dfs, datafile, is_program_audit=datafile.is_program_audit) + + return datafile, dfs + + def _parse_program_audit_file(self, request, dfs, fixture_name, year, quarter): + """Parse a program audit fixture and return the datafile and dfs.""" + datafile = request.getfixturevalue(fixture_name) + datafile.year = year + datafile.quarter = quarter + + dfs.datafile = datafile + dfs.save() + + parse_datafile(dfs, datafile, is_program_audit=datafile.is_program_audit) + + return datafile, dfs + + @pytest.mark.django_db + def test_program_audit_ftanf_record_counts(self, parsed_program_audit_ftanf): + """Test record counts for Program Audit FTANF file.""" + _datafile, _dfs = parsed_program_audit_ftanf + assert ProgramAudit_T1.objects.all().count() == 1 + assert ProgramAudit_T2.objects.all().count() == 2 + assert ProgramAudit_T3.objects.all().count() == 1 + + @pytest.mark.django_db + def test_program_audit_ftanf_errors(self, parsed_program_audit_ftanf): + """Test error types for Program Audit FTANF file.""" + datafile, _dfs = parsed_program_audit_ftanf + errors = ParserError.objects.filter(file=datafile).order_by("id") + assert len(errors) == 2 + for e in errors: + assert e.error_type == ParserErrorCategoryChoices.FIELD_VALUE + + @pytest.mark.django_db + def test_program_audit_ftanf_case_aggregates(self, parsed_program_audit_ftanf): + """Test case aggregates for Program Audit FTANF file.""" + _datafile, dfs = parsed_program_audit_ftanf + dfs.status = dfs.get_status() + assert dfs.status == DataFileSummary.Status.ACCEPTED_WITH_ERRORS + dfs.case_aggregates = aggregates.case_aggregates_by_month( + dfs.datafile, dfs.status + ) + assert dfs.case_aggregates == { + "months": [ + { + "month": "Jan", + "accepted_without_errors": 0, + "accepted_with_errors": 0, + }, + { + "month": "Feb", + "accepted_without_errors": 0, + "accepted_with_errors": 0, + }, + { + "month": "Mar", + "accepted_without_errors": 0, + "accepted_with_errors": 1, + }, + ], + "rejected": 0, + } + + @pytest.mark.django_db + def test_program_audit_duplicates_record_counts( + self, parsed_program_audit_duplicates + ): + """Test record counts for Program Audit duplicates file.""" + _datafile, _dfs = parsed_program_audit_duplicates + assert ProgramAudit_T1.objects.all().count() == 1 + assert ProgramAudit_T2.objects.all().count() == 3 + assert ProgramAudit_T3.objects.all().count() == 2 + + @pytest.mark.django_db + def test_program_audit_duplicates_errors(self, parsed_program_audit_duplicates): + """Test error counts for Program Audit duplicates file.""" + datafile, _dfs = parsed_program_audit_duplicates + errors = ParserError.objects.filter(file=datafile).order_by("id") + assert len(errors) == 7 + + duplicate_errors = errors.filter( + error_message__contains="Duplicate record detected" + ) + assert duplicate_errors.count() == 2 + + @pytest.mark.django_db + def test_program_audit_duplicates_case_aggregates( + self, parsed_program_audit_duplicates + ): + """Test case aggregates for Program Audit duplicates file.""" + _datafile, dfs = parsed_program_audit_duplicates + dfs.status = dfs.get_status() + assert dfs.status == DataFileSummary.Status.PARTIALLY_ACCEPTED + dfs.case_aggregates = aggregates.case_aggregates_by_month( + dfs.datafile, dfs.status + ) + assert dfs.case_aggregates == { + "months": [ + { + "month": "Jan", + "accepted_without_errors": 0, + "accepted_with_errors": 0, + }, + { + "month": "Feb", + "accepted_without_errors": 0, + "accepted_with_errors": 0, + }, + { + "month": "Mar", + "accepted_without_errors": 0, + "accepted_with_errors": 1, + }, + ], + "rejected": 2, + } + + @pytest.mark.parametrize( + "fixture_name", + [ + "program_audit_space_fill", + "program_audit_zero_fill", + ], + ) + @pytest.mark.django_db + def test_program_audit_space_zero_fill_record_counts( + self, request, fixture_name, dfs + ): + """Test record counts for Program Audit space/zero fill files.""" + _datafile, _dfs = self._parse_program_audit_file( + request, dfs, fixture_name, 2024, "Q1" + ) + assert ProgramAudit_T1.objects.all().count() == 1 + assert ProgramAudit_T2.objects.all().count() == 1 + assert ProgramAudit_T3.objects.all().count() == 3 + + @pytest.mark.parametrize( + "fixture_name", + [ + "program_audit_space_fill", + "program_audit_zero_fill", + ], + ) + @pytest.mark.django_db + def test_program_audit_space_zero_fill_errors( + self, request, fixture_name, dfs + ): + """Test error counts for Program Audit space/zero fill files.""" + datafile, _dfs = self._parse_program_audit_file( + request, dfs, fixture_name, 2024, "Q1" + ) + errors = ParserError.objects.filter(file=datafile).order_by("id") + assert len(errors) == 13 + + @pytest.mark.parametrize( + "fixture_name", + [ + "program_audit_space_fill", + "program_audit_zero_fill", + ], + ) + @pytest.mark.django_db + def test_program_audit_space_zero_fill_case_aggregates( + self, request, fixture_name, dfs + ): + """Test case aggregates for Program Audit space/zero fill files.""" + _datafile, dfs = self._parse_program_audit_file( + request, dfs, fixture_name, 2024, "Q1" + ) + dfs.status = dfs.get_status() + assert dfs.status == DataFileSummary.Status.ACCEPTED_WITH_ERRORS + dfs.case_aggregates = aggregates.case_aggregates_by_month( + dfs.datafile, dfs.status + ) + assert dfs.case_aggregates == { + "months": [ + { + "month": "Oct", + "accepted_without_errors": 0, + "accepted_with_errors": 1, + }, + { + "month": "Nov", + "accepted_without_errors": 0, + "accepted_with_errors": 0, + }, + { + "month": "Dec", + "accepted_without_errors": 0, + "accepted_with_errors": 0, + }, + ], + "rejected": 0, + } diff --git a/tdrs-backend/tdpservice/parsers/test/test_parser_internals.py b/tdrs-backend/tdpservice/parsers/test/test_parser_internals.py new file mode 100644 index 000000000..4adf49da0 --- /dev/null +++ b/tdrs-backend/tdpservice/parsers/test/test_parser_internals.py @@ -0,0 +1,214 @@ +"""Unit tests for parser factory and schema manager internals.""" + +import pytest + +from tdpservice.data_files.models import DataFile +from tdpservice.parsers.dataclasses import RawRow +from tdpservice.parsers.factory import ParserFactory +from tdpservice.parsers.fields import TransformField +from tdpservice.parsers.models import ParserErrorCategoryChoices +from tdpservice.parsers.parser_classes.base_parser import BaseParser +from tdpservice.parsers.parser_classes.fra_parser import FRAParser +from tdpservice.parsers.parser_classes.program_audit_parser import ProgramAuditParser +from tdpservice.parsers.parser_classes.tdr_parser import TanfDataReportParser +from tdpservice.parsers.schema_manager import SchemaManager + + +class TestParserFactory: + """Tests for ParserFactory class selection and instantiation.""" + + @pytest.mark.parametrize( + "program_type, expected", + [ + (DataFile.ProgramType.TANF, TanfDataReportParser), + (DataFile.ProgramType.SSP, TanfDataReportParser), + (DataFile.ProgramType.TRIBAL, TanfDataReportParser), + ], + ) + def test_get_class_tanf_like_programs(self, program_type, expected): + """Return TANF parser for TANF/SSP/Tribal.""" + assert ParserFactory.get_class(program_type) is expected + + def test_get_class_program_audit(self): + """Return ProgramAuditParser when program audit is requested.""" + assert ( + ParserFactory.get_class( + DataFile.ProgramType.TANF, is_program_audit=True + ) + is ProgramAuditParser + ) + + def test_get_class_fra(self): + """Return FRAParser for FRA program type.""" + assert ParserFactory.get_class(DataFile.ProgramType.FRA) is FRAParser + + def test_get_class_unknown_raises(self): + """Raise when no parser is available for program type.""" + with pytest.raises(ValueError, match="No parser available for program type"): + ParserFactory.get_class("UNKNOWN") + + def test_get_instance_passes_kwargs(self, monkeypatch): + """Ensure get_instance forwards kwargs and program metadata.""" + captured = {} + + class DummyParser: + def __init__(self, **kwargs): + self.kwargs = kwargs + + def fake_get_class(cls, program_type, is_program_audit=False): + captured["program_type"] = program_type + captured["is_program_audit"] = is_program_audit + return DummyParser + + monkeypatch.setattr(ParserFactory, "get_class", classmethod(fake_get_class)) + + instance = ParserFactory.get_instance( + program_type=DataFile.ProgramType.TANF, + is_program_audit=True, + datafile="datafile", + dfs="dfs", + section="Active Case Data", + ) + + assert captured == { + "program_type": DataFile.ProgramType.TANF, + "is_program_audit": True, + } + assert instance.kwargs == { + "datafile": "datafile", + "dfs": "dfs", + "section": "Active Case Data", + } + + +class TestSchemaManager: + """Tests for SchemaManager behavior.""" + + @pytest.mark.django_db + def test_parse_and_validate_unknown_record_type(self, small_correct_file): + """Return record precheck error for unknown record type.""" + manager = SchemaManager( + small_correct_file, + small_correct_file.program_type, + small_correct_file.section, + ) + row = RawRow( + data="Z9", + raw_len=2, + decoded_len=2, + row_num=5, + record_type="Z9", + ) + + result = manager.parse_and_validate(row) + + assert result.schemas == [] + assert len(result.records) == 1 + record, is_valid, errors = result.records[0] + assert record is None + assert is_valid is False + assert len(errors) == 1 + assert errors[0].error_message == "Unknown Record_Type was found." + assert errors[0].error_type == ParserErrorCategoryChoices.RECORD_PRE_CHECK + + @pytest.mark.django_db + def test_update_encrypted_fields_updates_transform_fields(self, small_correct_file): + """Update TransformField encryption flags across schemas.""" + manager = SchemaManager( + small_correct_file, + small_correct_file.program_type, + small_correct_file.section, + ) + + transform_fields = [] + for schemas in manager.schema_map.values(): + for schema in schemas: + for field in schema.fields: + if ( + isinstance(field, TransformField) + and "is_encrypted" in field.kwargs + ): + transform_fields.append(field) + + assert transform_fields + + manager.update_encrypted_fields(True) + + assert all(field.kwargs["is_encrypted"] is True for field in transform_fields) + + +class DummyField: + """Minimal field stand-in for duplicate message tests.""" + + def __init__(self, name, item, friendly_name): + """Store basic field metadata.""" + self.name = name + self.item = item + self.friendly_name = friendly_name + + +class DummySchema: + """Minimal schema stand-in for duplicate message tests.""" + + def __init__(self, fields): + """Map fields by name for test helpers.""" + self._fields = {field.name: field for field in fields} + + def get_partial_dup_fields(self): + """Return partial duplicate field names.""" + return list(self._fields.keys()) + + def get_field_by_name(self, name): + """Return field object by name.""" + return self._fields[name] + + +class DummyParser(BaseParser): + """Parser stub to access BaseParser helpers.""" + + def __init__(self): + """No-op init for isolated BaseParser helper tests.""" + pass + + def parse_and_validate(self): + """Implement required abstract method stub.""" + pass + + +class TestBaseParserMessages: + """Tests for BaseParser duplicate message helpers.""" + + def test_generate_exact_dup_error_msg(self): + """Generate exact duplicate error message.""" + parser = DummyParser() + msg = parser._generate_exact_dup_error_msg(None, "T1", 3, 2) + assert ( + msg + == "Duplicate record detected with record type T1 at line 3. " + "Record is a duplicate of the record at line number 2." + ) + + def test_generate_partial_dup_error_msg_single_field(self): + """Generate partial duplicate message for one field.""" + parser = DummyParser() + schema = DummySchema([DummyField("case_number", 4, "Case Number")]) + + msg = parser._generate_partial_dup_error_msg(schema, "T1", 3, 2) + + assert "Partial duplicate record detected with record type T1 at line 3." in msg + assert msg.endswith("Item 4 (Case Number).") + + def test_generate_partial_dup_error_msg_multiple_fields(self): + """Generate partial duplicate message for multiple fields.""" + parser = DummyParser() + schema = DummySchema( + [ + DummyField("case_number", 4, "Case Number"), + DummyField("rpt_month_year", 5, "Reporting Month Year"), + ] + ) + + msg = parser._generate_partial_dup_error_msg(schema, "T1", 3, 2) + + assert "Duplicated fields causing error:" in msg + assert "Item 4 (Case Number), and Item 5 (Reporting Month Year)." in msg diff --git a/tdrs-backend/tdpservice/parsers/test/test_serializer.py b/tdrs-backend/tdpservice/parsers/test/test_serializer.py index 402d76bc9..d0f2fadcc 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_serializer.py +++ b/tdrs-backend/tdpservice/parsers/test/test_serializer.py @@ -6,50 +6,49 @@ from tdpservice.parsers.test.factories import ParserErrorFactory -@pytest.fixture -def parser_error_instance(): - """Create a parser error instance.""" - return ParserErrorFactory.create() - - -@pytest.mark.django_db -def test_serializer_with_valid_data(parser_error_instance): - """If a serializer has valid data it will return a valid object.""" - request_instance = HttpRequest() - request_instance.query_params = { - "fields": "file, row_number, column_number, item_number, field_name,\ - category, error_message, error_type, created_at, fields_json, object_id,\ - case_number, rpt_month_year, content_type_id" - } - serializer = ParsingErrorSerializer( - parser_error_instance, - data={}, - context={"request": request_instance}, - partial=True, +class TestParsingErrorSerializer: + """Tests for parsing error serializer.""" + + @pytest.fixture + def parser_error_instance(self): + """Create a parser error instance.""" + return ParserErrorFactory.create() + + @pytest.mark.django_db + @pytest.mark.parametrize( + "fields,data,expected", + [ + ( + "file, row_number, column_number, item_number, field_name, " + "category, error_message, error_type, created_at, fields_json, " + "object_id, case_number, rpt_month_year, content_type_id", + {}, + True, + ), + ( + "file, row_number, column_number, item_number, field_name, " + "category, error_message, error_type, created_at, fields_json, " + "object_id, content_type_id", + {"row_number": "row number"}, + False, + ), + ], ) - assert serializer.is_valid() is True - - -@pytest.mark.django_db -def test_serializer_with_invalid_data(parser_error_instance): - """If a serializer has invalid data it will return an invalid object.""" - request_instance = HttpRequest() - request_instance.query_params = { - "fields": "file, row_number, column_number, item_number, field_name,\ - category, error_message, error_type, created_at, fields_json, object_id, content_type_id" - } - serializer = ParsingErrorSerializer( - parser_error_instance, - data={"row_number": "row number"}, - context={"request": request_instance}, - partial=True, - ) - assert serializer.is_valid() is False - - -@pytest.mark.django_db -def test_serializer_with_no_context(parser_error_instance): - """If a serializer has no context it will return an invalid object.""" - with pytest.raises(Exception) as e: - ParsingErrorSerializer(parser_error_instance, data={}, partial=True) - assert str(e.value) == "'context'" + def test_serializer_is_valid(self, parser_error_instance, fields, data, expected): + """Test serializer validity for positive and negative cases.""" + request_instance = HttpRequest() + request_instance.query_params = {"fields": fields} + serializer = ParsingErrorSerializer( + parser_error_instance, + data=data, + context={"request": request_instance}, + partial=True, + ) + assert serializer.is_valid() is expected + + @pytest.mark.django_db + def test_serializer_with_no_context(self, parser_error_instance): + """If a serializer has no context it will return an invalid object.""" + with pytest.raises(Exception) as e: + ParsingErrorSerializer(parser_error_instance, data={}, partial=True) + assert str(e.value) == "'context'" diff --git a/tdrs-backend/tdpservice/parsers/test/test_transforms.py b/tdrs-backend/tdpservice/parsers/test/test_transforms.py index 72019ae03..0029d4294 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_transforms.py +++ b/tdrs-backend/tdpservice/parsers/test/test_transforms.py @@ -9,38 +9,47 @@ ) -@pytest.mark.parametrize( - "value,digits,expected", - [ - ("1", 3, "001"), - ("10", 3, "010"), - ("100", 3, "100"), - ("1000", 3, "1000"), - ("1 ", 3, "01 "), - ("1 ", 3, "1 "), - ("1", 0, "1"), - ("1", -1, "1"), - ], -) -def test_zero_pad(value, digits, expected): - """Test zero_pad returns valid value.""" - transform = transforms.zero_pad(digits) - result = transform(value) - - assert result == expected - - -def test_tanf_ssn_decryption_func(): - """Test the TANF SSN decryption function.""" - assert tanf_ssn_decryption_func(None) is None - assert tanf_ssn_decryption_func("TANFSSN", is_encrypted=True) == "0ANFSSN" - assert tanf_ssn_decryption_func("TANFSSN", is_encrypted=False) == "TANFSSN" - assert tanf_ssn_decryption_func("@90#Y0B", is_encrypted=True) == "1256758" - - -def test_ssp_ssn_decryption_func(): - """Test the SSP SSN decryption function.""" - assert ssp_ssn_decryption_func(None) is None - assert ssp_ssn_decryption_func("SSPSSN", is_encrypted=False) == "SSPSSN" - assert ssp_ssn_decryption_func("SSPSSN", is_encrypted=True) == "SS4SSN" - assert ssp_ssn_decryption_func("@90#Y0B", is_encrypted=True) == "1256758" +class TestParserTransforms: + """Tests for parser transforms.""" + + @pytest.mark.parametrize( + "value,digits,expected", + [ + ("1", 3, "001"), + ("10", 3, "010"), + ("100", 3, "100"), + ("1000", 3, "1000"), + ("1 ", 3, "01 "), + ("1 ", 3, "1 "), + ("1", 0, "1"), + ("1", -1, "1"), + ], + ) + def test_zero_pad(self, value, digits, expected): + """Test zero_pad returns valid value.""" + transform = transforms.zero_pad(digits) + result = transform(value) + + assert result == expected + + @pytest.mark.parametrize( + "decrypt_func,value,is_encrypted,expected", + [ + (tanf_ssn_decryption_func, None, None, None), + (tanf_ssn_decryption_func, "TANFSSN", True, "0ANFSSN"), + (tanf_ssn_decryption_func, "TANFSSN", False, "TANFSSN"), + (tanf_ssn_decryption_func, "@90#Y0B", True, "1256758"), + (ssp_ssn_decryption_func, None, None, None), + (ssp_ssn_decryption_func, "SSPSSN", False, "SSPSSN"), + (ssp_ssn_decryption_func, "SSPSSN", True, "SS4SSN"), + (ssp_ssn_decryption_func, "@90#Y0B", True, "1256758"), + ], + ) + def test_ssn_decryption_func(self, decrypt_func, value, is_encrypted, expected): + """Test SSN decryption functions for base, positive, and negative cases.""" + if is_encrypted is None: + result = decrypt_func(value) + else: + result = decrypt_func(value, is_encrypted=is_encrypted) + + assert result == expected diff --git a/tdrs-backend/tdpservice/parsers/test/test_util.py b/tdrs-backend/tdpservice/parsers/test/test_util.py index 02c713cec..1c0359028 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_util.py +++ b/tdrs-backend/tdpservice/parsers/test/test_util.py @@ -45,530 +45,519 @@ def validator_to_deprecate(): return make_validator(lambda _: False, lambda _: "Failed.") -def test_deprecate_validator(): - """Test completely deprecated validator.""" - row = RawRow(data="12345", raw_len=5, decoded_len=5, row_num=1, record_type="") - schema = HeaderSchema(model=None, preparsing_validators=[deprecated_validator()]) - - schema.prepare(None) - is_valid, errors = schema.run_preparsing_validators(row, None) - assert is_valid is False - assert len(errors) == 1 - - error = errors[0] - assert error.error_message == "Failed." - assert error.deprecated is True - - -def test_deprecate_call(): - """Test deprecated invocation of a validator.""" - row = RawRow(data="12345", raw_len=5, decoded_len=5, row_num=1, record_type="") - schema = HeaderSchema( - model=None, - preparsing_validators=[ - deprecate_call(validator_to_deprecate()), - passing_validator(), +class TestParserUtil: + """Tests for parser utilities.""" + + def test_deprecate_validator(self): + """Test completely deprecated validator.""" + row = RawRow(data="12345", raw_len=5, decoded_len=5, row_num=1, record_type="") + schema = HeaderSchema(model=None, preparsing_validators=[deprecated_validator()]) + + schema.prepare(None) + is_valid, errors = schema.run_preparsing_validators(row, None) + assert is_valid is False + assert len(errors) == 1 + + error = errors[0] + assert error.error_message == "Failed." + assert error.deprecated is True + + def test_deprecate_call(self): + """Test deprecated invocation of a validator.""" + row = RawRow(data="12345", raw_len=5, decoded_len=5, row_num=1, record_type="") + schema = HeaderSchema( + model=None, + preparsing_validators=[ + deprecate_call(validator_to_deprecate()), + passing_validator(), + ], + ) + schema.prepare(None) + + is_valid, errors = schema.run_preparsing_validators(row, None) + assert is_valid is False + assert len(errors) == 1 + + error = errors[0] + assert error.error_message == "Failed." + assert error.deprecated is True + + def test_run_preparsing_validators_returns_valid(self): + """Test run_preparsing_validators executes all preparsing_validators provided in schema.""" + row = RawRow(data="12345", raw_len=5, decoded_len=5, row_num=1, record_type="") + schema = HeaderSchema(model=None, preparsing_validators=[passing_validator()]) + schema.prepare(None) + + is_valid, errors = schema.run_preparsing_validators(row, None) + assert is_valid is True + assert errors == [] + + def test_run_preparsing_validators_returns_invalid_and_errors(self): + """Test that run_preparsing_validators executes all preparsing_validators provided in schema and returns errors.""" + row = RawRow(data="12345", raw_len=5, decoded_len=5, row_num=1, record_type="") + schema = HeaderSchema( + model=None, preparsing_validators=[passing_validator(), failing_validator()] + ) + schema.prepare(None) + + is_valid, errors = schema.run_preparsing_validators(row, None) + assert is_valid is False + assert errors[0].error_message == "Value is not valid." + + def test_parse_line_parses_line_from_schema_to_dict(self): + """Test that parse_line parses a string into a dict given start and end indices for all fields.""" + line = "12345001" + schema = HeaderSchema( + model=dict, + fields=[ + Field( + item=1, + name="first", + friendly_name="first", + type=FieldType.ALPHA_NUMERIC, + startIndex=0, + endIndex=3, + ), + Field( + item=2, + name="second", + friendly_name="second", + type=FieldType.ALPHA_NUMERIC, + startIndex=3, + endIndex=4, + ), + Field( + item=3, + name="third", + friendly_name="third", + type=FieldType.ALPHA_NUMERIC, + startIndex=4, + endIndex=5, + ), + Field( + item=4, + name="fourth", + friendly_name="fourth", + type=FieldType.NUMERIC, + startIndex=5, + endIndex=7, + ), + Field( + item=5, + name="fifth", + friendly_name="fifth", + type=FieldType.NUMERIC, + startIndex=7, + endIndex=8, + ), + ], + ) + + length = len(line) + row = RawRow( + data=line, raw_len=length, decoded_len=length, row_num=1, record_type="" + ) + record = schema.parse_row(row) + + assert record["first"] == "123" + assert record["second"] == "4" + assert record["third"] == "5" + assert record["fourth"] == 0 + assert record["fifth"] == 1 + + def test_parse_line_parses_line_from_schema_to_object(self): + """Test that parse_line parses a string into an object given start and end indices for all fields.""" + + class TestModel: + first = None + second = None + third = None + fourth = None + fifth = None + + line = "12345001" + schema = HeaderSchema( + model=TestModel, + fields=[ + Field( + item=1, + name="first", + friendly_name="first", + type=FieldType.ALPHA_NUMERIC, + startIndex=0, + endIndex=3, + ), + Field( + item=2, + name="second", + friendly_name="second", + type=FieldType.ALPHA_NUMERIC, + startIndex=3, + endIndex=4, + ), + Field( + item=3, + name="third", + friendly_name="third", + type=FieldType.ALPHA_NUMERIC, + startIndex=4, + endIndex=5, + ), + Field( + item=4, + name="fourth", + friendly_name="fourth", + type=FieldType.NUMERIC, + startIndex=5, + endIndex=7, + ), + Field( + item=5, + name="fifth", + friendly_name="fifth", + type=FieldType.NUMERIC, + startIndex=7, + endIndex=8, + ), + ], + ) + + length = len(line) + row = RawRow( + data=line, raw_len=length, decoded_len=length, row_num=1, record_type="" + ) + record = schema.parse_row(row) + + assert record.first == "123" + assert record.second == "4" + assert record.third == "5" + assert record.fourth == 0 + assert record.fifth == 1 + + def test_run_field_validators_returns_valid_with_dict(self): + """Test that run_field_validators can validate all fields against parsed data dict.""" + instance = {"first": "123", "second": "4", "third": "5"} + schema = HeaderSchema( + model=None, + fields=[ + Field( + item=1, + name="first", + friendly_name="first", + type=FieldType.ALPHA_NUMERIC, + startIndex=0, + endIndex=3, + validators=[passing_validator()], + ), + Field( + item=2, + name="second", + friendly_name="second", + type=FieldType.ALPHA_NUMERIC, + startIndex=3, + endIndex=4, + validators=[passing_validator()], + ), + Field( + item=3, + name="third", + friendly_name="third", + type=FieldType.ALPHA_NUMERIC, + startIndex=4, + endIndex=5, + validators=[passing_validator()], + ), + ], + ) + schema.prepare(None) + + is_valid, errors = schema.run_field_validators(instance, 1) + assert is_valid is True + assert errors == [] + + def test_run_field_validators_returns_valid_with_object(self): + """Test that run_field_validators can validate all fields against parsed data object.""" + + class TestModel: + first = None + second = None + third = None + + instance = TestModel + instance.first = "123" + instance.second = "4" + instance.third = "5" + + model = instance + + schema = HeaderSchema( + model=model, + fields=[ + Field( + item=1, + name="first", + friendly_name="first", + type=FieldType.ALPHA_NUMERIC, + startIndex=0, + endIndex=3, + validators=[passing_validator()], + ), + Field( + item=2, + name="second", + friendly_name="second", + type=FieldType.ALPHA_NUMERIC, + startIndex=3, + endIndex=4, + validators=[passing_validator()], + ), + Field( + item=3, + name="third", + friendly_name="third", + type=FieldType.ALPHA_NUMERIC, + startIndex=4, + endIndex=5, + validators=[passing_validator()], + ), + ], + ) + schema.prepare(None) + + is_valid, errors = schema.run_field_validators(instance, 1) + assert is_valid is True + assert errors == [] + + def test_run_field_validators_returns_invalid_with_dict(self): + """Test that run_field_validators can validate all fields against parsed data dict and return errors.""" + instance = {"first": "123", "second": "4", "third": "5"} + schema = HeaderSchema( + model=None, + fields=[ + Field( + item=1, + name="first", + friendly_name="first", + type=FieldType.ALPHA_NUMERIC, + startIndex=0, + endIndex=3, + validators=[passing_validator(), failing_validator()], + ), + Field( + item=2, + name="second", + friendly_name="second", + type=FieldType.ALPHA_NUMERIC, + startIndex=3, + endIndex=4, + validators=[passing_validator()], + ), + Field( + item=3, + name="third", + friendly_name="third", + type=FieldType.ALPHA_NUMERIC, + startIndex=4, + endIndex=5, + validators=[passing_validator()], + ), + ], + ) + schema.prepare(None) + + is_valid, errors = schema.run_field_validators(instance, 1) + assert is_valid is False + assert errors[0].error_message == "Value is not valid." + + def test_run_field_validators_returns_invalid_with_object(self): + """Test that run_field_validators can validate all fields against parsed data object and return errors.""" + + class TestModel: + first = None + second = None + third = None + + instance = TestModel + instance.first = "123" + instance.second = "4" + instance.third = "5" + + model = instance + + schema = HeaderSchema( + model=model, + fields=[ + Field( + item=1, + name="first", + friendly_name="first", + type=FieldType.ALPHA_NUMERIC, + startIndex=0, + endIndex=3, + validators=[passing_validator(), failing_validator()], + ), + Field( + item=2, + name="second", + friendly_name="second", + type=FieldType.ALPHA_NUMERIC, + startIndex=3, + endIndex=4, + validators=[passing_validator()], + ), + Field( + item=3, + name="third", + friendly_name="third", + type=FieldType.ALPHA_NUMERIC, + startIndex=4, + endIndex=5, + validators=[passing_validator()], + ), + ], + ) + schema.prepare(None) + + is_valid, errors = schema.run_field_validators(instance, 1) + assert is_valid is False + assert errors[0].error_message == "Value is not valid." + + @pytest.mark.parametrize( + "first,second", + [ + ("", ""), + (" ", " "), + ("#", "##"), + (None, None), ], ) - schema.prepare(None) - - is_valid, errors = schema.run_preparsing_validators(row, None) - assert is_valid is False - assert len(errors) == 1 - - error = errors[0] - assert error.error_message == "Failed." - assert error.deprecated is True - - -def test_run_preparsing_validators_returns_valid(): - """Test run_preparsing_validators executes all preparsing_validators provided in schema.""" - row = RawRow(data="12345", raw_len=5, decoded_len=5, row_num=1, record_type="") - schema = HeaderSchema(model=None, preparsing_validators=[passing_validator()]) - schema.prepare(None) - - is_valid, errors = schema.run_preparsing_validators(row, None) - assert is_valid is True - assert errors == [] - - -def test_run_preparsing_validators_returns_invalid_and_errors(): - """Test that run_preparsing_validators executes all preparsing_validators provided in schema and returns errors.""" - row = RawRow(data="12345", raw_len=5, decoded_len=5, row_num=1, record_type="") - schema = HeaderSchema( - model=None, preparsing_validators=[passing_validator(), failing_validator()] - ) - schema.prepare(None) - - is_valid, errors = schema.run_preparsing_validators(row, None) - assert is_valid is False - assert errors[0].error_message == "Value is not valid." - - -def test_parse_line_parses_line_from_schema_to_dict(): - """Test that parse_line parses a string into a dict given start and end indices for all fields.""" - line = "12345001" - schema = HeaderSchema( - model=dict, - fields=[ - Field( - item=1, - name="first", - friendly_name="first", - type=FieldType.ALPHA_NUMERIC, - startIndex=0, - endIndex=3, - ), - Field( - item=2, - name="second", - friendly_name="second", - type=FieldType.ALPHA_NUMERIC, - startIndex=3, - endIndex=4, - ), - Field( - item=3, - name="third", - friendly_name="third", - type=FieldType.ALPHA_NUMERIC, - startIndex=4, - endIndex=5, - ), - Field( - item=4, - name="fourth", - friendly_name="fourth", - type=FieldType.NUMERIC, - startIndex=5, - endIndex=7, - ), - Field( - item=5, - name="fifth", - friendly_name="fifth", - type=FieldType.NUMERIC, - startIndex=7, - endIndex=8, - ), - ], - ) - - length = len(line) - row = RawRow( - data=line, raw_len=length, decoded_len=length, row_num=1, record_type="" - ) - record = schema.parse_row(row) - - assert record["first"] == "123" - assert record["second"] == "4" - assert record["third"] == "5" - assert record["fourth"] == 0 - assert record["fifth"] == 1 - - -def test_parse_line_parses_line_from_schema_to_object(): - """Test that parse_line parses a string into an object given start and end indices for all fields.""" - - class TestModel: - first = None - second = None - third = None - fourth = None - fifth = None - - line = "12345001" - schema = HeaderSchema( - model=TestModel, - fields=[ - Field( - item=1, - name="first", - friendly_name="first", - type=FieldType.ALPHA_NUMERIC, - startIndex=0, - endIndex=3, - ), - Field( - item=2, - name="second", - friendly_name="second", - type=FieldType.ALPHA_NUMERIC, - startIndex=3, - endIndex=4, - ), - Field( - item=3, - name="third", - friendly_name="third", - type=FieldType.ALPHA_NUMERIC, - startIndex=4, - endIndex=5, - ), - Field( - item=4, - name="fourth", - friendly_name="fourth", - type=FieldType.NUMERIC, - startIndex=5, - endIndex=7, - ), - Field( - item=5, - name="fifth", - friendly_name="fifth", - type=FieldType.NUMERIC, - startIndex=7, - endIndex=8, - ), - ], - ) - - length = len(line) - row = RawRow( - data=line, raw_len=length, decoded_len=length, row_num=1, record_type="" - ) - record = schema.parse_row(row) - - assert record.first == "123" - assert record.second == "4" - assert record.third == "5" - assert record.fourth == 0 - assert record.fifth == 1 - - -def test_run_field_validators_returns_valid_with_dict(): - """Test that run_field_validators can validate all fields against parsed data dict.""" - instance = {"first": "123", "second": "4", "third": "5"} - schema = HeaderSchema( - model=None, - fields=[ - Field( - item=1, - name="first", - friendly_name="first", - type=FieldType.ALPHA_NUMERIC, - startIndex=0, - endIndex=3, - validators=[passing_validator()], - ), - Field( - item=2, - name="second", - friendly_name="second", - type=FieldType.ALPHA_NUMERIC, - startIndex=3, - endIndex=4, - validators=[passing_validator()], - ), - Field( - item=3, - name="third", - friendly_name="third", - type=FieldType.ALPHA_NUMERIC, - startIndex=4, - endIndex=5, - validators=[passing_validator()], - ), - ], - ) - schema.prepare(None) - - is_valid, errors = schema.run_field_validators(instance, 1) - assert is_valid is True - assert errors == [] - - -def test_run_field_validators_returns_valid_with_object(): - """Test that run_field_validators can validate all fields against parsed data object.""" - - class TestModel: - first = None - second = None - third = None - - instance = TestModel - instance.first = "123" - instance.second = "4" - instance.third = "5" - - model = instance - - schema = HeaderSchema( - model=model, - fields=[ - Field( - item=1, - name="first", - friendly_name="first", - type=FieldType.ALPHA_NUMERIC, - startIndex=0, - endIndex=3, - validators=[passing_validator()], - ), - Field( - item=2, - name="second", - friendly_name="second", - type=FieldType.ALPHA_NUMERIC, - startIndex=3, - endIndex=4, - validators=[passing_validator()], - ), - Field( - item=3, - name="third", - friendly_name="third", - type=FieldType.ALPHA_NUMERIC, - startIndex=4, - endIndex=5, - validators=[passing_validator()], - ), - ], - ) - schema.prepare(None) - - is_valid, errors = schema.run_field_validators(instance, 1) - assert is_valid is True - assert errors == [] - - -def test_run_field_validators_returns_invalid_with_dict(): - """Test that run_field_validators can validate all fields against parsed data dict and return errors.""" - instance = {"first": "123", "second": "4", "third": "5"} - schema = HeaderSchema( - model=None, - fields=[ - Field( - item=1, - name="first", - friendly_name="first", - type=FieldType.ALPHA_NUMERIC, - startIndex=0, - endIndex=3, - validators=[passing_validator(), failing_validator()], - ), - Field( - item=2, - name="second", - friendly_name="second", - type=FieldType.ALPHA_NUMERIC, - startIndex=3, - endIndex=4, - validators=[passing_validator()], - ), - Field( - item=3, - name="third", - friendly_name="third", - type=FieldType.ALPHA_NUMERIC, - startIndex=4, - endIndex=5, - validators=[passing_validator()], - ), - ], - ) - schema.prepare(None) - - is_valid, errors = schema.run_field_validators(instance, 1) - assert is_valid is False - assert errors[0].error_message == "Value is not valid." - - -def test_run_field_validators_returns_invalid_with_object(): - """Test that run_field_validators can validate all fields against parsed data object and return errors.""" - - class TestModel: - first = None - second = None - third = None - - instance = TestModel - instance.first = "123" - instance.second = "4" - instance.third = "5" - - model = instance - - schema = HeaderSchema( - model=model, - fields=[ - Field( - item=1, - name="first", - friendly_name="first", - type=FieldType.ALPHA_NUMERIC, - startIndex=0, - endIndex=3, - validators=[passing_validator(), failing_validator()], - ), - Field( - item=2, - name="second", - friendly_name="second", - type=FieldType.ALPHA_NUMERIC, - startIndex=3, - endIndex=4, - validators=[passing_validator()], - ), - Field( - item=3, - name="third", - friendly_name="third", - type=FieldType.ALPHA_NUMERIC, - startIndex=4, - endIndex=5, - validators=[passing_validator()], - ), + def test_field_validators_blank_and_required_returns_error(self, first, second): + """Test required field returns error if value not provided (blank).""" + instance = { + "first": first, + "second": second, + } + schema = HeaderSchema( + model=None, + fields=[ + Field( + item=1, + name="first", + friendly_name="first", + type=FieldType.ALPHA_NUMERIC, + startIndex=0, + endIndex=1, + required=True, + validators=[ + passing_validator(), + ], + ), + Field( + item=2, + name="second", + friendly_name="second", + type=FieldType.ALPHA_NUMERIC, + startIndex=1, + endIndex=3, + required=True, + validators=[ + passing_validator(), + ], + ), + ], + ) + schema.prepare(None) + + is_valid, errors = schema.run_field_validators(instance, 1) + assert is_valid is False + assert ( + errors[0].error_message + == "HEADER Item 1 (first): field is required but a value was not provided." + ) + assert ( + errors[1].error_message + == "HEADER Item 2 (second): field is required but a value was not provided." + ) + + @pytest.mark.parametrize( + "first, expected_valid, expected_errors", + [ + (" ", True, []), + ("####", True, []), + (None, True, []), ], ) - schema.prepare(None) - - is_valid, errors = schema.run_field_validators(instance, 1) - assert is_valid is False - assert errors[0].error_message == "Value is not valid." - - -@pytest.mark.parametrize( - "first,second", - [ - ("", ""), - (" ", " "), - ("#", "##"), - (None, None), - ], -) -def test_field_validators_blank_and_required_returns_error(first, second): - """Test required field returns error if value not provided (blank).""" - instance = { - "first": first, - "second": second, - } - schema = HeaderSchema( - model=None, - fields=[ - Field( - item=1, - name="first", - friendly_name="first", - type=FieldType.ALPHA_NUMERIC, - startIndex=0, - endIndex=1, - required=True, - validators=[ - passing_validator(), - ], - ), - Field( - item=2, - name="second", - friendly_name="second", - type=FieldType.ALPHA_NUMERIC, - startIndex=1, - endIndex=3, - required=True, - validators=[ - passing_validator(), - ], - ), + def test_field_validators_blank_and_not_required_returns_valid( + self, + first, expected_valid, expected_errors + ): + """Test not required field returns valid if value not provided (blank).""" + instance = { + "first": first, + } + schema = HeaderSchema( + model=None, + fields=[ + Field( + item=1, + name="first", + friendly_name="first", + type=FieldType.ALPHA_NUMERIC, + startIndex=0, + endIndex=3, + required=False, + validators=[passing_validator(), failing_validator()], + ), + ], + ) + schema.prepare(None) + + is_valid, errors = schema.run_field_validators(instance, 1) + assert is_valid is expected_valid + assert errors == expected_errors + + def test_run_postparsing_validators_returns_valid(self): + """Test run_postparsing_validators executes all postparsing_validators provided in schema.""" + instance = {} + schema = HeaderSchema(model=None, postparsing_validators=[passing_validator()]) + schema.prepare(None) + + is_valid, errors = schema.run_postparsing_validators(instance, 1) + assert is_valid is True + assert errors == [] + + @pytest.fixture + def test_datafile_empty_file(self, stt_user, stt): + """Fixture for empty_file.""" + return create_test_datafile("empty_file", stt_user, stt) + + @pytest.mark.parametrize( + "rpt_date_str,date_str,expected", + [ + ("20200102", "20100101", 10), + ("20200102", "20100106", 9), + ("20200101", "20200102", 0), + ("20200101", "20210102", -1), ], ) - schema.prepare(None) - - is_valid, errors = schema.run_field_validators(instance, 1) - assert is_valid is False - assert ( - errors[0].error_message - == "HEADER Item 1 (first): field is required but a value was not provided." - ) - assert ( - errors[1].error_message - == "HEADER Item 2 (second): field is required but a value was not provided." - ) - - -@pytest.mark.parametrize( - "first, expected_valid, expected_errors", - [ - (" ", True, []), - ("####", True, []), - (None, True, []), - ], -) -def test_field_validators_blank_and_not_required_returns_valid( - first, expected_valid, expected_errors -): - """Test not required field returns valid if value not provided (blank).""" - instance = { - "first": first, - } - schema = HeaderSchema( - model=None, - fields=[ - Field( - item=1, - name="first", - friendly_name="first", - type=FieldType.ALPHA_NUMERIC, - startIndex=0, - endIndex=3, - required=False, - validators=[passing_validator(), failing_validator()], - ), + def test_get_years_apart(self, rpt_date_str, date_str, expected): + """Test the get_years_apart util function.""" + rpt_date = datetime.strptime(rpt_date_str, "%Y%m%d") + date = datetime.strptime(date_str, "%Y%m%d") + assert int(get_years_apart(rpt_date, date)) == expected + + @pytest.mark.parametrize( + "options, expected", + [ + ([1, 2, 3, 4], "[1, 2, 3, 4]"), + (["1", "2", "3", "4"], "[1, 2, 3, 4]"), + (["a", "b", "c", "d"], "[a, b, c, d]"), + (("a", "b", "c", "d"), "[a, b, c, d]"), + (["'a'", "'b'", "'c'", "'d'"], "['a', 'b', 'c', 'd']"), + (["words", "are very", "weird"], "[words, are very, weird]"), ], ) - schema.prepare(None) - - is_valid, errors = schema.run_field_validators(instance, 1) - assert is_valid is expected_valid - assert errors == expected_errors - - -def test_run_postparsing_validators_returns_valid(): - """Test run_postparsing_validators executes all postparsing_validators provided in schema.""" - instance = {} - schema = HeaderSchema(model=None, postparsing_validators=[passing_validator()]) - schema.prepare(None) - - is_valid, errors = schema.run_postparsing_validators(instance, 1) - assert is_valid is True - assert errors == [] - - -@pytest.fixture -def test_datafile_empty_file(stt_user, stt): - """Fixture for empty_file.""" - return create_test_datafile("empty_file", stt_user, stt) - - -@pytest.mark.parametrize( - "rpt_date_str,date_str,expected", - [ - ("20200102", "20100101", 10), - ("20200102", "20100106", 9), - ("20200101", "20200102", 0), - ("20200101", "20210102", -1), - ], -) -def test_get_years_apart(rpt_date_str, date_str, expected): - """Test the get_years_apart util function.""" - rpt_date = datetime.strptime(rpt_date_str, "%Y%m%d") - date = datetime.strptime(date_str, "%Y%m%d") - assert int(get_years_apart(rpt_date, date)) == expected - - -@pytest.mark.parametrize( - "options, expected", - [ - ([1, 2, 3, 4], "[1, 2, 3, 4]"), - (["1", "2", "3", "4"], "[1, 2, 3, 4]"), - (["a", "b", "c", "d"], "[a, b, c, d]"), - (("a", "b", "c", "d"), "[a, b, c, d]"), - (["'a'", "'b'", "'c'", "'d'"], "['a', 'b', 'c', 'd']"), - (["words", "are very", "weird"], "[words, are very, weird]"), - ], -) -def test_clean_options_string(options, expected): - """Test `clean_options_string` util func.""" - result = clean_options_string(options) - assert result == expected + def test_clean_options_string(self, options, expected): + """Test `clean_options_string` util func.""" + result = clean_options_string(options) + assert result == expected diff --git a/tdrs-backend/tdpservice/parsers/util.py b/tdrs-backend/tdpservice/parsers/util.py index bb1de0000..0769932a9 100644 --- a/tdrs-backend/tdpservice/parsers/util.py +++ b/tdrs-backend/tdpservice/parsers/util.py @@ -60,11 +60,6 @@ def fiscal_to_calendar(year, fiscal_quarter): return year, "Q{}".format(array[ind_qtr - 1]) # return the previous quarter -def calendar_to_fiscal(calendar_year, fiscal_quarter): - """Decrement the calendar year if in Q1.""" - return calendar_year - 1 if fiscal_quarter == "Q1" else calendar_year - - def transform_to_months(quarter): """Return a list of months in a quarter depending the quarter's format.""" match quarter: diff --git a/tdrs-backend/tdpservice/scheduling/parser_task.py b/tdrs-backend/tdpservice/scheduling/parser_task.py index 28cf127ec..c0c5ad641 100644 --- a/tdrs-backend/tdpservice/scheduling/parser_task.py +++ b/tdrs-backend/tdpservice/scheduling/parser_task.py @@ -36,11 +36,11 @@ logger = settings.PARSER_LOGGER -def set_reparse_file_meta_model_failed_state(reparse_id, file_meta): +def set_reparse_file_meta_model_state(reparse_id, file_meta, is_success): """Set ReparseFileMeta fields to indicate a parse failure.""" if reparse_id: file_meta.finished = True - file_meta.success = False + file_meta.success = is_success file_meta.finished_at = timezone.now() file_meta.save() @@ -81,6 +81,7 @@ def parse(data_file_id, reparse_id=None): ) file_meta = None + reparse_success = True if reparse_id: file_meta = ReparseFileMeta.objects.get( data_file_id=data_file_id, reparse_meta_id=reparse_id @@ -106,20 +107,7 @@ def parse(data_file_id, reparse_id=None): f"{dfs.status}." ) - if reparse_id is not None: - file_meta.num_records_created = dfs.total_number_of_records_created - file_meta.cat_4_errors_generated = ParserError.objects.filter( - file_id=data_file_id, - error_type=ParserErrorCategoryChoices.CASE_CONSISTENCY, - ).count() - file_meta.finished = True - file_meta.success = True - file_meta.finished_at = timezone.now() - file_meta.save() - ReparseMeta.set_total_num_records_post( - ReparseMeta.objects.get(pk=reparse_id) - ) - else: + if reparse_id is None: qs = User.objects.filter( stt=data_file.stt, account_approval_status=AccountApprovalStatusChoices.APPROVED, @@ -135,14 +123,14 @@ def parse(data_file_id, reparse_id=None): except DecoderUnknownException: dfs.set_status(DataFileSummary.Status.REJECTED) dfs.save() - set_reparse_file_meta_model_failed_state(reparse_id, file_meta) + reparse_success = False except DatabaseError as e: log_parser_exception( data_file, f"Encountered Database exception in parser_task.py: \n{e}", "error", ) - set_reparse_file_meta_model_failed_state(reparse_id, file_meta) + reparse_success = False except Exception: generate_error = ErrorGeneratorFactory(data_file).get_generator( ErrorGeneratorType.MSG_ONLY_PRECHECK, @@ -169,7 +157,7 @@ def parse(data_file_id, reparse_id=None): ), "exception", ) - set_reparse_file_meta_model_failed_state(reparse_id, file_meta) + reparse_success = False finally: logger.info(f"DataFile parsing finished for file -> {repr(data_file)}.") error_report_generator = ErrorReportFactory.get_error_report_generator( @@ -179,3 +167,14 @@ def parse(data_file_id, reparse_id=None): set_error_report(dfs, error_report) logger.handlers[2].doRollover(data_file) update_dfs(dfs, data_file) + + if reparse_id is not None: + file_meta.num_records_created = dfs.total_number_of_records_created + file_meta.cat_4_errors_generated = ParserError.objects.filter( + file_id=data_file_id, + error_type=ParserErrorCategoryChoices.CASE_CONSISTENCY, + ).count() + ReparseMeta.set_total_num_records_post( + ReparseMeta.objects.get(pk=reparse_id) + ) + set_reparse_file_meta_model_state(reparse_id, file_meta, reparse_success) diff --git a/tdrs-backend/tdpservice/scheduling/test/test_parser_task.py b/tdrs-backend/tdpservice/scheduling/test/test_parser_task.py new file mode 100644 index 000000000..1ecb2e2e7 --- /dev/null +++ b/tdrs-backend/tdpservice/scheduling/test/test_parser_task.py @@ -0,0 +1,354 @@ +"""Tests for parser task helpers and flow control.""" + +import io +from types import SimpleNamespace + +import pytest +from django.db.utils import DatabaseError + +from tdpservice.data_files.models import DataFile, ReparseFileMeta +from tdpservice.data_files.test.factories import DataFileFactory +from tdpservice.parsers.models import DataFileSummary +from tdpservice.parsers.util import DecoderUnknownException +from tdpservice.scheduling import parser_task +from tdpservice.search_indexes.models.reparse_meta import ReparseMeta + + +class DummyHandler: + """Logger handler stub to capture rollover calls.""" + + def __init__(self): + self.called = False + self.level = 0 + + def doRollover(self, data_file): + """Record rollover invocation.""" + self.called = True + + def handle(self, record): + """No-op handler for logger internals.""" + return True + + +class DummyParser: + """Parser stub used for flow control tests.""" + + def __init__(self, exc=None): + self.exc = exc + self.called = False + + def parse_and_validate(self): + """Invoke a configured exception or no-op.""" + self.called = True + if self.exc is not None: + raise self.exc + + +DEFAULT_FILENAMES = { + DataFile.Section.ACTIVE_CASE_DATA: "ADS.E2J.FTP1.TS72", + DataFile.Section.CLOSED_CASE_DATA: "ADS.E2J.FTP2.TS72", + DataFile.Section.AGGREGATE_DATA: "ADS.E2J.FTP3.TS72", + DataFile.Section.STRATUM_DATA: "ADS.E2J.FTP4.TS72", + DataFile.Section.FRA_WORK_OUTCOME_TANF_EXITERS: "ADS.FRA.FTP1.TS72", + DataFile.Section.FRA_SECONDRY_SCHOOL_ATTAINMENT: "ADS.FRA.FTP2.TS72", + DataFile.Section.FRA_SUPPLEMENT_WORK_OUTCOMES: "ADS.FRA.FTP3.TS72", +} + + +def ensure_stt_filenames(stt): + """Set default STT filenames when missing to unblock parse logging.""" + if not stt.filenames: + stt.filenames = DEFAULT_FILENAMES.copy() + stt.save(update_fields=["filenames"]) + + +def setup_parse_mocks(monkeypatch, dfs=None): + """Patch common dependencies for parser_task.parse tests.""" + handlers = [DummyHandler(), DummyHandler(), DummyHandler()] + monkeypatch.setattr(parser_task.logger, "handlers", handlers, raising=False) + monkeypatch.setattr(parser_task, "change_log_filename", lambda *a, **k: None) + + def fake_update_dfs(dfs, data_file): + dfs.save() + + monkeypatch.setattr(parser_task, "update_dfs", fake_update_dfs) + monkeypatch.setattr(parser_task, "set_error_report", lambda *a, **k: None) + if dfs is not None: + monkeypatch.setattr( + parser_task.DataFileSummary.objects, "create", lambda **kwargs: dfs + ) + + class DummyReport: + def generate(self): + return io.BytesIO(b"report") + + monkeypatch.setattr( + parser_task.ErrorReportFactory, + "get_error_report_generator", + staticmethod(lambda data_file: DummyReport()), + ) + return handlers + + +@pytest.mark.django_db +def test_update_dfs_uses_fra_aggregates(monkeypatch, stt): + """Use FRA aggregates for FRA program types.""" + datafile = DataFileFactory( + stt=stt, + version=1, + program_type=DataFile.ProgramType.FRA, + section=DataFile.Section.FRA_WORK_OUTCOME_TANF_EXITERS, + ) + dfs = DataFileSummary.objects.create( + datafile=datafile, status=DataFileSummary.Status.ACCEPTED + ) + + monkeypatch.setattr(parser_task, "fra_total_errors", lambda df: {"fra": 1}) + + parser_task.update_dfs(dfs, datafile) + + dfs.refresh_from_db() + assert dfs.case_aggregates == {"fra": 1} + + +@pytest.mark.django_db +def test_update_dfs_uses_case_aggregates(monkeypatch, stt): + """Use case aggregates for case data sections.""" + datafile = DataFileFactory( + stt=stt, + version=2, + program_type=DataFile.ProgramType.TANF, + section=DataFile.Section.ACTIVE_CASE_DATA, + ) + dfs = DataFileSummary.objects.create( + datafile=datafile, status=DataFileSummary.Status.ACCEPTED + ) + + monkeypatch.setattr(parser_task, "case_aggregates_by_month", lambda *a: {"case": 2}) + monkeypatch.setattr( + parser_task, + "total_errors_by_month", + lambda *a: pytest.fail("total_errors_by_month should not be used"), + ) + + parser_task.update_dfs(dfs, datafile) + + dfs.refresh_from_db() + assert dfs.case_aggregates == {"case": 2} + + +@pytest.mark.django_db +def test_update_dfs_uses_total_errors(monkeypatch, stt): + """Use total errors for non-case data sections.""" + datafile = DataFileFactory( + stt=stt, + version=3, + program_type=DataFile.ProgramType.TANF, + section=DataFile.Section.AGGREGATE_DATA, + ) + dfs = DataFileSummary.objects.create( + datafile=datafile, status=DataFileSummary.Status.ACCEPTED + ) + + monkeypatch.setattr( + parser_task, + "case_aggregates_by_month", + lambda *a: pytest.fail("case_aggregates_by_month should not be used"), + ) + monkeypatch.setattr( + parser_task, "total_errors_by_month", lambda *a: {"total": 3} + ) + + parser_task.update_dfs(dfs, datafile) + + dfs.refresh_from_db() + assert dfs.case_aggregates == {"total": 3} + + +def test_set_error_report_sets_filename(): + """Set error report file name based on original filename.""" + + class DummyDataFile: + original_filename = "sample.txt" + + class DummySummary: + def __init__(self): + self.datafile = DummyDataFile() + self.error_report = None + self.saved = False + + def save(self): + self.saved = True + + dfs = DummySummary() + + parser_task.set_error_report(dfs, io.BytesIO(b"report")) + + assert dfs.saved is True + assert dfs.error_report.name == "sample.txt_error_report" + + +@pytest.mark.django_db +def test_parse_success_sends_email(monkeypatch, data_analyst): + """Send notification email on successful parse.""" + datafile = DataFileFactory(stt=data_analyst.stt, version=4) + ensure_stt_filenames(datafile.stt) + dfs = DataFileSummary.objects.create( + datafile=datafile, status=DataFileSummary.Status.PENDING + ) + handlers = setup_parse_mocks(monkeypatch, dfs=dfs) + dummy_parser = DummyParser() + + monkeypatch.setattr( + parser_task.ParserFactory, "get_instance", lambda **kwargs: dummy_parser + ) + + captured = {} + + def fake_send(dfs, recipients): + captured["recipients"] = list(recipients) + + monkeypatch.setattr(parser_task, "send_data_submitted_email", fake_send) + + parser_task.parse(datafile.id) + + assert dummy_parser.called is True + assert data_analyst.username in captured["recipients"] + assert handlers[2].called is True + + +@pytest.mark.django_db +def test_parse_success_reparse_updates_file_meta(monkeypatch, stt): + """Update reparse metadata on success.""" + datafile = DataFileFactory(stt=stt, version=5) + ensure_stt_filenames(datafile.stt) + dfs = DataFileSummary.objects.create( + datafile=datafile, status=DataFileSummary.Status.PENDING + ) + meta_model = ReparseMeta.objects.create(db_backup_location="s3://backup") + file_meta = ReparseFileMeta.objects.create( + data_file=datafile, reparse_meta=meta_model + ) + setup_parse_mocks(monkeypatch, dfs=dfs) + dummy_parser = DummyParser() + + monkeypatch.setattr( + parser_task.ParserFactory, "get_instance", lambda **kwargs: dummy_parser + ) + monkeypatch.setattr( + parser_task.ParserError.objects, + "filter", + lambda *a, **k: SimpleNamespace(count=lambda: 2), + ) + monkeypatch.setattr( + parser_task.ReparseMeta, "set_total_num_records_post", lambda *a, **k: None + ) + + parser_task.parse(datafile.id, reparse_id=meta_model.pk) + + file_meta.refresh_from_db() + assert file_meta.finished is True + assert file_meta.success is True + assert file_meta.cat_4_errors_generated == 2 + assert file_meta.finished_at is not None + + +@pytest.mark.django_db +def test_parse_decoder_unknown_sets_reparse_failed(monkeypatch, stt): + """Set rejected status and failed reparse state on decode errors.""" + datafile = DataFileFactory(stt=stt, version=6) + ensure_stt_filenames(datafile.stt) + dfs = DataFileSummary.objects.create( + datafile=datafile, status=DataFileSummary.Status.PENDING + ) + meta_model = ReparseMeta.objects.create(db_backup_location="s3://backup") + file_meta = ReparseFileMeta.objects.create( + data_file=datafile, reparse_meta=meta_model + ) + setup_parse_mocks(monkeypatch, dfs=dfs) + dummy_parser = DummyParser(exc=DecoderUnknownException("decode")) + + monkeypatch.setattr( + parser_task.ParserFactory, "get_instance", lambda **kwargs: dummy_parser + ) + + parser_task.parse(datafile.id, reparse_id=meta_model.pk) + + file_meta.refresh_from_db() + dfs = DataFileSummary.objects.get(datafile=datafile) + assert dfs.status == DataFileSummary.Status.REJECTED + assert file_meta.finished is True + assert file_meta.success is False + + +@pytest.mark.django_db +def test_parse_database_error_sets_reparse_failed(monkeypatch, stt): + """Mark reparse failed on database error.""" + datafile = DataFileFactory(stt=stt, version=7) + ensure_stt_filenames(datafile.stt) + dfs = DataFileSummary.objects.create( + datafile=datafile, status=DataFileSummary.Status.PENDING + ) + meta_model = ReparseMeta.objects.create(db_backup_location="s3://backup") + file_meta = ReparseFileMeta.objects.create( + data_file=datafile, reparse_meta=meta_model + ) + setup_parse_mocks(monkeypatch, dfs=dfs) + dummy_parser = DummyParser(exc=DatabaseError("db")) + + monkeypatch.setattr( + parser_task.ParserFactory, "get_instance", lambda **kwargs: dummy_parser + ) + monkeypatch.setattr(parser_task, "log_parser_exception", lambda *a, **k: None) + + parser_task.parse(datafile.id, reparse_id=meta_model.pk) + + file_meta.refresh_from_db() + assert file_meta.finished is True + assert file_meta.success is False + + +@pytest.mark.django_db +def test_parse_generic_exception_rejects_and_logs(monkeypatch, stt): + """Create error and reject on unexpected exceptions.""" + datafile = DataFileFactory(stt=stt, version=8) + ensure_stt_filenames(datafile.stt) + dfs = DataFileSummary.objects.create( + datafile=datafile, status=DataFileSummary.Status.PENDING + ) + meta_model = ReparseMeta.objects.create(db_backup_location="s3://backup") + file_meta = ReparseFileMeta.objects.create( + data_file=datafile, reparse_meta=meta_model + ) + setup_parse_mocks(monkeypatch, dfs=dfs) + dummy_parser = DummyParser(exc=RuntimeError("boom")) + + monkeypatch.setattr( + parser_task.ParserFactory, "get_instance", lambda **kwargs: dummy_parser + ) + monkeypatch.setattr(parser_task, "log_parser_exception", lambda *a, **k: None) + + saved = {"called": False} + + def fake_get_generator(self, generator_type, row_number): + def generate(generator_args): + class DummyError: + def save(self_inner): + saved["called"] = True + + return DummyError() + + return generate + + monkeypatch.setattr( + parser_task.ErrorGeneratorFactory, "get_generator", fake_get_generator + ) + + parser_task.parse(datafile.id, reparse_id=meta_model.pk) + + dfs = DataFileSummary.objects.get(datafile=datafile) + file_meta.refresh_from_db() + assert dfs.status == DataFileSummary.Status.REJECTED + assert saved["called"] is True + assert file_meta.finished is True + assert file_meta.success is False diff --git a/tdrs-backend/tdpservice/search_indexes/test/test_reparse_command.py b/tdrs-backend/tdpservice/search_indexes/test/test_reparse_command.py new file mode 100644 index 000000000..6790542aa --- /dev/null +++ b/tdrs-backend/tdpservice/search_indexes/test/test_reparse_command.py @@ -0,0 +1,148 @@ +"""Tests for search index reparse command helpers.""" + +import datetime + +import pytest +from django.db.utils import DatabaseError + +from tdpservice.data_files.test.factories import DataFileFactory +from tdpservice.search_indexes.models.reparse_meta import ReparseMeta +from tdpservice.search_indexes.reparse import clean_reparse, handle_datafiles + + +@pytest.fixture +def log_context(): + """Return a stubbed log context.""" + return {"user_id": 1, "action_flag": 1, "object_repr": "Test"} + + +@pytest.mark.django_db +def test_handle_datafiles_adds_reparse_and_queues(monkeypatch, stt, log_context): + """Ensure datafiles are associated and queued for parsing.""" + meta_model = ReparseMeta.objects.create(db_backup_location="s3://backup") + file_one = DataFileFactory(stt=stt, version=1) + file_two = DataFileFactory(stt=stt, version=2) + + calls = [] + + def fake_delay(file_id, reparse_id): + calls.append((file_id, reparse_id)) + + monkeypatch.setattr( + "tdpservice.search_indexes.reparse.parser_task.parse.delay", fake_delay + ) + + handle_datafiles([file_one, file_two], meta_model, log_context) + + assert file_one.reparses.filter(pk=meta_model.pk).exists() + assert file_two.reparses.filter(pk=meta_model.pk).exists() + assert calls == [ + (file_one.pk, meta_model.pk), + (file_two.pk, meta_model.pk), + ] + + +@pytest.mark.django_db +def test_handle_datafiles_database_error(monkeypatch, stt, log_context): + """Raise DatabaseError when reparse association fails.""" + meta_model = ReparseMeta.objects.create(db_backup_location="s3://backup") + datafile = DataFileFactory(stt=stt, version=1) + + def raise_db_error(*args, **kwargs): + raise DatabaseError("boom") + + monkeypatch.setattr(datafile, "save", raise_db_error) + monkeypatch.setattr("tdpservice.search_indexes.reparse.log", lambda *a, **k: None) + + with pytest.raises(DatabaseError): + handle_datafiles([datafile], meta_model, log_context) + + +@pytest.mark.django_db +def test_handle_datafiles_generic_error(monkeypatch, stt, log_context): + """Raise generic exception when queueing fails.""" + meta_model = ReparseMeta.objects.create(db_backup_location="s3://backup") + datafile = DataFileFactory(stt=stt, version=1) + + def raise_generic(*args, **kwargs): + raise RuntimeError("boom") + + monkeypatch.setattr( + "tdpservice.search_indexes.reparse.parser_task.parse.delay", raise_generic + ) + monkeypatch.setattr("tdpservice.search_indexes.reparse.log", lambda *a, **k: None) + + with pytest.raises(RuntimeError): + handle_datafiles([datafile], meta_model, log_context) + + +@pytest.mark.django_db +def test_clean_reparse_single_file_updates_meta(monkeypatch, stt): + """Ensure clean_reparse populates metadata and queues datafiles.""" + datafile = DataFileFactory(stt=stt, version=1, quarter="Q2", year=2023) + + calls = {} + + def fake_handle(files, meta, context): + calls["files"] = list(files) + calls["meta"] = meta + calls["context"] = context + + monkeypatch.setattr( + "tdpservice.search_indexes.reparse.handle_datafiles", fake_handle + ) + monkeypatch.setattr("tdpservice.search_indexes.reparse.log", lambda *a, **k: None) + monkeypatch.setattr( + "tdpservice.search_indexes.reparse.get_number_of_records", lambda files: 10 + ) + monkeypatch.setattr( + "tdpservice.search_indexes.reparse.calculate_timeout", + lambda count, total: datetime.timedelta(minutes=5), + ) + monkeypatch.setattr("tdpservice.search_indexes.reparse.backup", lambda *a, **k: None) + monkeypatch.setattr( + "tdpservice.search_indexes.reparse.count_total_num_records", + lambda *a, **k: 123, + ) + monkeypatch.setattr( + "tdpservice.search_indexes.reparse.delete_associated_models", + lambda *a, **k: None, + ) + monkeypatch.setattr( + "tdpservice.search_indexes.reparse.get_log_context", + lambda user: {"user_id": user.id}, + ) + monkeypatch.setattr( + "tdpservice.search_indexes.reparse.assert_sequential_execution", + lambda *a, **k: True, + ) + + clean_reparse([str(datafile.id)]) + + meta = ReparseMeta.objects.latest("pk") + assert meta.fiscal_year == 2023 + assert meta.fiscal_quarter == "Q2" + assert meta.total_num_records_initial == 123 + assert meta.timeout_at == meta.created_at + datetime.timedelta(minutes=5) + assert f"_rpv{meta.pk}_" in meta.db_backup_location + assert meta.db_backup_location.startswith("/tmp/reparsing_backup") + + assert calls["meta"].pk == meta.pk + assert calls["files"][0].pk == datafile.pk + + +@pytest.mark.django_db +def test_clean_reparse_requires_sequential_execution(monkeypatch, stt): + """Raise when reparse is not sequentially safe.""" + datafile = DataFileFactory(stt=stt, version=1) + + monkeypatch.setattr("tdpservice.search_indexes.reparse.log", lambda *a, **k: None) + monkeypatch.setattr( + "tdpservice.search_indexes.reparse.assert_sequential_execution", + lambda *a, **k: False, + ) + + with pytest.raises(Exception, match="Sequential execution required"): + clean_reparse([str(datafile.id)]) + + assert ReparseMeta.objects.count() == 0 diff --git a/tdrs-backend/tdpservice/settings/cloudgov.py b/tdrs-backend/tdpservice/settings/cloudgov.py index 52f8f2897..7eb71e087 100644 --- a/tdrs-backend/tdpservice/settings/cloudgov.py +++ b/tdrs-backend/tdpservice/settings/cloudgov.py @@ -5,6 +5,7 @@ import os from distutils.util import strtobool +from tdpservice.common.util import get_cloudgov_broker_db_numbers from tdpservice.settings.common import Common logger = logging.getLogger(__name__) @@ -27,28 +28,6 @@ def get_cloudgov_service_creds_by_instance_name(services, instance_name): ) -def get_cloudgov_broker_db_numbers(cloudgov_name): - """ - Get the appropriate redis broker db numbers for an environment. - - Returns a tuple of (broker_db_number, results_db_number) - """ - match cloudgov_name: - case "raft": - return ("0", "1") - case "qasp": - return ("2", "3") - case "a11y": - return ("4", "5") - case "develop": - return ("0", "1") - case "staging": - return ("2", "3") - case "prod": - return ("0", "1") - return ("0", "1") - - class CloudGov(Common): """Base settings class for applications deployed in Cloud.gov.""" @@ -166,12 +145,22 @@ class CloudGov(Common): redis_settings = cloudgov_services["aws-elasticache-redis"][0]["credentials"] REDIS_URI = f"rediss://:{redis_settings['password']}@{redis_settings['host']}:{redis_settings['port']}" - (broker_db_number, results_db_number) = get_cloudgov_broker_db_numbers( - cloudgov_name - ) + brokers = get_cloudgov_broker_db_numbers(cloudgov_name) + + CELERY_BROKER_URL = REDIS_URI + "/" + brokers["celery"] + CELERY_RESULT_BACKEND = REDIS_URI + "/" + brokers["celery"] + CACHES = { + "default": { + "BACKEND": "django.core.cache.backends.locmem.LocMemCache", + } + } - CELERY_BROKER_URL = REDIS_URI + "/" + broker_db_number - CELERY_RESULT_BACKEND = REDIS_URI + "/" + broker_db_number + for c, n in brokers["caches"].items(): + CACHES[c] = { + "BACKEND": "django.core.cache.backends.redis.RedisCache", + "LOCATION": f"{REDIS_URI}/{n}", + "KEY_PREFIX": f"{cloudgov_name}-{c}", # does include "prod" for prod, can specify per env in classes below + } OTEL_EXPORTER_OTLP_ENDPOINT = os.getenv( "OTEL_EXPORTER_OTLP_ENDPOINT", "http://tempo.apps.internal:4317" diff --git a/tdrs-backend/tdpservice/settings/common.py b/tdrs-backend/tdpservice/settings/common.py index 4695d567d..62afe7837 100644 --- a/tdrs-backend/tdpservice/settings/common.py +++ b/tdrs-backend/tdpservice/settings/common.py @@ -191,6 +191,8 @@ class Common(Configuration): "PORT": os.getenv("DB_PORT"), } } + # Allow DB connections to persist for 10 min + CONN_MAX_AGE = 600 # General APPEND_SLASH = True @@ -518,8 +520,8 @@ class Common(Configuration): REDIS_URI = os.getenv("REDIS_URI", "redis://redis-server:6379") logger.debug("REDIS_URI: " + REDIS_URI) - CELERY_BROKER_URL = REDIS_URI - CELERY_RESULT_BACKEND = REDIS_URI + CELERY_BROKER_URL = REDIS_URI + "/0" + CELERY_RESULT_BACKEND = REDIS_URI + "/0" CELERY_ACCEPT_CONTENT = ["application/json"] CELERY_TASK_SERIALIZER = "json" CELERY_RESULT_SERIALIZER = "json" @@ -622,6 +624,21 @@ class Common(Configuration): }, } + DEFAULT_CACHE_TIMEOUT = 300 + CACHES = { + "default": { + "BACKEND": "django.core.cache.backends.locmem.LocMemCache", + }, + "stts": { + "BACKEND": "django.core.cache.backends.redis.RedisCache", + "LOCATION": f"{REDIS_URI}/1", + }, + "feature-flags": { + "BACKEND": "django.core.cache.backends.redis.RedisCache", + "LOCATION": f"{REDIS_URI}/2", + }, + } + CYPRESS_TOKEN = os.getenv("CYPRESS_TOKEN", None) FIXTURE_DIRS = [os.path.join(BASE_DIR, "fixtures")] diff --git a/tdrs-backend/tdpservice/settings/local.py b/tdrs-backend/tdpservice/settings/local.py index f8312f08f..01a8700f7 100644 --- a/tdrs-backend/tdpservice/settings/local.py +++ b/tdrs-backend/tdpservice/settings/local.py @@ -1,4 +1,5 @@ """Define configuration settings for local environment.""" + import os from distutils.util import strtobool @@ -55,3 +56,6 @@ class Local(Common): ) SENTRY_DSN = None + + Common.CACHES["stts"]["KEY_PREFIX"] = "local" + Common.CACHES["feature-flags"]["KEY_PREFIX"] = "local" diff --git a/tdrs-backend/tdpservice/stts/test/test_api.py b/tdrs-backend/tdpservice/stts/test/test_api.py index 87f17ae91..3bdc9262c 100644 --- a/tdrs-backend/tdpservice/stts/test/test_api.py +++ b/tdrs-backend/tdpservice/stts/test/test_api.py @@ -1,11 +1,19 @@ """API STT Tests.""" + +from unittest.mock import MagicMock, patch + from django.contrib.auth import get_user_model +from django.core.cache import caches +from django.test import TestCase, override_settings from django.urls import reverse import pytest from rest_framework import status +from rest_framework.test import APIClient +from tdpservice.conftest import UserFactory from tdpservice.stts.models import STT, Region +from tdpservice.stts.views import STTApiAlphaView User = get_user_model() @@ -67,28 +75,6 @@ def test_can_get_stts(api_client, stt_user, stts): assert STT.objects.filter(name=state_name).exists() -@pytest.mark.django_db -def test_can_get_alpha_stts(api_client, stt_user, stts): - """Test endpoint returns the alphabetized listing of STTs.""" - api_client.login(username=stt_user.username, password="test_password") - response = api_client.get(reverse("stts-alpha")) - assert response.status_code == status.HTTP_200_OK - assert len(response.data) == STT.objects.count() - - state_name = response.data[0]["name"] - assert STT.objects.filter(name=state_name).exists() - - -@pytest.mark.django_db -def test_alpha_stts_is_sorted(api_client, stt_user, stts): - """Test alphabetized endpoint is alphabetized.""" - api_client.login(username=stt_user.username, password="test_password") - response = api_client.get(reverse("stts-alpha")) - response_names = [datum["name"] for datum in response.data] - database_names = STT.objects.values_list("name", flat=True).order_by("name") - assert response_names == list(database_names) - - @pytest.mark.django_db def test_can_get_by_region_stts(api_client, stt_user, stts): """Test endpoint returns the alphabetized listing of STTs.""" @@ -115,9 +101,111 @@ def test_can_get_by_region_stts(api_client, stt_user, stts): @pytest.mark.django_db +@override_settings( + CACHES={ + "stts": { + "BACKEND": "django.core.cache.backends.locmem.LocMemCache", + "LOCATION": "unique-test-cache-location", # Unique location to avoid conflicts + "KEY_PREFIX": "test", + }, + } +) def test_stts_and_stts_alpha_are_dissimilar(api_client, stt_user, stts): """The default STTs endpoint is not sorted the same as the alpha.""" api_client.login(username=stt_user.username, password="test_password") alpha_response = api_client.get(reverse("stts-alpha")) default_response = api_client.get(reverse("stts")) assert not alpha_response.data == default_response.data + + +@pytest.mark.django_db +@override_settings( + CACHES={ + "stts": { + "BACKEND": "django.core.cache.backends.locmem.LocMemCache", + "LOCATION": "unique-test-cache-location", # Unique location to avoid conflicts + "KEY_PREFIX": "test", + }, + } +) +def test_can_get_alpha_stts(api_client, stt_user, stts): + """Test endpoint returns the alphabetized listing of STTs.""" + api_client.login(username=stt_user.username, password="test_password") + response = api_client.get(reverse("stts-alpha")) + assert response.status_code == status.HTTP_200_OK + assert len(response.data) == STT.objects.count() + + state_name = response.data[0]["name"] + assert STT.objects.filter(name=state_name).exists() + + +@pytest.mark.django_db +@override_settings( + CACHES={ + "stts": { + "BACKEND": "django.core.cache.backends.locmem.LocMemCache", + "LOCATION": "unique-test-cache-location", # Unique location to avoid conflicts + "KEY_PREFIX": "test", + }, + } +) +def test_alpha_stts_is_sorted(api_client, stt_user, stts): + """Test alphabetized endpoint is alphabetized.""" + api_client.login(username=stt_user.username, password="test_password") + response = api_client.get(reverse("stts-alpha")) + response_names = [datum["name"] for datum in response.data] + database_names = STT.objects.values_list("name", flat=True).order_by("name") + assert response_names == list(database_names) + + +@override_settings( + CACHES={ + "stts": { + "BACKEND": "django.core.cache.backends.locmem.LocMemCache", + "LOCATION": "unique-test-cache-location", # Unique location to avoid conflicts + "KEY_PREFIX": "test", + }, + } +) +class TestSTTApiAlphaViewCache(TestCase): + """Tests for the STTApiAlphaView class.""" + + api_client = APIClient() + + def setUp(self): + """Run before all tests in TestCase.""" + super().setUp() + cache = caches["stts"] + cache.clear() + + user = UserFactory.create() + self.api_client.login(username=user.username, password="test_password") + + def test_existing_cache_avoids_lookup(self): + """Test that no lookup is performed if flags exist in the cache.""" + mock_queryset = MagicMock() + with patch.object( + STTApiAlphaView, "get_queryset", return_value=mock_queryset + ) as mock_method: + # request and check the cache was cold + response = self.api_client.get(reverse("stts-alpha")) + assert response.status_code == status.HTTP_200_OK + assert mock_method.called + + mock_method.reset_mock() + + # the cache should be warm now, request again + response = self.api_client.get(reverse("stts-alpha")) + assert response.status_code == status.HTTP_200_OK + assert not mock_method.called + + def test_no_cache_forces_lookup(self): + """Test that a lookup is performed if there are no flags in the cache.""" + mock_queryset = MagicMock() + with patch.object( + STTApiAlphaView, "get_queryset", return_value=mock_queryset + ) as mock_method: + # request and check the cache was cold + response = self.api_client.get(reverse("stts-alpha")) + assert response.status_code == status.HTTP_200_OK + assert mock_method.called diff --git a/tdrs-backend/tdpservice/stts/views.py b/tdrs-backend/tdpservice/stts/views.py index c58efc5d0..dc9decb4d 100644 --- a/tdrs-backend/tdpservice/stts/views.py +++ b/tdrs-backend/tdpservice/stts/views.py @@ -1,7 +1,11 @@ """Define API views for user class.""" + import logging +from django.conf import settings from django.db.models import Prefetch +from django.utils.decorators import method_decorator +from django.views.decorators.cache import cache_page from rest_framework import generics from rest_framework.permissions import IsAuthenticated @@ -32,6 +36,13 @@ class STTApiAlphaView(generics.ListAPIView): queryset = STT.objects.order_by("name") serializer_class = STTSerializer + @method_decorator( + cache_page(settings.DEFAULT_CACHE_TIMEOUT, cache="stts", key_prefix="alpha") + ) + def list(self, request): + """Get the stt list from the cache if available, else fetch the queryset.""" + return super().list(request) + class STTApiView(generics.ListAPIView): """Simple view to get all STTs.""" diff --git a/tdrs-backend/tdpservice/urls.py b/tdrs-backend/tdpservice/urls.py index fe46f8c08..4597a0938 100755 --- a/tdrs-backend/tdpservice/urls.py +++ b/tdrs-backend/tdpservice/urls.py @@ -11,8 +11,9 @@ from drf_yasg import openapi from drf_yasg.views import get_schema_view from rest_framework.permissions import AllowAny +from rest_framework.routers import DefaultRouter -from .core.views import write_logs +from .core.views import FeatureFlagViewset, write_logs from .users.api.authorization_check import AuthorizationCheck, PlgAuthorizationCheck from .users.api.login import ( CypressLoginDotGovAuthenticationOverride, @@ -23,6 +24,8 @@ from .users.api.logout import LogoutUser from .users.api.logout_redirect_oidc import LogoutRedirectOIDC +router = DefaultRouter() + admin.autodiscover() admin.site.login = login_required(admin.site.login) admin.site.site_header = "Django administration" @@ -46,6 +49,10 @@ path("security/", include("tdpservice.security.urls")), ] +router.register("feature-flags", FeatureFlagViewset, basename="feature-flag") + +urlpatterns += router.urls + if settings.DEBUG: urlpatterns.append( path( diff --git a/tdrs-backend/tdpservice/users/test/test_api/test_views.py b/tdrs-backend/tdpservice/users/test/test_api/test_views.py new file mode 100644 index 000000000..3ce3b7cc8 --- /dev/null +++ b/tdrs-backend/tdpservice/users/test/test_api/test_views.py @@ -0,0 +1,330 @@ +"""Additional viewset tests for users app coverage.""" + +import pytest +from django.contrib.auth.models import Permission +from django.contrib.contenttypes.models import ContentType +from rest_framework import status +from rest_framework.exceptions import MethodNotAllowed +from rest_framework.test import APIRequestFactory + +from tdpservice.users.models import ( + AccountApprovalStatusChoices, + ChangeRequestAuditLog, + Feedback, + User, + UserChangeRequest, +) +from tdpservice.users.test.factories import FeedbackFactory +from tdpservice.users.views import ( + ChangeRequestAuditLogViewSet, + CypressAdminUserViewSet, + FeedbackViewSet, + UserChangeRequestViewSet, +) + + +@pytest.fixture +def feedback_payload(): + """Return baseline feedback payload.""" + return { + "rating": 3, + "feedback": "Test feedback", + "anonymous": False, + "page_url": "https://localhost/", + "feedback_type": "general-feedback", + "component": "general-feedback", + "widget_id": "general-feedback", + "attachments": [], + } + + +@pytest.mark.django_db +def test_request_access_get_returns_405(api_client, data_analyst): + """Reject GET request for request_access action.""" + api_client.login(username=data_analyst.username, password="test_password") + + response = api_client.get("/v1/users/request_access/") + + assert response.status_code == status.HTTP_405_METHOD_NOT_ALLOWED + + +@pytest.mark.django_db +def test_request_access_sets_stt_and_permission(api_client, data_analyst): + """Persist access request updates and FRA permission.""" + content_type = ContentType.objects.get_for_model(User) + Permission.objects.get_or_create( + codename="has_fra_access", + content_type=content_type, + defaults={"name": "Has FRA Access"}, + ) + + api_client.login(username=data_analyst.username, password="test_password") + stt_id = data_analyst.stt_id + + payload = { + "first_name": "Test", + "last_name": "User", + "stt": data_analyst.stt.id, + "has_fra_access": True, + } + + response = api_client.patch("/v1/users/request_access/", payload, format="json") + + assert response.status_code == status.HTTP_200_OK + + data_analyst.refresh_from_db() + assert ( + data_analyst.account_approval_status + == AccountApprovalStatusChoices.ACCESS_REQUEST + ) + assert data_analyst.access_requested_date is not None + assert data_analyst.stt_id == stt_id + assert data_analyst.user_permissions.filter(codename="has_fra_access").exists() + + +@pytest.mark.django_db +def test_request_access_missing_permission_returns_400( + api_client, data_analyst, monkeypatch +): + """Return 400 when FRA permission is missing.""" + api_client.login(username=data_analyst.username, password="test_password") + + def raise_does_not_exist(*args, **kwargs): + raise Permission.DoesNotExist() + + monkeypatch.setattr(Permission.objects, "get", raise_does_not_exist) + + payload = { + "first_name": "Test", + "last_name": "User", + "stt": data_analyst.stt.id, + "has_fra_access": True, + } + response = api_client.patch("/v1/users/request_access/", payload, format="json") + + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert "has_fra_access permission does not exist." in str(response.data) + + +@pytest.mark.django_db +def test_update_profile_direct_update_admin(api_client, ofa_system_admin): + """Allow direct profile update for system admins.""" + api_client.login(username=ofa_system_admin.username, password="test_password") + + payload = { + "first_name": "Direct", + "last_name": "Update", + "create_change_requests": False, + } + + response = api_client.patch("/v1/users/update_profile/", payload, format="json") + + assert response.status_code == status.HTTP_200_OK + assert response.data["first_name"] == "Direct" + assert response.data["last_name"] == "Update" + + +@pytest.mark.django_db +def test_update_profile_direct_update_non_admin_denied(api_client, data_analyst): + """Reject direct profile updates from non-admin users.""" + api_client.login(username=data_analyst.username, password="test_password") + + payload = { + "first_name": "Direct", + "last_name": "Update", + "stt": data_analyst.stt.id, + "create_change_requests": False, + } + + response = api_client.patch("/v1/users/update_profile/", payload, format="json") + + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert "Only administrators can update user profiles directly." in str(response.data) + + +@pytest.mark.django_db +def test_user_change_request_queryset_filters_by_user( + data_analyst, user, ofa_system_admin +): + """Return only owner change requests for non-admin users.""" + user_request = UserChangeRequest.objects.create( + user=user, + requested_by=user, + field_name="first_name", + current_value="A", + requested_value="B", + ) + analyst_request = UserChangeRequest.objects.create( + user=data_analyst, + requested_by=data_analyst, + field_name="last_name", + current_value="C", + requested_value="D", + ) + + factory = APIRequestFactory() + view = UserChangeRequestViewSet() + + request = factory.get("/v1/change-requests/") + request.user = data_analyst + view.request = request + assert set(view.get_queryset()) == {analyst_request} + + request.user = ofa_system_admin + view.request = request + assert set(view.get_queryset()) == {user_request, analyst_request} + + +@pytest.mark.django_db +def test_change_request_audit_log_queryset_filters_by_admin( + data_analyst, ofa_system_admin +): + """Restrict audit logs to OFA system admins.""" + change_request = UserChangeRequest.objects.create( + user=data_analyst, + requested_by=data_analyst, + field_name="first_name", + current_value="A", + requested_value="B", + ) + log = ChangeRequestAuditLog.objects.create( + change_request=change_request, + action="created", + performed_by=data_analyst, + details={"field": "first_name"}, + ) + + factory = APIRequestFactory() + view = ChangeRequestAuditLogViewSet() + + request = factory.get("/v1/change-request-logs/") + request.user = data_analyst + view.request = request + assert list(view.get_queryset()) == [] + + request.user = ofa_system_admin + view.request = request + assert set(view.get_queryset()) == {log} + + +@pytest.mark.django_db +def test_feedback_create_anonymous_sets_anonymous(api_client, feedback_payload): + """Force anonymous feedback when user is not authenticated.""" + response = api_client.post("/v1/feedback/", feedback_payload, format="json") + + assert response.status_code == status.HTTP_201_CREATED + + feedback = Feedback.objects.get(id=response.data["id"]) + assert feedback.anonymous is True + assert feedback.user is None + + +@pytest.mark.django_db +def test_feedback_create_authenticated_sets_user( + api_client, data_analyst, feedback_payload +): + """Attach user to feedback when not anonymous.""" + api_client.login(username=data_analyst.username, password="test_password") + + response = api_client.post("/v1/feedback/", feedback_payload, format="json") + + assert response.status_code == status.HTTP_201_CREATED + + feedback = Feedback.objects.get(id=response.data["id"]) + assert feedback.anonymous is False + assert str(feedback.user_id) == str(data_analyst.id) + + +@pytest.mark.django_db +def test_feedback_create_invalid_returns_400(api_client, feedback_payload): + """Return validation errors for invalid feedback payloads.""" + feedback_payload.pop("rating") + + response = api_client.post("/v1/feedback/", feedback_payload, format="json") + + assert response.status_code == status.HTTP_400_BAD_REQUEST + + +@pytest.mark.django_db +def test_feedback_update_sets_user_to_none_when_anonymous( + api_client, data_analyst +): + """Clear feedback user when marked anonymous.""" + feedback = FeedbackFactory.create(user=data_analyst, anonymous=False) + api_client.login(username=data_analyst.username, password="test_password") + + response = api_client.patch( + f"/v1/feedback/{feedback.id}/", {"anonymous": True}, format="json" + ) + + assert response.status_code == status.HTTP_200_OK + + feedback.refresh_from_db() + assert feedback.anonymous is True + assert feedback.user is None + + +@pytest.mark.django_db +def test_feedback_update_sets_user_when_not_anonymous( + api_client, data_analyst +): + """Set feedback user when marked not anonymous.""" + feedback = FeedbackFactory.create(user=None, anonymous=True) + api_client.login(username=data_analyst.username, password="test_password") + + response = api_client.patch( + f"/v1/feedback/{feedback.id}/", {"anonymous": False}, format="json" + ) + + assert response.status_code == status.HTTP_200_OK + + feedback.refresh_from_db() + assert feedback.anonymous is False + assert str(feedback.user_id) == str(data_analyst.id) + + +@pytest.mark.django_db +def test_feedback_update_invalid_returns_400(api_client, data_analyst): + """Return validation errors for invalid feedback updates.""" + feedback = FeedbackFactory.create(user=data_analyst, anonymous=False) + api_client.login(username=data_analyst.username, password="test_password") + + response = api_client.patch( + f"/v1/feedback/{feedback.id}/", {"rating": "bad"}, format="json" + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + + +@pytest.mark.django_db +def test_feedback_destroy_is_not_allowed(data_analyst): + """Disallow feedback deletion.""" + feedback = FeedbackFactory.create(user=data_analyst) + view = FeedbackViewSet() + request = APIRequestFactory().delete(f"/v1/feedback/{feedback.id}/") + request.user = data_analyst + + response = view.destroy(request, pk=feedback.id) + + assert isinstance(response, MethodNotAllowed) + + +@pytest.mark.django_db +def test_cypress_admin_user_viewset_set_status_updates(user): + """Update user approval status with Cypress viewset helpers.""" + view = CypressAdminUserViewSet() + + response = view.set_pending(None, user.id) + user.refresh_from_db() + assert response.status_code == status.HTTP_200_OK + assert user.account_approval_status == AccountApprovalStatusChoices.PENDING + + response = view.set_initial(None, user.id) + user.refresh_from_db() + assert response.status_code == status.HTTP_200_OK + assert user.account_approval_status == AccountApprovalStatusChoices.INITIAL + + response = view.set_approved(None, user.id) + user.refresh_from_db() + assert response.status_code == status.HTTP_200_OK + assert user.account_approval_status == AccountApprovalStatusChoices.APPROVED diff --git a/tdrs-backend/tdpservice/users/test/test_filters_forms.py b/tdrs-backend/tdpservice/users/test/test_filters_forms.py new file mode 100644 index 000000000..1738947f1 --- /dev/null +++ b/tdrs-backend/tdpservice/users/test/test_filters_forms.py @@ -0,0 +1,221 @@ +"""Tests for user admin filters and forms.""" + +from urllib.parse import urlencode + +import pytest +from django.contrib import admin +from django.contrib.admin.sites import AdminSite +from django.contrib.auth.models import Group +from django.core.exceptions import ValidationError +from django.test import RequestFactory + +from tdpservice.users.constants import REGIONAL_ROLES +from tdpservice.users.filters import ActiveStatusListFilter +from tdpservice.users.forms import UserForm +from tdpservice.users.models import AccountApprovalStatusChoices, User +from tdpservice.users.test.factories import UserFactory + + +class DummyChangeList: + """Simple changelist stub for filter choice rendering.""" + + def get_query_string(self, params): + """Return a query string with provided parameters.""" + return f"?{urlencode(params)}" + + +def build_filter(value=None): + """Create an ActiveStatusListFilter with optional value.""" + params = {} + if value is not None: + params["active_status"] = value + request = RequestFactory().get("/admin/users/user/", params) + model_admin = admin.ModelAdmin(User, AdminSite()) + return ActiveStatusListFilter(request, request.GET.copy(), User, model_admin) + + +def get_group(name): + """Return a group with the given name, creating it when needed.""" + group, _ = Group.objects.get_or_create(name=name) + return group + + +@pytest.mark.django_db +def test_active_status_filter_lookups_and_default_value(): + """Ensure lookups are defined and value defaults to active users.""" + status_filter = build_filter() + + assert status_filter.lookups(None, None) == ( + ("active_users", "Show Active Users"), + ("all_users", "Show All Users"), + ("inactive_users", "Show Inactive Users"), + ) + assert status_filter.value() == "active_users" + + +@pytest.mark.django_db +def test_active_status_filter_queryset_variants(): + """Filter users based on activation status options.""" + inactive_user = UserFactory.create( + account_approval_status=AccountApprovalStatusChoices.DEACTIVATED + ) + active_user = UserFactory.create( + account_approval_status=AccountApprovalStatusChoices.APPROVED + ) + other_user = UserFactory.create( + account_approval_status=AccountApprovalStatusChoices.INITIAL + ) + scoped_ids = [inactive_user.id, active_user.id, other_user.id] + queryset = User.objects.filter(id__in=scoped_ids) + + def ids_for(qs): + return {str(pk) for pk in qs.values_list("id", flat=True)} + + inactive_filter = build_filter("inactive_users") + inactive_ids = ids_for(inactive_filter.queryset(None, queryset)) + assert inactive_ids == {str(inactive_user.id)} + + active_filter = build_filter("active_users") + active_ids = ids_for(active_filter.queryset(None, queryset)) + assert active_ids == {str(active_user.id), str(other_user.id)} + + all_filter = build_filter("all_users") + all_ids = ids_for(all_filter.queryset(None, queryset)) + assert all_ids == { + str(inactive_user.id), + str(active_user.id), + str(other_user.id), + } + + unknown_filter = build_filter("unknown") + unknown_ids = ids_for(unknown_filter.queryset(None, queryset)) + assert unknown_ids == { + str(inactive_user.id), + str(active_user.id), + str(other_user.id), + } + + +@pytest.mark.django_db +def test_active_status_filter_choices_excludes_all_option(): + """Render filter choices without an implicit All option.""" + status_filter = build_filter("active_users") + + choices = list(status_filter.choices(DummyChangeList())) + + assert [choice["display"] for choice in choices] == [ + "Show Active Users", + "Show All Users", + "Show Inactive Users", + ] + + +@pytest.mark.django_db +def test_user_form_clean_rejects_multiple_groups(): + """Disallow assigning more than one group in clean.""" + form = UserForm() + form.cleaned_data = { + "groups": [get_group("OFA Admin"), get_group("Data Analyst")], + "regions": [], + "stt": None, + } + + with pytest.raises(ValidationError) as excinfo: + form.clean() + + assert excinfo.value.messages == ["User should not have multiple groups."] + + +@pytest.mark.django_db +def test_user_form_clean_requires_location_for_regional_roles(): + """Regional role users must have a region or stt.""" + regional_group = get_group(next(iter(REGIONAL_ROLES))) + form = UserForm() + form.cleaned_data = {"groups": [regional_group], "regions": [], "stt": None} + + with pytest.raises(ValidationError) as excinfo: + form.clean() + + assert excinfo.value.messages == [ + "Users in regional roles must have at least one region or location assigned." + ] + + +@pytest.mark.django_db +def test_user_form_clean_rejects_region_and_stt(region, stt): + """Disallow assigning both region and stt.""" + regional_group = get_group(next(iter(REGIONAL_ROLES))) + form = UserForm() + form.cleaned_data = {"groups": [regional_group], "regions": [region], "stt": stt} + + with pytest.raises(ValidationError) as excinfo: + form.clean() + + assert excinfo.value.messages == [ + "A user may only have a Region or STT assigned, not both." + ] + + +@pytest.mark.django_db +def test_user_form_clean_rejects_regions_for_non_regional_roles(region): + """Non-regional users should not have regions.""" + form = UserForm() + form.cleaned_data = { + "groups": [get_group("OFA Admin")], + "regions": [region], + "stt": None, + } + + with pytest.raises(ValidationError) as excinfo: + form.clean() + + assert excinfo.value.messages == [ + "Users without regional roles should not be assigned regions." + ] + + +@pytest.mark.django_db +def test_user_form_clean_accepts_valid_assignments(region, stt): + """Allow valid regional or non-regional assignments.""" + regional_group = get_group(next(iter(REGIONAL_ROLES))) + non_regional_group = get_group("OFA Admin") + + regional_form = UserForm() + regional_form.cleaned_data = { + "groups": [regional_group], + "regions": [region], + "stt": None, + } + assert regional_form.clean() == regional_form.cleaned_data + + non_regional_form = UserForm() + non_regional_form.cleaned_data = { + "groups": [non_regional_group], + "regions": [], + "stt": stt, + } + assert non_regional_form.clean() == non_regional_form.cleaned_data + + +@pytest.mark.django_db +def test_user_form_clean_groups(): + """Validate group selection in clean_groups.""" + form = UserForm() + form.cleaned_data = {"groups": [get_group("OFA Admin")]} + assert form.clean_groups() == form.cleaned_data["groups"] + + form.cleaned_data = {"groups": [get_group("OFA Admin"), get_group("Developer")]} + with pytest.raises(ValidationError) as excinfo: + form.clean_groups() + assert excinfo.value.messages == ["User should not have multiple groups"] + + +@pytest.mark.django_db +def test_user_form_clean_feature_flags_defaults_to_dict(): + """Normalize feature flags to an empty dict when missing.""" + form = UserForm() + form.cleaned_data = {} + assert form.clean_feature_flags() == {} + + form.cleaned_data = {"feature_flags": {"flag": True}} + assert form.clean_feature_flags() == {"flag": True} diff --git a/tdrs-frontend/cypress/e2e/common-steps/data_files.js b/tdrs-frontend/cypress/e2e/common-steps/data_files.js index 9bc7094b8..7c6cb2fac 100644 --- a/tdrs-frontend/cypress/e2e/common-steps/data_files.js +++ b/tdrs-frontend/cypress/e2e/common-steps/data_files.js @@ -24,9 +24,11 @@ export const uploadFile = (file_input, file_path, willError = false) => { 'exist' ) cy.get('.usa-alert__text').should('not.exist') - cy.get('button') - .contains('Submit') - .should('not.be.disabled', { timeout: 5000 }) + cy.contains('button', 'Submit', { timeout: 5000 }).should( + 'have.attr', + 'data-has-uploaded-files', + 'true' + ) } } @@ -201,10 +203,12 @@ export const uploadSectionFile = ( 'is-loading' ) cy.get('.usa-alert__text').should('not.exist') - cy.get('button') - .contains('Submit') - .should('not.be.disabled', { timeout: 5000 }) - cy.contains('button', 'Submit').should('be.enabled').click() + cy.contains('button', 'Submit', { timeout: 5000 }).should( + 'have.attr', + 'data-has-uploaded-files', + 'true' + ) + cy.contains('button', 'Submit').click() cy.wait('@dataFileSubmit', { timeout: 60000 }).then(({ response }) => { const id = response?.body?.id if (!id) throw new Error('Missing data_file id in response') diff --git a/tdrs-frontend/cypress/e2e/profile/profile-helpers.js b/tdrs-frontend/cypress/e2e/profile/profile-helpers.js index 01fd2920c..60968e3a1 100644 --- a/tdrs-frontend/cypress/e2e/profile/profile-helpers.js +++ b/tdrs-frontend/cypress/e2e/profile/profile-helpers.js @@ -12,13 +12,17 @@ export const navigateToProfile = () => { } export const clickEditProfile = () => { - cy.get('button').contains('Edit Profile').should('be.visible').click() + cy.get('button').contains('Edit Profile').as('editProfile') + cy.get('@editProfile').should('be.visible') + cy.get('@editProfile').click() // Wait for the form to appear cy.get('#firstName').should('be.visible') } export const clickEditAccessRequest = () => { - cy.get('button').contains('Edit Access Request').should('be.visible').click() + cy.get('button').contains('Edit Access Request').as('editAccess') + cy.get('@editAccess').should('be.visible') + cy.get('@editAccess').click() // Wait for the form to appear cy.get('#firstName').should('be.visible') } @@ -114,7 +118,9 @@ export const verifyProfileField = (fieldLabel, expectedValue) => { } } else { // Fallback: still assert text exists somewhere to surface a real failure - cy.wrap($body).contains(expectedValue, { timeout: 15000 }).should('exist') + cy.wrap($body) + .contains(expectedValue, { timeout: 15000 }) + .should('exist') } } }) diff --git a/tdrs-frontend/package.json b/tdrs-frontend/package.json index a49ae3884..c69385268 100644 --- a/tdrs-frontend/package.json +++ b/tdrs-frontend/package.json @@ -12,7 +12,6 @@ "@grafana/faro-web-tracing": "^1.18.1", "@lagunovsky/redux-react-router": "^4.3.2", "@uswds/uswds": "3.10.0", - "axios": "^1.7.7", "classnames": "^2.5.1", "detect-file-encoding-and-language": "^2.4.0", "@faker-js/faker": "^9.2.0", @@ -97,7 +96,6 @@ "collectCoverageFrom": [ "src/**/*.{js,jsx}", "!src/**/index.js", - "!src/axios-instance.js", "!src/serviceWorker.js", "!src/configureStore.js", "!src/mirage.js", @@ -117,7 +115,6 @@ "node_modules/(?!(@grafana|web-vitals)/)" ], "moduleNameMapper": { - "^axios$": "axios/dist/node/axios.cjs", "@uswds/uswds/src/js/components": "@uswds/uswds/packages/uswds-core/src/js/index.js", "\\.(jpg|jpeg|png|gif|eot|otf|webp|svg|ttf|woff|woff2|mp4|webm|wav|mp3|m4a|aac|oga)$": "jest-transform-stub" } diff --git a/tdrs-frontend/src/__mocks__/@grafana/faro-react.js b/tdrs-frontend/src/__mocks__/@grafana/faro-react.js index 501442b3b..0c7d93d74 100644 --- a/tdrs-frontend/src/__mocks__/@grafana/faro-react.js +++ b/tdrs-frontend/src/__mocks__/@grafana/faro-react.js @@ -4,3 +4,9 @@ import { Routes } from 'react-router-dom' export const FaroRoutes = ({ children }) => { return {children} } + +export const faro = { + api: { + getTraceContext: jest.fn(() => null), + }, +} diff --git a/tdrs-frontend/src/__mocks__/fetch-instance.js b/tdrs-frontend/src/__mocks__/fetch-instance.js new file mode 100644 index 000000000..5715a5079 --- /dev/null +++ b/tdrs-frontend/src/__mocks__/fetch-instance.js @@ -0,0 +1,32 @@ +/** + * Mocks fetch-instance HTTP requests. + */ + +export const get = jest.fn(() => + Promise.resolve({ + data: {}, + error: null, + status: 200, + ok: true, + }) +) + +export const post = jest.fn(() => + Promise.resolve({ + data: {}, + error: null, + status: 200, + ok: true, + }) +) + +export const patch = jest.fn(() => + Promise.resolve({ + data: {}, + error: null, + status: 200, + ok: true, + }) +) + +export default { get, post, patch } diff --git a/tdrs-frontend/src/actions/auth.js b/tdrs-frontend/src/actions/auth.js index 6b0147c6b..7df341e49 100644 --- a/tdrs-frontend/src/actions/auth.js +++ b/tdrs-frontend/src/actions/auth.js @@ -1,6 +1,8 @@ -import axiosInstance from '../axios-instance' +import { get } from '../fetch-instance' import { logErrorToServer } from '../utils/eventLogger' +import { CLEAR_FEATURE_FLAGS, fetchFeatureFlags } from './featureFlags' + export const FETCH_AUTH = 'FETCH_AUTH' export const SET_AUTH = 'SET_AUTH' export const SET_AUTH_ERROR = 'SET_AUTH_ERROR' @@ -41,27 +43,27 @@ export const SET_MOCK_LOGIN_STATE = 'SET_MOCK_LOGIN_STATE' export const fetchAuth = () => async (dispatch) => { dispatch({ type: FETCH_AUTH }) - try { - const URL = `${process.env.REACT_APP_BACKEND_URL}/auth_check` - const { data } = await axiosInstance.get(URL, { - withCredentials: true, - }) - - if (data?.inactive) { - dispatch({ type: SET_DEACTIVATED }) - } else if (data?.user) { - const { user, csrf } = data - - // Work around for csrf cookie issue we encountered in production. - axiosInstance.defaults.headers.common['X-CSRFToken'] = csrf + const URL = `${process.env.REACT_APP_BACKEND_URL}/auth_check` + const { data, ok, error } = await get(URL) - dispatch({ type: SET_AUTH, payload: { user } }) - } else { - dispatch({ type: CLEAR_AUTH }) - } - } catch (error) { + if (!ok) { logErrorToServer(SET_AUTH_ERROR) dispatch({ type: SET_AUTH_ERROR, payload: { error } }) + dispatch({ type: CLEAR_FEATURE_FLAGS }) + return + } + + if (data?.inactive) { + dispatch({ type: SET_DEACTIVATED }) + dispatch({ type: CLEAR_FEATURE_FLAGS }) + } else if (data?.user) { + const { user } = data + + dispatch({ type: SET_AUTH, payload: { user } }) + dispatch(fetchFeatureFlags()) + } else { + dispatch({ type: CLEAR_AUTH }) + dispatch({ type: CLEAR_FEATURE_FLAGS }) } } diff --git a/tdrs-frontend/src/actions/auth.test.js b/tdrs-frontend/src/actions/auth.test.js index fca4f8638..495493648 100644 --- a/tdrs-frontend/src/actions/auth.test.js +++ b/tdrs-frontend/src/actions/auth.test.js @@ -1,4 +1,4 @@ -import axios from 'axios' +import { get } from '../fetch-instance' import { thunk } from 'redux-thunk' import configureStore from 'redux-mock-store' import { v4 as uuidv4 } from 'uuid' @@ -10,17 +10,23 @@ import { SET_AUTH_ERROR, SET_DEACTIVATED, } from './auth' +import { CLEAR_FEATURE_FLAGS, FETCH_FEATURE_FLAGS } from './featureFlags' + +jest.mock('../fetch-instance') describe('actions/auth.js', () => { const mockStore = configureStore([thunk]) it('fetches a user and sets user info, when the user is authenticated', async () => { const mockUser = { id: uuidv4(), email: 'hi@bye.com' } - axios.get.mockImplementationOnce(() => + get.mockImplementationOnce(() => Promise.resolve({ data: { user: mockUser, }, + ok: true, + status: 200, + error: null, }) ) const store = mockStore() @@ -31,14 +37,19 @@ describe('actions/auth.js', () => { expect(actions[0].type).toBe(FETCH_AUTH) expect(actions[1].type).toBe(SET_AUTH) expect(actions[1].payload.user).toStrictEqual(mockUser) + expect(actions[2].type).toBe(CLEAR_FEATURE_FLAGS) + expect(actions[3].type).toBe(FETCH_FEATURE_FLAGS) }) it('clears the auth state, if user is not authenticated', async () => { - axios.get.mockImplementationOnce(() => + get.mockImplementationOnce(() => Promise.resolve({ data: { authenticated: false, }, + ok: true, + status: 200, + error: null, }) ) const store = mockStore() @@ -48,11 +59,17 @@ describe('actions/auth.js', () => { const actions = store.getActions() expect(actions[0].type).toBe(FETCH_AUTH) expect(actions[1].type).toBe(CLEAR_AUTH) + expect(actions[2].type).toBe(CLEAR_FEATURE_FLAGS) }) it('dispatches an error to the store if the API errors', async () => { - axios.get.mockImplementationOnce(() => - Promise.reject(Error({ message: 'something went wrong' })) + get.mockImplementationOnce(() => + Promise.resolve({ + data: null, + ok: false, + status: 500, + error: new Error('something went wrong'), + }) ) const store = mockStore() @@ -61,15 +78,19 @@ describe('actions/auth.js', () => { const actions = store.getActions() expect(actions[0].type).toBe(FETCH_AUTH) expect(actions[1].type).toBe(SET_AUTH_ERROR) + expect(actions[2].type).toBe(CLEAR_FEATURE_FLAGS) }) it('clears the auth state and triggers dispatches if the API returns `inactive`', async () => { - axios.get.mockImplementationOnce(() => + get.mockImplementationOnce(() => Promise.resolve({ data: { authenticated: false, inactive: true, }, + ok: true, + status: 200, + error: null, }) ) const store = mockStore() @@ -79,5 +100,6 @@ describe('actions/auth.js', () => { const actions = store.getActions() expect(actions[0].type).toBe(FETCH_AUTH) expect(actions[1].type).toBe(SET_DEACTIVATED) + expect(actions[2].type).toBe(CLEAR_FEATURE_FLAGS) }) }) diff --git a/tdrs-frontend/src/actions/featureFlags.js b/tdrs-frontend/src/actions/featureFlags.js new file mode 100644 index 000000000..1d5acd731 --- /dev/null +++ b/tdrs-frontend/src/actions/featureFlags.js @@ -0,0 +1,28 @@ +import { get } from '../fetch-instance' + +export const FETCH_FEATURE_FLAGS = 'FETCH_FEATURE_FLAGS' +export const SET_FEATURE_FLAGS = 'SET_FEATURE_FLAGS' +export const SET_FEATURE_FLAGS_ERROR = 'SET_FEATURE_FLAGS_ERROR' +export const CLEAR_FEATURE_FLAGS = 'CLEAR_FEATURE_FLAGS' + +export const fetchFeatureFlags = () => async (dispatch) => { + dispatch({ type: CLEAR_FEATURE_FLAGS }) + dispatch({ type: FETCH_FEATURE_FLAGS }) + + const lastFetched = Date.now() + + try { + const URL = `${process.env.REACT_APP_BACKEND_URL}/feature-flags/` + const response = await get(URL) + + if (response.data) { + const flags = response.data + dispatch({ + type: SET_FEATURE_FLAGS, + payload: { flags, lastFetched }, + }) + } + } catch (error) { + dispatch({ type: SET_FEATURE_FLAGS_ERROR, payload: { error, lastFetched } }) + } +} diff --git a/tdrs-frontend/src/actions/featureFlags.test.js b/tdrs-frontend/src/actions/featureFlags.test.js new file mode 100644 index 000000000..f0bb2551a --- /dev/null +++ b/tdrs-frontend/src/actions/featureFlags.test.js @@ -0,0 +1,50 @@ +import { get } from '../fetch-instance' +import { thunk } from 'redux-thunk' +import configureStore from 'redux-mock-store' +import { + fetchFeatureFlags, + FETCH_FEATURE_FLAGS, + SET_FEATURE_FLAGS, + SET_FEATURE_FLAGS_ERROR, + CLEAR_FEATURE_FLAGS, +} from './featureFlags' + +jest.mock('../fetch-instance') + +describe('actions/featureFlags.js', () => { + const mockStore = configureStore([thunk]) + + it('fetches and sets feature flags from api', async () => { + const mockFlag = { name: 'test-flag', enabled: true, config: {} } + const testDate = Date.now() + get.mockImplementationOnce(() => + Promise.resolve({ + data: [mockFlag], + }) + ) + const store = mockStore() + + await store.dispatch(fetchFeatureFlags()) + + const actions = store.getActions() + expect(actions[0].type).toBe(CLEAR_FEATURE_FLAGS) + expect(actions[1].type).toBe(FETCH_FEATURE_FLAGS) + expect(actions[2].type).toBe(SET_FEATURE_FLAGS) + expect(actions[2].payload.flags).toStrictEqual([mockFlag]) + expect(actions[2].payload.lastFetched).toBeGreaterThanOrEqual(testDate) + }) + + it('dispatches an error to the store if the API errors', async () => { + get.mockImplementationOnce(() => + Promise.reject(Error({ message: 'something went wrong' })) + ) + const store = mockStore() + + await store.dispatch(fetchFeatureFlags()) + + const actions = store.getActions() + expect(actions[0].type).toBe(CLEAR_FEATURE_FLAGS) + expect(actions[1].type).toBe(FETCH_FEATURE_FLAGS) + expect(actions[2].type).toBe(SET_FEATURE_FLAGS_ERROR) + }) +}) diff --git a/tdrs-frontend/src/actions/fraReports.js b/tdrs-frontend/src/actions/fraReports.js index b3b7ddc64..26aabb38c 100644 --- a/tdrs-frontend/src/actions/fraReports.js +++ b/tdrs-frontend/src/actions/fraReports.js @@ -1,6 +1,5 @@ import { v4 as uuidv4 } from 'uuid' -import axios from 'axios' -import axiosInstance from '../axios-instance' +import { get, post } from '../fetch-instance' import { objectToUrlParams } from '../utils/stringUtils' const BACKEND_URL = process.env.REACT_APP_BACKEND_URL @@ -23,35 +22,31 @@ export const getFraSubmissionHistory = payload: { isLoadingSubmissionHistory: true }, }) - try { - const requestParams = { - stt: stt.id, - file_type: reportType, - year: fiscalYear, - quarter: fiscalQuarter, - } - - const response = await axios.get( - `${BACKEND_URL}/data_files/?${objectToUrlParams(requestParams)}`, - { - responseType: 'json', - } - ) + const requestParams = { + stt: stt.id, + file_type: reportType, + year: fiscalYear, + quarter: fiscalQuarter, + } + const { data, ok, error } = await get( + `${BACKEND_URL}/data_files/?${objectToUrlParams(requestParams)}` + ) + + if (ok) { dispatch({ type: SET_FRA_SUBMISSION_HISTORY, - payload: { submissionHistory: response?.data }, + payload: { submissionHistory: data }, }) - onSuccess() - } catch (error) { + } else { onError(error) - } finally { - dispatch({ - type: SET_IS_LOADING_SUBMISSION_HISTORY, - payload: { isLoadingSubmissionHistory: false }, - }) } + + dispatch({ + type: SET_IS_LOADING_SUBMISSION_HISTORY, + payload: { isLoadingSubmissionHistory: false }, + }) } export const uploadFraReport = @@ -82,25 +77,24 @@ export const uploadFraReport = formData.append(key, value) } - try { - const response = await axiosInstance.post( - `${process.env.REACT_APP_BACKEND_URL}/data_files/`, - formData, - { - headers: { 'Content-Type': 'multipart/form-data' }, - withCredentials: true, - } - ) + const { data, ok, error } = await post( + `${process.env.REACT_APP_BACKEND_URL}/data_files/`, + formData + ) - onSuccess(response?.data) - } catch (error) { - onError(error) - } finally { - dispatch({ - type: SET_IS_UPLOADING_FRA_REPORT, - payload: { isUploadingFraReport: false }, + if (ok) { + onSuccess(data) + } else { + onError({ + message: error?.message || 'Error', + response: { data }, }) } + + dispatch({ + type: SET_IS_UPLOADING_FRA_REPORT, + payload: { isUploadingFraReport: false }, + }) } export const downloadOriginalSubmission = @@ -109,12 +103,13 @@ export const downloadOriginalSubmission = try { if (!id) throw new Error('No id provided to download action') - const response = await axios.get( + const { data, ok, error } = await get( `${BACKEND_URL}/data_files/${id}/download/`, { responseType: 'blob' } ) - const data = response.data + if (!ok) throw error + const url = window.URL.createObjectURL(new Blob([data])) const link = document.createElement('a') @@ -137,12 +132,11 @@ export const downloadOriginalSubmission = } export const getFraSubmissionStatus = async (datafile_id) => { - try { - const response = await axios.get( - `${BACKEND_URL}/data_files/${datafile_id}/` - ) - return response - } catch (axiosError) { - throw axiosError + const { data, ok, error } = await get( + `${BACKEND_URL}/data_files/${datafile_id}/` + ) + if (!ok) { + throw error } + return { data, ok: true } } diff --git a/tdrs-frontend/src/actions/fraReports.test.js b/tdrs-frontend/src/actions/fraReports.test.js index b5f3f95dd..369d1c317 100644 --- a/tdrs-frontend/src/actions/fraReports.test.js +++ b/tdrs-frontend/src/actions/fraReports.test.js @@ -1,4 +1,4 @@ -import axios from 'axios' +import { get, post } from '../fetch-instance' import { thunk } from 'redux-thunk' import configureStore from 'redux-mock-store' @@ -9,24 +9,33 @@ import { SET_IS_UPLOADING_FRA_REPORT, uploadFraReport, downloadOriginalSubmission, + getFraSubmissionStatus, } from './fraReports' +jest.mock('../fetch-instance') + describe('actions/fraReports', () => { - jest.mock('axios') - const mockAxios = axios const mockStore = configureStore([thunk]) + beforeEach(() => { + get.mockClear() + post.mockClear() + }) + describe('getFraSubmissionHistory', () => { it('should handle success without callbacks', async () => { const store = mockStore() - mockAxios.get.mockResolvedValue({ + get.mockResolvedValue({ data: { yay: 'we did it' }, + ok: true, + status: 200, + error: null, }) await store.dispatch( getFraSubmissionHistory({ - stt: 'stt', + stt: { id: 'stt' }, reportType: 'something', fiscalQuarter: '1', fiscalYear: '2', @@ -52,23 +61,22 @@ describe('actions/fraReports', () => { isLoadingSubmissionHistory: false, }) - expect(axios.get).toHaveBeenCalledTimes(1) + expect(get).toHaveBeenCalledTimes(1) }) it('should handle fail without callbacks', async () => { const store = mockStore() - mockAxios.get.mockRejectedValue({ - message: 'Error', - response: { - status: 400, - data: { detail: 'Mock fail response' }, - }, + get.mockResolvedValue({ + data: null, + ok: false, + status: 400, + error: new Error('Mock fail response'), }) await store.dispatch( getFraSubmissionHistory({ - stt: 'stt', + stt: { id: 'stt' }, reportType: 'something', fiscalQuarter: '1', fiscalYear: '2', @@ -89,14 +97,17 @@ describe('actions/fraReports', () => { isLoadingSubmissionHistory: false, }) - expect(axios.get).toHaveBeenCalledTimes(1) + expect(get).toHaveBeenCalledTimes(1) }) it('should call onSuccess', async () => { const store = mockStore() - mockAxios.get.mockResolvedValue({ + get.mockResolvedValue({ data: { yay: 'we did it' }, + ok: true, + status: 200, + error: null, }) const onSuccess = jest.fn() @@ -105,7 +116,7 @@ describe('actions/fraReports', () => { await store.dispatch( getFraSubmissionHistory( { - stt: 'stt', + stt: { id: 'stt' }, reportType: 'something', fiscalQuarter: '1', fiscalYear: '2', @@ -134,7 +145,7 @@ describe('actions/fraReports', () => { isLoadingSubmissionHistory: false, }) - expect(axios.get).toHaveBeenCalledTimes(1) + expect(get).toHaveBeenCalledTimes(1) expect(onSuccess).toHaveBeenCalledTimes(1) expect(onError).toHaveBeenCalledTimes(0) @@ -143,12 +154,11 @@ describe('actions/fraReports', () => { it('should call onError', async () => { const store = mockStore() - mockAxios.get.mockRejectedValue({ - message: 'Error', - response: { - status: 400, - data: { detail: 'Mock fail response' }, - }, + get.mockResolvedValue({ + data: null, + ok: false, + status: 400, + error: new Error('Mock fail response'), }) const onSuccess = jest.fn() @@ -157,7 +167,7 @@ describe('actions/fraReports', () => { await store.dispatch( getFraSubmissionHistory( { - stt: 'stt', + stt: { id: 'stt' }, reportType: 'something', fiscalQuarter: '1', fiscalYear: '2', @@ -181,7 +191,7 @@ describe('actions/fraReports', () => { isLoadingSubmissionHistory: false, }) - expect(axios.get).toHaveBeenCalledTimes(1) + expect(get).toHaveBeenCalledTimes(1) expect(onSuccess).toHaveBeenCalledTimes(0) expect(onError).toHaveBeenCalledTimes(1) @@ -192,22 +202,21 @@ describe('actions/fraReports', () => { it('should handle success without callbacks', async () => { const store = mockStore() - mockAxios.post.mockResolvedValue({ + post.mockResolvedValue({ data: { yay: 'success' }, - }) - - mockAxios.get.mockResolvedValue({ - data: { yay: 'we did it' }, + ok: true, + status: 200, + error: null, }) await store.dispatch( uploadFraReport({ - stt: 'stt', + stt: { id: 'stt' }, reportType: 'something', fiscalQuarter: '1', fiscalYear: '2', - file: 'bytes', - user: 'me', + file: { name: 'bytes' }, + user: { id: 'me' }, }) ) @@ -225,33 +234,28 @@ describe('actions/fraReports', () => { isUploadingFraReport: false, }) - expect(axios.post).toHaveBeenCalledTimes(1) - expect(axios.get).toHaveBeenCalledTimes(0) + expect(post).toHaveBeenCalledTimes(1) + expect(get).toHaveBeenCalledTimes(0) }) it('should handle fail without callbacks', async () => { const store = mockStore() - mockAxios.post.mockRejectedValue({ - message: 'Error', - response: { - status: 400, - data: { detail: 'Mock fail response' }, - }, - }) - - mockAxios.get.mockResolvedValue({ - data: { yay: 'we did it' }, + post.mockResolvedValue({ + data: null, + ok: false, + status: 400, + error: new Error('Mock fail response'), }) await store.dispatch( uploadFraReport({ - stt: 'stt', + stt: { id: 'stt' }, reportType: 'something', fiscalQuarter: '1', fiscalYear: '2', - file: 'bytes', - user: 'me', + file: { name: 'bytes' }, + user: { id: 'me' }, }) ) @@ -269,19 +273,18 @@ describe('actions/fraReports', () => { isUploadingFraReport: false, }) - expect(axios.post).toHaveBeenCalledTimes(1) - expect(axios.get).toHaveBeenCalledTimes(0) + expect(post).toHaveBeenCalledTimes(1) + expect(get).toHaveBeenCalledTimes(0) }) it('should call onSuccess', async () => { const store = mockStore() - mockAxios.post.mockResolvedValue({ + post.mockResolvedValue({ data: { yay: 'success' }, - }) - - mockAxios.get.mockResolvedValue({ - data: { yay: 'we did it' }, + ok: true, + status: 200, + error: null, }) const onSuccess = jest.fn() @@ -290,12 +293,12 @@ describe('actions/fraReports', () => { await store.dispatch( uploadFraReport( { - stt: 'stt', + stt: { id: 'stt' }, reportType: 'something', fiscalQuarter: '1', fiscalYear: '2', - file: 'bytes', - user: 'me', + file: { name: 'bytes' }, + user: { id: 'me' }, }, onSuccess, onError @@ -316,8 +319,8 @@ describe('actions/fraReports', () => { isUploadingFraReport: false, }) - expect(axios.post).toHaveBeenCalledTimes(1) - expect(axios.get).toHaveBeenCalledTimes(0) + expect(post).toHaveBeenCalledTimes(1) + expect(get).toHaveBeenCalledTimes(0) expect(onSuccess).toHaveBeenCalledTimes(1) expect(onError).toHaveBeenCalledTimes(0) @@ -326,16 +329,11 @@ describe('actions/fraReports', () => { it('should call onError', async () => { const store = mockStore() - mockAxios.post.mockRejectedValue({ - message: 'Error', - response: { - status: 400, - data: { detail: 'Mock fail response' }, - }, - }) - - mockAxios.get.mockResolvedValue({ - data: { yay: 'we did it' }, + post.mockResolvedValue({ + data: null, + ok: false, + status: 400, + error: new Error('Mock fail response'), }) const onSuccess = jest.fn() @@ -344,12 +342,12 @@ describe('actions/fraReports', () => { await store.dispatch( uploadFraReport( { - stt: 'stt', + stt: { id: 'stt' }, reportType: 'something', fiscalQuarter: '1', fiscalYear: '2', - file: 'bytes', - user: 'me', + file: { name: 'bytes' }, + user: { id: 'me' }, }, onSuccess, onError @@ -370,8 +368,8 @@ describe('actions/fraReports', () => { isUploadingFraReport: false, }) - expect(axios.post).toHaveBeenCalledTimes(1) - expect(axios.get).toHaveBeenCalledTimes(0) + expect(post).toHaveBeenCalledTimes(1) + expect(get).toHaveBeenCalledTimes(0) expect(onSuccess).toHaveBeenCalledTimes(0) expect(onError).toHaveBeenCalledTimes(1) @@ -385,5 +383,110 @@ describe('actions/fraReports', () => { const actions = store.getActions() expect(actions.length).toEqual(0) }) + + it('downloads file on success', async () => { + const store = mockStore() + const blob = new Blob(['file-content']) + + get.mockResolvedValue({ + data: blob, + ok: true, + status: 200, + error: null, + }) + + const mockLink = { + href: '', + setAttribute: jest.fn(), + click: jest.fn(), + } + jest.spyOn(document, 'createElement').mockReturnValue(mockLink) + jest.spyOn(document.body, 'appendChild').mockImplementation(() => {}) + jest.spyOn(document.body, 'removeChild').mockImplementation(() => {}) + window.URL.createObjectURL = jest.fn(() => 'blob:test-url') + + await store.dispatch( + downloadOriginalSubmission({ + id: 42, + fileName: 'report.txt', + year: '2025', + quarter: 'Q1', + section: 'Active Case Data', + }) + ) + + expect(get).toHaveBeenCalledWith( + expect.stringContaining('/data_files/42/download/'), + { responseType: 'blob' } + ) + expect(mockLink.setAttribute).toHaveBeenCalledWith( + 'download', + 'report (2025-Q1-Active Case Data).txt' + ) + expect(mockLink.click).toHaveBeenCalled() + expect(document.body.removeChild).toHaveBeenCalledWith(mockLink) + + document.createElement.mockRestore() + document.body.appendChild.mockRestore() + document.body.removeChild.mockRestore() + }) + + it('logs error when API returns non-ok response', async () => { + const store = mockStore() + const consoleSpy = jest.spyOn(console, 'error').mockImplementation() + + get.mockResolvedValue({ + data: null, + ok: false, + status: 500, + error: new Error('Server error'), + }) + + await store.dispatch( + downloadOriginalSubmission({ + id: 42, + fileName: 'report.txt', + year: '2025', + quarter: 'Q1', + section: 'Active Case Data', + }) + ) + + expect(consoleSpy).toHaveBeenCalledWith( + 'error downloading file', + expect.any(Error) + ) + consoleSpy.mockRestore() + }) + }) + + describe('getFraSubmissionStatus', () => { + it('returns data on success', async () => { + get.mockResolvedValue({ + data: { status: 'complete' }, + ok: true, + status: 200, + error: null, + }) + + const result = await getFraSubmissionStatus(99) + + expect(get).toHaveBeenCalledWith( + expect.stringContaining('/data_files/99/') + ) + expect(result).toEqual({ data: { status: 'complete' }, ok: true }) + }) + + it('throws error on non-ok response', async () => { + const mockError = new Error('Not found') + get.mockResolvedValue({ + data: null, + ok: false, + status: 404, + error: mockError, + }) + + await expect(getFraSubmissionStatus(99)).rejects.toThrow('Not found') + }) }) }) diff --git a/tdrs-frontend/src/actions/reports.js b/tdrs-frontend/src/actions/reports.js index ca4c388c4..4662ba132 100644 --- a/tdrs-frontend/src/actions/reports.js +++ b/tdrs-frontend/src/actions/reports.js @@ -1,7 +1,6 @@ import { v4 as uuidv4 } from 'uuid' -import axios from 'axios' -import axiosInstance from '../axios-instance' +import { get, post } from '../fetch-instance' import { logErrorToServer } from '../utils/eventLogger' import removeFileInputErrorState from '../utils/removeFileInputErrorState' import { quarters } from '../components/Reports/utils' @@ -58,22 +57,13 @@ export const getAvailableFileList = dispatch({ type: FETCH_FILE_LIST, }) - try { - let url = `${BACKEND_URL}/data_files/?year=${year}&stt=${stt.id}&file_type=${file_type}` - if (quarter) { - url += `&quarter=${quarter}` - } - const response = await axios.get(url, { - responseType: 'json', - }) - dispatch({ - type: SET_FILE_LIST, - payload: { - data: response?.data, - }, - }) - onSuccess() - } catch (error) { + let url = `${BACKEND_URL}/data_files/?year=${year}&stt=${stt.id}&file_type=${file_type}` + if (quarter) { + url += `&quarter=${quarter}` + } + const { data, ok, error } = await get(url) + + if (!ok) { dispatch({ type: FETCH_FILE_LIST_ERROR, payload: { @@ -84,7 +74,16 @@ export const getAvailableFileList = section, }, }) + return } + + dispatch({ + type: SET_FILE_LIST, + payload: { + data, + }, + }) + onSuccess() } export const download = @@ -93,13 +92,12 @@ export const download = try { if (!id) throw new Error('No id was provided to download action.') dispatch({ type: START_FILE_DOWNLOAD }) - const response = await axios.get( + const { data, ok, error } = await get( `${BACKEND_URL}/data_files/${id}/download/`, - { - responseType: 'blob', - } + { responseType: 'blob' } ) - const data = response.data + + if (!ok) throw error // Create a link and associate it with the blob returned from the file // download - this allows us to trigger the file download dialog without @@ -203,18 +201,19 @@ export const submit = for (const [key, value] of Object.entries(dataFile)) { formData.append(key, value) } - return axiosInstance.post( - `${process.env.REACT_APP_BACKEND_URL}/data_files/`, - formData, - { - headers: { 'Content-Type': 'multipart/form-data' }, - withCredentials: true, - } - ) + return post(`${process.env.REACT_APP_BACKEND_URL}/data_files/`, formData) }) return Promise.all(submissionRequests) .then((responses) => { + // Check if any responses have errors + const failedResponse = responses.find((r) => !r.ok) + if (failedResponse) { + const err = new Error(failedResponse.error?.message || 'Error') + err.response = { data: failedResponse.data } + throw err + } + setLocalAlertState({ active: true, type: 'success', @@ -279,12 +278,11 @@ export const setStt = (stt) => (dispatch) => { } export const getTanfSubmissionStatus = async (datafile_id) => { - try { - const response = await axios.get( - `${BACKEND_URL}/data_files/${datafile_id}/` - ) - return response - } catch (axiosError) { - throw axiosError + const { data, ok, error } = await get( + `${BACKEND_URL}/data_files/${datafile_id}/` + ) + if (!ok) { + throw error } + return { data, ok: true } } diff --git a/tdrs-frontend/src/actions/reports.test.js b/tdrs-frontend/src/actions/reports.test.js index fdf7527ff..f8438f8ed 100644 --- a/tdrs-frontend/src/actions/reports.test.js +++ b/tdrs-frontend/src/actions/reports.test.js @@ -1,4 +1,4 @@ -import axios from 'axios' +import { get, post } from '../fetch-instance' import { thunk } from 'redux-thunk' import configureStore from 'redux-mock-store' import { v4 as uuidv4 } from 'uuid' @@ -19,6 +19,8 @@ import { SET_FILE_SUBMITTED, } from './reports' +jest.mock('../fetch-instance') + describe('actions/reports', () => { const mockStore = configureStore([thunk]) @@ -61,9 +63,12 @@ describe('actions/reports', () => { it('should dispatch OPEN_FILE_DIALOG when a file has been successfully downloaded', async () => { window.URL.createObjectURL = jest.fn(() => null) - axios.get.mockImplementationOnce(() => + get.mockImplementationOnce(() => Promise.resolve({ data: 'Some text', + ok: true, + status: 200, + error: null, }) ) const store = mockStore() @@ -86,7 +91,7 @@ describe('actions/reports', () => { }) it('should dispatch SET_FILE_LIST', async () => { - axios.get.mockImplementationOnce(() => + get.mockImplementationOnce(() => Promise.resolve({ data: [ { @@ -100,6 +105,9 @@ describe('actions/reports', () => { uuid: uuidv4(), }, ], + ok: true, + status: 200, + error: null, }) ) const store = mockStore() @@ -120,7 +128,7 @@ describe('actions/reports', () => { it('should dispatch SET_FILE_SUBMITTED', async () => { const uuid = uuidv4() - axios.post.mockImplementationOnce(() => + post.mockImplementationOnce(() => Promise.resolve({ data: { extension: 'txt', @@ -131,6 +139,9 @@ describe('actions/reports', () => { slug: uuid, year: 2021, }, + ok: true, + status: 200, + error: null, }) ) const store = mockStore() @@ -157,7 +168,7 @@ describe('actions/reports', () => { const actions = store.getActions() try { - expect(axios.post).toHaveBeenCalledTimes(1) + expect(post).toHaveBeenCalledTimes(1) expect(actions[0].type).toBe(SET_FILE_SUBMITTED) } catch (err) { throw actions[0].payload.error @@ -183,13 +194,12 @@ describe('actions/reports', () => { 'should set local alert state on submission failure', async (data, msg) => { const uuid = uuidv4() - axios.post.mockImplementationOnce(() => - Promise.reject({ + post.mockImplementationOnce(() => + Promise.resolve({ + data, + ok: false, status: 400, - message: 'Error', - response: { - data, - }, + error: new Error('Error'), }) ) const store = mockStore() @@ -216,7 +226,7 @@ describe('actions/reports', () => { }) ) - expect(axios.post).toHaveBeenCalledTimes(1) + expect(post).toHaveBeenCalledTimes(1) expect(setLocalAlertState).toHaveBeenCalledWith({ active: true, message: msg || 'Error: Something went wrong', diff --git a/tdrs-frontend/src/actions/requestAccess.js b/tdrs-frontend/src/actions/requestAccess.js index 01a4b86ab..9f0eac5fd 100644 --- a/tdrs-frontend/src/actions/requestAccess.js +++ b/tdrs-frontend/src/actions/requestAccess.js @@ -1,5 +1,5 @@ import { SET_AUTH } from './auth' -import axios from 'axios' +import { patch } from '../fetch-instance' import { logErrorToServer } from '../utils/eventLogger' export const PATCH_REQUEST_ACCESS = 'PATCH_REQUEST_ACCESS' @@ -11,30 +11,29 @@ export const requestAccess = ({ firstName, lastName, stt, regions, hasFRAAccess }) => async (dispatch) => { dispatch({ type: PATCH_REQUEST_ACCESS }) - try { - const URL = `${process.env.REACT_APP_BACKEND_URL}/users/request_access/` - const user = { - first_name: firstName, - last_name: lastName, - stt: stt?.id, - regions: regions ? [...regions] : [], - has_fra_access: hasFRAAccess, - } - const { data } = await axios.patch(URL, user, { - withCredentials: true, - }) + const URL = `${process.env.REACT_APP_BACKEND_URL}/users/request_access/` + const user = { + first_name: firstName, + last_name: lastName, + stt: stt?.id, + regions: regions ? [...regions] : [], + has_fra_access: hasFRAAccess, + } + const { data, ok, error } = await patch(URL, user) - if (data) { - dispatch({ type: SET_REQUEST_ACCESS }) - dispatch({ - type: SET_AUTH, - payload: { user: data }, - }) - } else { - dispatch({ type: CLEAR_REQUEST_ACCESS }) - } - } catch (error) { + if (!ok) { logErrorToServer(SET_REQUEST_ACCESS_ERROR) dispatch({ type: SET_REQUEST_ACCESS_ERROR, payload: { error } }) + return + } + + if (data) { + dispatch({ type: SET_REQUEST_ACCESS }) + dispatch({ + type: SET_AUTH, + payload: { user: data }, + }) + } else { + dispatch({ type: CLEAR_REQUEST_ACCESS }) } } diff --git a/tdrs-frontend/src/actions/requestAccess.test.js b/tdrs-frontend/src/actions/requestAccess.test.js index 137e442a5..0be8450a1 100644 --- a/tdrs-frontend/src/actions/requestAccess.test.js +++ b/tdrs-frontend/src/actions/requestAccess.test.js @@ -1,4 +1,4 @@ -import axios from 'axios' +import { patch } from '../fetch-instance' import { thunk } from 'redux-thunk' import configureStore from 'redux-mock-store' @@ -10,22 +10,29 @@ import { CLEAR_REQUEST_ACCESS, } from './requestAccess' +jest.mock('../fetch-instance') + describe('actions/requestAccess.js', () => { const mockStore = configureStore([thunk]) it('sends a PATCH request when requestAccess is called', async () => { - axios.patch = jest.fn().mockResolvedValue({ - data: { - first_name: 'harry', - last_name: 'potter', - stt: { - code: 'AK', - id: 2, - name: 'Alaska', - type: 'state', + patch.mockImplementationOnce(() => + Promise.resolve({ + data: { + first_name: 'harry', + last_name: 'potter', + stt: { + code: 'AK', + id: 2, + name: 'Alaska', + type: 'state', + }, }, - }, - }) + ok: true, + status: 200, + error: null, + }) + ) const profileInfo = { firstName: 'harry', lastName: 'potter', @@ -42,7 +49,14 @@ describe('actions/requestAccess.js', () => { }) it('clears the request access state if there is no data returned from the API', async () => { - axios.patch = jest.fn().mockResolvedValue({}) + patch.mockImplementationOnce(() => + Promise.resolve({ + data: null, + ok: true, + status: 200, + error: null, + }) + ) const profileInfo = { firstName: 'harry', lastName: 'potter', @@ -59,7 +73,14 @@ describe('actions/requestAccess.js', () => { }) it('dispatches an error to the store if the API errors', async () => { - axios.patch = jest.fn().mockRejectedValue(new Error('threw an error')) + patch.mockImplementationOnce(() => + Promise.resolve({ + data: null, + ok: false, + status: 500, + error: new Error('threw an error'), + }) + ) const profileInfo = { firstName: 'harry', lastName: 'potter', diff --git a/tdrs-frontend/src/actions/sttList.js b/tdrs-frontend/src/actions/sttList.js index dbf74aa99..84be2a850 100644 --- a/tdrs-frontend/src/actions/sttList.js +++ b/tdrs-frontend/src/actions/sttList.js @@ -1,4 +1,4 @@ -import axios from 'axios' +import { get } from '../fetch-instance' import { logErrorToServer } from '../utils/eventLogger' export const FETCH_STTS = 'FETCH_STTS' @@ -32,26 +32,25 @@ export const CLEAR_STTS = 'CLEAR_STTS' */ export const fetchSttList = () => async (dispatch) => { dispatch({ type: FETCH_STTS }) - try { - const URL = `${process.env.REACT_APP_BACKEND_URL}/stts/alpha` - const { data } = await axios.get(URL, { - withCredentials: true, - }) + const URL = `${process.env.REACT_APP_BACKEND_URL}/stts/alpha` + const { data, ok, error } = await get(URL) - if (data) { - // shouldn't this logic be done by the backend serializer? - data.forEach((item, i) => { - if (item.name === 'Federal Government') { - data.splice(i, 1) - data.unshift(item) - } - }) - dispatch({ type: SET_STTS, payload: { data } }) - } else { - dispatch({ type: CLEAR_STTS }) - } - } catch (error) { + if (!ok) { logErrorToServer(SET_STTS_ERROR) dispatch({ type: SET_STTS_ERROR, payload: { error } }) + return + } + + if (data) { + // shouldn't this logic be done by the backend serializer? + data.forEach((item, i) => { + if (item.name === 'Federal Government') { + data.splice(i, 1) + data.unshift(item) + } + }) + dispatch({ type: SET_STTS, payload: { data } }) + } else { + dispatch({ type: CLEAR_STTS }) } } diff --git a/tdrs-frontend/src/actions/sttList.test.js b/tdrs-frontend/src/actions/sttList.test.js index 3d053917f..7eb5aa54b 100644 --- a/tdrs-frontend/src/actions/sttList.test.js +++ b/tdrs-frontend/src/actions/sttList.test.js @@ -1,4 +1,4 @@ -import axios from 'axios' +import { get } from '../fetch-instance' import { thunk } from 'redux-thunk' import configureStore from 'redux-mock-store' @@ -10,13 +10,18 @@ import { CLEAR_STTS, } from './sttList' +jest.mock('../fetch-instance') + describe('actions/stts.js', () => { const mockStore = configureStore([thunk]) it('fetches a list of stts, when the user is authenticated', async () => { - axios.get.mockImplementationOnce(() => + get.mockImplementationOnce(() => Promise.resolve({ data: [{ id: 1, type: 'state', code: 'AL', name: 'Alabama' }], + ok: true, + status: 200, + error: null, }) ) const store = mockStore() @@ -32,7 +37,7 @@ describe('actions/stts.js', () => { }) it('fetches a list of stts and puts "Federal Government" as the first option if it exists, when the user is authenticated', async () => { - axios.get.mockImplementationOnce(() => + get.mockImplementationOnce(() => Promise.resolve({ data: [ { id: 1, type: 'state', code: 'AL', name: 'Alabama' }, @@ -43,6 +48,9 @@ describe('actions/stts.js', () => { name: 'Federal Government', }, ], + ok: true, + status: 200, + error: null, }) ) const store = mockStore() @@ -64,7 +72,14 @@ describe('actions/stts.js', () => { }) it('clears the stt state, if user is not authenticated', async () => { - axios.get.mockImplementationOnce(() => Promise.resolve({ test: {} })) + get.mockImplementationOnce(() => + Promise.resolve({ + data: null, + ok: true, + status: 200, + error: null, + }) + ) const store = mockStore() await store.dispatch(fetchSttList()) @@ -75,8 +90,14 @@ describe('actions/stts.js', () => { }) it('dispatches an error to the store if the API errors', async () => { - axios.get.mockImplementationOnce(() => - Promise.reject(Error({ message: 'something went wrong' })) + const mockError = new Error('something went wrong') + get.mockImplementationOnce(() => + Promise.resolve({ + data: null, + ok: false, + status: 500, + error: mockError, + }) ) const store = mockStore() @@ -86,7 +107,7 @@ describe('actions/stts.js', () => { expect(actions[0].type).toBe(FETCH_STTS) expect(actions[1].type).toBe(SET_STTS_ERROR) expect(actions[1].payload).toStrictEqual({ - error: Error({ message: 'something went wrong' }), + error: mockError, }) }) }) diff --git a/tdrs-frontend/src/actions/updateUserRequest.js b/tdrs-frontend/src/actions/updateUserRequest.js index d36710450..dc85a46a0 100644 --- a/tdrs-frontend/src/actions/updateUserRequest.js +++ b/tdrs-frontend/src/actions/updateUserRequest.js @@ -1,4 +1,4 @@ -import axios from 'axios' +import { patch } from '../fetch-instance' import { logErrorToServer } from '../utils/eventLogger' import { SET_AUTH } from './auth' @@ -11,33 +11,32 @@ export const updateUserRequest = ({ firstName, lastName, stt, regions, hasFRAAccess }) => async (dispatch) => { dispatch({ type: PATCH_REQUEST_USER_UPDATE }) - try { - const URL = `${process.env.REACT_APP_BACKEND_URL}/users/update_profile/` - const user = { - first_name: firstName, - last_name: lastName, - stt: stt?.id, - has_fra_access: hasFRAAccess, - create_change_requests: true, - // backend requires region value if region key is present. - // this guards it so the key isn't present if value isn't - ...(Array.isArray(regions) && regions.length > 0 ? { regions } : {}), - } - const { data } = await axios.patch(URL, user, { - withCredentials: true, - }) + const URL = `${process.env.REACT_APP_BACKEND_URL}/users/update_profile/` + const user = { + first_name: firstName, + last_name: lastName, + stt: stt?.id, + has_fra_access: hasFRAAccess, + create_change_requests: true, + // backend requires region value if region key is present. + // this guards it so the key isn't present if value isn't + ...(Array.isArray(regions) && regions.length > 0 ? { regions } : {}), + } + const { data, ok, error } = await patch(URL, user) - if (data) { - dispatch({ type: SET_REQUEST_USER_UPDATE }) - dispatch({ - type: SET_AUTH, - payload: { user: data }, - }) - } else { - dispatch({ type: CLEAR_REQUEST_USER_UPDATE }) - } - } catch (error) { + if (!ok) { logErrorToServer(SET_REQUEST_USER_UPDATE_ERROR) dispatch({ type: SET_REQUEST_USER_UPDATE_ERROR, payload: { error } }) + return + } + + if (data) { + dispatch({ type: SET_REQUEST_USER_UPDATE }) + dispatch({ + type: SET_AUTH, + payload: { user: data }, + }) + } else { + dispatch({ type: CLEAR_REQUEST_USER_UPDATE }) } } diff --git a/tdrs-frontend/src/actions/updateUserRequest.test.js b/tdrs-frontend/src/actions/updateUserRequest.test.js index a5f017be5..41381335a 100644 --- a/tdrs-frontend/src/actions/updateUserRequest.test.js +++ b/tdrs-frontend/src/actions/updateUserRequest.test.js @@ -1,4 +1,4 @@ -import axios from 'axios' +import { patch } from '../fetch-instance' import configureStore from 'redux-mock-store' import { thunk } from 'redux-thunk' import { SET_AUTH } from './auth' @@ -10,6 +10,8 @@ import { updateUserRequest, } from './updateUserRequest' +jest.mock('../fetch-instance') + const middlewares = [thunk] const mockStore = configureStore(middlewares) @@ -32,7 +34,12 @@ describe('updateUserRequest', () => { has_fra_access: true, pending_requests: 1, } - axios.patch.mockResolvedValue({ data: apiUserResponse }) + patch.mockResolvedValue({ + data: apiUserResponse, + ok: true, + status: 200, + error: null, + }) await store.dispatch(updateUserRequest(mockInput)) @@ -61,7 +68,12 @@ describe('updateUserRequest', () => { has_fra_access: false, pending_requests: 0, } - axios.patch.mockResolvedValue({ data: apiUserResponse }) + patch.mockResolvedValue({ + data: apiUserResponse, + ok: true, + status: 200, + error: null, + }) await store.dispatch(updateUserRequest(mockInput)) @@ -83,7 +95,12 @@ describe('updateUserRequest', () => { hasFRAAccess: false, } - axios.patch.mockRejectedValue(new Error('threw and error')) + patch.mockResolvedValue({ + data: null, + ok: false, + status: 500, + error: new Error('threw an error'), + }) await store.dispatch(updateUserRequest(mockInput)) @@ -92,7 +109,7 @@ describe('updateUserRequest', () => { expect(actions[1].type).toBe(SET_REQUEST_USER_UPDATE_ERROR) }) - it('dispatches an error to the store if the API errors', async () => { + it('clears the state if the API returns no data', async () => { const store = mockStore() const mockInput = { @@ -102,7 +119,12 @@ describe('updateUserRequest', () => { hasFRAAccess: false, } - axios.patch.mockResolvedValue({}) + patch.mockResolvedValue({ + data: null, + ok: true, + status: 200, + error: null, + }) await store.dispatch(updateUserRequest(mockInput)) diff --git a/tdrs-frontend/src/axios-instance.js b/tdrs-frontend/src/axios-instance.js deleted file mode 100644 index 3a7dfdac8..000000000 --- a/tdrs-frontend/src/axios-instance.js +++ /dev/null @@ -1,52 +0,0 @@ -import axios from 'axios' -import { faro } from '@grafana/faro-react' - -// Need a custom instance of axios so we can set the csrf keys on auth_check -// Work around for csrf cookie issue we encountered in production. -// It may still be possible to do this with a cookie, and something on the -// frontend (most likely) is misconfigured. the configuration has alluded -// us thus far, and this implementation is functionally equivalent to -// using cookies. - -const axiosInstance = axios.create() - -// Add an interceptor to include trace context in outgoing requests for custom instance and default instance -axios.interceptors.request.use((config) => { - try { - // Add service name to the request - config.headers = config.headers || {} - config.headers['x-service-name'] = 'tdp-frontend' - // Add trace context if Faro is initialized - if (faro && faro.api) { - const traceContext = faro.api.getTraceContext() - if (traceContext) { - // Add W3C trace context headers - Object.assign(config.headers, traceContext) - } - } - } catch (e) { - console.error('Failed to add trace context', e) - } - return config -}) - -axiosInstance.interceptors.request.use((config) => { - try { - // Add service name to the request - config.headers = config.headers || {} - config.headers['x-service-name'] = 'tdp-frontend' - // Add trace context if Faro is initialized - if (faro && faro.api) { - const traceContext = faro.api.getTraceContext() - if (traceContext) { - // Add W3C trace context headers - Object.assign(config.headers, traceContext) - } - } - } catch (e) { - console.error('Failed to add trace context', e) - } - return config -}) - -export default axiosInstance diff --git a/tdrs-frontend/src/components/Button/Button.jsx b/tdrs-frontend/src/components/Button/Button.jsx index f4581223c..5471799ca 100644 --- a/tdrs-frontend/src/components/Button/Button.jsx +++ b/tdrs-frontend/src/components/Button/Button.jsx @@ -18,6 +18,7 @@ function Button({ target = '_blank', href, buttonKey = null, + ...rest }) { const isBig = size ? size === 'big' : false const isSmall = size ? size === 'small' : false @@ -61,6 +62,7 @@ function Button({ disabled={disabled} aria-disabled={disabled} buttonkey={buttonKey} + {...rest} > {children} @@ -78,6 +80,7 @@ function Button({ aria-disabled={disabled || undefined} onClick={handleClick} buttonkey={buttonKey} + {...rest} > {children} diff --git a/tdrs-frontend/src/components/Feedback/FeedbackForm.jsx b/tdrs-frontend/src/components/Feedback/FeedbackForm.jsx index f4d129aef..5aea1c4dd 100644 --- a/tdrs-frontend/src/components/Feedback/FeedbackForm.jsx +++ b/tdrs-frontend/src/components/Feedback/FeedbackForm.jsx @@ -1,6 +1,6 @@ /* eslint-disable no-unused-vars */ import React, { useCallback, useEffect, useRef, useState } from 'react' -import axiosInstance from '../../axios-instance' +import { post, patch } from '../../fetch-instance' import classNames from 'classnames' import FeedbackRadioSelectGroup from './FeedbackRadioSelectGroup' import { useSelector } from 'react-redux' @@ -50,22 +50,12 @@ const FeedbackForm = ({ } const postFeedback = useCallback(async (payload) => { - return axiosInstance.post(`${BACKEND_URL}/feedback/`, payload, { - headers: { 'Content-Type': 'application/json' }, - withCredentials: true, - }) + return post(`${BACKEND_URL}/feedback/`, payload) }, []) const updateFeedback = useCallback( async (payload) => { - return axiosInstance.patch( - `${BACKEND_URL}/feedback/${feedbackID}/`, - payload, - { - headers: { 'Content-Type': 'application/json' }, - withCredentials: true, - } - ) + return patch(`${BACKEND_URL}/feedback/${feedbackID}/`, payload) }, [feedbackID] ) @@ -118,19 +108,17 @@ const FeedbackForm = ({ const submitFeedback = useCallback( async (payload) => { - try { - const response = feedbackID - ? await updateFeedback(payload) - : await postFeedback(payload) + const response = feedbackID + ? await updateFeedback(payload) + : await postFeedback(payload) - if (response.status === 200 || response.status === 201) { - setFeedbackID(response.data.id) - } - return response - } catch (error) { - console.error('Error submitting feedback:', error) + if (response.ok) { + setFeedbackID(response.data.id) + } else { + console.error('Error submitting feedback:', response.error) onRequestError?.() } + return response }, [ feedbackID, @@ -151,28 +139,15 @@ const FeedbackForm = ({ return } - try { - const payload = constructPayload() - const response = await submitFeedback(payload) + const payload = constructPayload() + const response = await submitFeedback(payload) - if (response.status === 200 || response.status === 201) { - onFeedbackSubmit() - onRequestSuccess?.() - resetStatesOnceSubmitted() - } else { - console.error('Unexpected response: ', response) - onRequestError?.() - } - } catch (error) { - const status = error?.response?.status - if (status === 400) { - console.error('Error submitting feedback: ', error.response) - } else { - console.error( - 'An unexpected error occurred. Please try again later.', - error - ) - } + if (response.ok) { + onFeedbackSubmit() + onRequestSuccess?.() + resetStatesOnceSubmitted() + } else { + console.error('Unexpected response: ', response) onRequestError?.() } }, [ diff --git a/tdrs-frontend/src/components/Feedback/FeedbackForm.test.js b/tdrs-frontend/src/components/Feedback/FeedbackForm.test.js index 2b9fb066b..713eb2236 100644 --- a/tdrs-frontend/src/components/Feedback/FeedbackForm.test.js +++ b/tdrs-frontend/src/components/Feedback/FeedbackForm.test.js @@ -1,10 +1,10 @@ import React from 'react' import * as reactRedux from 'react-redux' -import { render, screen, fireEvent, waitFor } from '@testing-library/react' +import { render, screen, fireEvent, waitFor, act } from '@testing-library/react' import '@testing-library/jest-dom' import FeedbackForm from './FeedbackForm' import { useSelector } from 'react-redux' -import axiosInstance from '../../axios-instance' +import { post, patch } from '../../fetch-instance' // Mock the Redux selector jest.mock('react-redux', () => ({ @@ -12,7 +12,7 @@ jest.mock('react-redux', () => ({ useSelector: jest.fn(), })) -jest.mock('../../axios-instance') +jest.mock('../../fetch-instance') jest.mock('../../assets/feedback/very-dissatisfied-feedback.svg', () => { const React = require('react') @@ -78,8 +78,8 @@ describe('Feedback Form tests', () => { const mockOnFeedbackSubmit = jest.fn() beforeEach(() => { - axiosInstance.post.mockClear() - axiosInstance.patch.mockClear() + post.mockClear() + patch.mockClear() mockOnFeedbackSubmit.mockClear() // Default to authenticated for most tests reactRedux.useSelector.mockImplementation(() => true) @@ -197,7 +197,7 @@ describe('Feedback Form tests', () => { ).toBeInTheDocument() }) - expect(axiosInstance.post).not.toHaveBeenCalledWith( + expect(post).not.toHaveBeenCalledWith( expect.anything(), expect.objectContaining({ rating: expect.any(Number) }) ) @@ -221,10 +221,17 @@ describe('Feedback Form tests', () => { ).toBeInTheDocument() }) - expect(axiosInstance.post).not.toHaveBeenCalled() + expect(post).not.toHaveBeenCalled() }) it('clears error message after rating is selected', async () => { + post.mockResolvedValue({ + ok: true, + status: 201, + data: { id: 1 }, + error: null, + }) + render( { }) it('submits feedback with rating, message, and anonymous flag', async () => { - axiosInstance.post.mockResolvedValue({ status: 201, data: { id: 1 } }) - axiosInstance.patch.mockResolvedValue({ status: 200, data: { id: 1 } }) + post.mockResolvedValue({ + ok: true, + status: 201, + data: { id: 1 }, + error: null, + }) + patch.mockResolvedValue({ + ok: true, + status: 200, + data: { id: 1 }, + error: null, + }) render( { const ratingInput = screen.getByTestId('feedback-radio-input-4') fireEvent.click(ratingInput) + // Wait for rating click's async post to complete and feedbackID to be set + await waitFor(() => expect(post).toHaveBeenCalled()) + // Simulate entering feedback message fireEvent.change(screen.getByTestId('feedback-message-input'), { target: { value: 'Great!! test feedback' }, @@ -274,7 +294,7 @@ describe('Feedback Form tests', () => { fireEvent.click(screen.getByRole('button', { name: /send feedback/i })) await waitFor(() => { - expect(axiosInstance.post).toHaveBeenCalledWith( + expect(patch).toHaveBeenCalledWith( expect.stringContaining('/feedback/'), { rating: 4, @@ -283,10 +303,6 @@ describe('Feedback Form tests', () => { feedback_type: 'general_feedback', page_url: 'http://localhost/', anonymous: true, - }, - { - headers: { 'Content-Type': 'application/json' }, - withCredentials: true, } ) }) @@ -294,8 +310,18 @@ describe('Feedback Form tests', () => { }) it('submits with rating and no feedback message', async () => { - axiosInstance.post.mockResolvedValue({ status: 201, data: { id: 1 } }) - axiosInstance.patch.mockResolvedValue({ status: 200, data: { id: 1 } }) + post.mockResolvedValue({ + ok: true, + status: 201, + data: { id: 1 }, + error: null, + }) + patch.mockResolvedValue({ + ok: true, + status: 200, + data: { id: 1 }, + error: null, + }) render( { /> ) - fireEvent.click(screen.getByTestId('feedback-radio-input-3')) + await act(async () => { + fireEvent.click(screen.getByTestId('feedback-radio-input-3')) + }) + expect(post).toHaveBeenCalled() + fireEvent.click(screen.getByRole('button', { name: /send feedback/i })) await waitFor(() => - expect(axiosInstance.post).toHaveBeenCalledWith( + expect(patch).toHaveBeenCalledWith( expect.stringContaining('/feedback/'), { component: 'general-website', @@ -317,18 +347,24 @@ describe('Feedback Form tests', () => { rating: 3, feedback: '', anonymous: false, - }, - { - headers: { 'Content-Type': 'application/json' }, - withCredentials: true, } ) ) }) it('submits form using Enter key on submit button', async () => { - axiosInstance.post.mockResolvedValue({ status: 201, data: { id: 1 } }) - axiosInstance.patch.mockResolvedValue({ status: 200, data: { id: 1 } }) + post.mockResolvedValue({ + ok: true, + status: 201, + data: { id: 1 }, + error: null, + }) + patch.mockResolvedValue({ + ok: true, + status: 200, + data: { id: 1 }, + error: null, + }) render( { fireEvent.keyDown(button, { key: 'Enter', code: 'Enter' }) await waitFor(() => { - expect(axiosInstance.post).toHaveBeenCalled() + expect(post).toHaveBeenCalled() expect(mockOnFeedbackSubmit).toHaveBeenCalled() }) }) it('submits form with Cmd/Ctrl + Enter from inside textarea', async () => { - axiosInstance.post.mockResolvedValue({ status: 201, data: { id: 1 } }) - axiosInstance.patch.mockResolvedValue({ status: 200, data: { id: 1 } }) + post.mockResolvedValue({ + ok: true, + status: 201, + data: { id: 1 }, + error: null, + }) + patch.mockResolvedValue({ + ok: true, + status: 200, + data: { id: 1 }, + error: null, + }) render( { const textarea = screen.getByTestId('feedback-message-input') textarea.focus() fireEvent.change(textarea, { target: { value: 'Quick feedback' } }) - fireEvent.click(screen.getByTestId('feedback-radio-input-3')) + + await act(async () => { + fireEvent.click(screen.getByTestId('feedback-radio-input-3')) + }) + expect(post).toHaveBeenCalled() fireEvent.keyDown(window, { key: 'Enter', metaKey: true }) // Mac - // OR use ctrlKey: true for Windows await waitFor(() => { - expect(axiosInstance.post).toHaveBeenCalled() + expect(patch).toHaveBeenCalled() expect(mockOnFeedbackSubmit).toHaveBeenCalled() }) }) it('resets form fields after successful submission', async () => { - axiosInstance.post.mockResolvedValue({ status: 201, data: { id: 1 } }) - axiosInstance.patch.mockResolvedValue({ status: 200, data: { id: 1 } }) + post.mockResolvedValue({ + ok: true, + status: 201, + data: { id: 1 }, + error: null, + }) + patch.mockResolvedValue({ + ok: true, + status: 200, + data: { id: 1 }, + error: null, + }) render( { }) it('does not reset form on failed feedback submission', async () => { - axiosInstance.post.mockResolvedValueOnce({ status: 500 }) + const consoleSpy = jest.spyOn(console, 'error').mockImplementation(() => {}) + post.mockResolvedValue({ + ok: false, + status: 500, + data: null, + error: new Error('Server error'), + }) render( { ) fireEvent.click(screen.getByTestId('feedback-radio-input-2')) + + // Wait for rating click's async post to complete + await waitFor(() => expect(post).toHaveBeenCalled()) + fireEvent.change(screen.getByTestId('feedback-message-input'), { target: { value: 'Should not reset' }, }) @@ -424,12 +493,16 @@ describe('Feedback Form tests', () => { 'Should not reset' ) ) + consoleSpy.mockRestore() }) - it('logs error message when API returns 400', async () => { + it('logs error message when API returns error', async () => { const consoleSpy = jest.spyOn(console, 'error').mockImplementation(() => {}) - axiosInstance.post.mockRejectedValueOnce({ - response: { status: 400 }, + post.mockResolvedValue({ + ok: false, + status: 400, + data: null, + error: new Error('Bad request'), }) render( @@ -448,9 +521,14 @@ describe('Feedback Form tests', () => { consoleSpy.mockRestore() }) - it('logs an error if feedbackPost returns non-200 status', async () => { + it('logs an error if feedbackPost returns non-ok status', async () => { const consoleSpy = jest.spyOn(console, 'error').mockImplementation(() => {}) - axiosInstance.post.mockResolvedValueOnce({ status: 500 }) + post.mockResolvedValue({ + ok: false, + status: 500, + data: null, + error: new Error('Server error'), + }) render( @@ -468,9 +546,14 @@ describe('Feedback Form tests', () => { consoleSpy.mockRestore() }) - it('logs fallback error if API throws unexpected error', async () => { + it('logs error if API returns error', async () => { const consoleSpy = jest.spyOn(console, 'error').mockImplementation(() => {}) - axiosInstance.post.mockRejectedValueOnce(new Error('Network down')) + post.mockResolvedValue({ + ok: false, + status: 500, + data: null, + error: new Error('Network down'), + }) render( @@ -481,7 +564,7 @@ describe('Feedback Form tests', () => { await waitFor(() => expect(consoleSpy).toHaveBeenCalledWith( - 'An unexpected error occurred. Please try again later.', + 'Error submitting feedback:', expect.any(Object) ) ) @@ -511,8 +594,18 @@ describe('Feedback Form tests', () => { }) it('submits minimal fields when isGeneralFeedback is false', async () => { - axiosInstance.post.mockResolvedValue({ status: 201, data: { id: 1 } }) - axiosInstance.patch.mockResolvedValue({ status: 200, data: { id: 1 } }) + post.mockResolvedValue({ + ok: true, + status: 201, + data: { id: 1 }, + error: null, + }) + patch.mockResolvedValue({ + ok: true, + status: 200, + data: { id: 1 }, + error: null, + }) render( { /> ) - // Provide required rating - fireEvent.click(screen.getByTestId('feedback-radio-input-5')) + // Provide required rating and wait for async post to complete + await act(async () => { + fireEvent.click(screen.getByTestId('feedback-radio-input-5')) + }) + expect(post).toHaveBeenCalled() // Skip comment input (allowed) fireEvent.click(screen.getByRole('button', { name: /send feedback/i })) await waitFor(() => { - expect(axiosInstance.post).toHaveBeenCalledWith( - expect.any(String), - { - attachments: [], - component: 'data-file-submission', - feedback_type: 'fra_submission_feedback', - page_url: 'http://localhost/', - widget_id: 'unknown-submission-feedback', - rating: 5, - feedback: '', // comment left blank - anonymous: false, // anonymous checkbox hidden - }, - { - headers: { 'Content-Type': 'application/json' }, - withCredentials: true, - } - ) + expect(patch).toHaveBeenCalledWith(expect.any(String), { + attachments: [], + component: 'data-file-submission', + feedback_type: 'fra_submission_feedback', + page_url: 'http://localhost/', + widget_id: 'unknown-submission-feedback', + rating: 5, + feedback: '', // comment left blank + anonymous: false, // anonymous checkbox hidden + }) expect(mockOnFeedbackSubmit).toHaveBeenCalled() }) }) diff --git a/tdrs-frontend/src/components/FeedbackReports/AdminFeedbackReports.jsx b/tdrs-frontend/src/components/FeedbackReports/AdminFeedbackReports.jsx index 50b724438..8afd02960 100644 --- a/tdrs-frontend/src/components/FeedbackReports/AdminFeedbackReports.jsx +++ b/tdrs-frontend/src/components/FeedbackReports/AdminFeedbackReports.jsx @@ -1,6 +1,6 @@ import { useState, useEffect, useRef, useCallback } from 'react' import { useSearchParams } from 'react-router-dom' -import axiosInstance from '../../axios-instance' +import { get, post } from '../../fetch-instance' import createFileInputErrorState from '../../utils/createFileInputErrorState' import FeedbackReportsUpload from './FeedbackReportsUpload' import FeedbackReportsHistory from './FeedbackReportsHistory' @@ -63,25 +63,22 @@ function AdminFeedbackReports() { } setHistoryLoading(true) - try { - const response = await axiosInstance.get( - `${process.env.REACT_APP_BACKEND_URL}/reports/report-sources/`, - { - params: { year: selectedYear }, - withCredentials: true, - } - ) - setUploadHistory(response.data.results) - } catch (error) { + const { data, ok, error } = await get( + `${process.env.REACT_APP_BACKEND_URL}/reports/report-sources/`, + { params: { year: selectedYear } } + ) + + if (ok) { + setUploadHistory(data.results) + } else { console.error('Failed to fetch upload history:', error) setAlert({ active: true, type: 'error', message: 'Failed to load upload history. Please refresh the page.', }) - } finally { - setHistoryLoading(false) } + setHistoryLoading(false) }, [selectedYear]) // Fetch upload history when year changes @@ -208,16 +205,12 @@ function AdminFeedbackReports() { formData.append('year', selectedYear) formData.append('date_extracted_on', getDatePickerValue()) - try { - await axiosInstance.post( - `${process.env.REACT_APP_BACKEND_URL}/reports/report-sources/`, - formData, - { - headers: { 'Content-Type': 'multipart/form-data' }, - withCredentials: true, - } - ) + const { data, ok } = await post( + `${process.env.REACT_APP_BACKEND_URL}/reports/report-sources/`, + formData + ) + if (ok) { setAlert({ active: true, type: 'success', @@ -235,11 +228,11 @@ function AdminFeedbackReports() { // Refresh upload history fetchUploadHistory() - } catch (error) { + } else { const errorMessage = - error.response?.data?.file?.[0] || - error.response?.data?.detail || - error.response?.data?.message || + data?.file?.[0] || + data?.detail || + data?.message || 'Upload failed. Please try again.' setAlert({ @@ -247,9 +240,9 @@ function AdminFeedbackReports() { type: 'error', message: errorMessage, }) - } finally { - setLoading(false) } + + setLoading(false) } /** diff --git a/tdrs-frontend/src/components/FeedbackReports/AdminFeedbackReports.test.js b/tdrs-frontend/src/components/FeedbackReports/AdminFeedbackReports.test.js index d3fe11468..8f14f84c5 100644 --- a/tdrs-frontend/src/components/FeedbackReports/AdminFeedbackReports.test.js +++ b/tdrs-frontend/src/components/FeedbackReports/AdminFeedbackReports.test.js @@ -5,9 +5,9 @@ import { MemoryRouter } from 'react-router-dom' import configureStore from 'redux-mock-store' import { thunk } from 'redux-thunk' import AdminFeedbackReports from './AdminFeedbackReports' -import axiosInstance from '../../axios-instance' +import { get, post } from '../../fetch-instance' -jest.mock('../../axios-instance') +jest.mock('../../fetch-instance') jest.mock('../../utils/createFileInputErrorState') jest.mock('@uswds/uswds/src/js/components', () => ({ fileInput: { @@ -39,7 +39,12 @@ describe('AdminFeedbackReports', () => { jest.clearAllMocks() // Mock successful history fetch by default - axiosInstance.get.mockResolvedValue({ data: { results: [] } }) + get.mockResolvedValue({ + data: { results: [] }, + ok: true, + status: 200, + error: null, + }) // Mock FileReader for async file handling global.FileReader = jest.fn().mockImplementation(() => ({ @@ -263,15 +268,23 @@ describe('AdminFeedbackReports', () => { }) it('successfully uploads a file with date and shows success message', async () => { - axiosInstance.post.mockResolvedValue({ + post.mockResolvedValue({ data: { id: 1, status: 'PENDING', original_filename: 'FY2025.zip', }, + ok: true, + status: 200, + error: null, }) - axiosInstance.get.mockResolvedValue({ data: { results: [] } }) + get.mockResolvedValue({ + data: { results: [] }, + ok: true, + status: 200, + error: null, + }) renderComponent() @@ -298,23 +311,20 @@ describe('AdminFeedbackReports', () => { }) // Verify POST was called with year and date - expect(axiosInstance.post).toHaveBeenCalledWith( + expect(post).toHaveBeenCalledWith( expect.stringContaining('/reports/report-sources/'), - expect.any(FormData), - expect.objectContaining({ - headers: { 'Content-Type': 'multipart/form-data' }, - withCredentials: true, - }) + expect.any(FormData) ) }) it('shows error message when upload fails', async () => { - axiosInstance.post.mockRejectedValue({ - response: { - data: { - file: ['Invalid zip file structure'], - }, + post.mockResolvedValue({ + data: { + file: ['Invalid zip file structure'], }, + ok: false, + status: 400, + error: new Error('HTTP 400'), }) renderComponent() @@ -338,8 +348,14 @@ describe('AdminFeedbackReports', () => { }) it('shows loading state during upload', async () => { - axiosInstance.post.mockImplementation( - () => new Promise((resolve) => setTimeout(resolve, 100)) + post.mockImplementation( + () => + new Promise((resolve) => + setTimeout( + () => resolve({ data: null, ok: true, status: 200, error: null }), + 100 + ) + ) ) renderComponent() @@ -377,7 +393,12 @@ describe('AdminFeedbackReports', () => { }, ] - axiosInstance.get.mockResolvedValue({ data: { results: mockHistory } }) + get.mockResolvedValue({ + data: { results: mockHistory }, + ok: true, + status: 200, + error: null, + }) renderComponent() @@ -388,17 +409,21 @@ describe('AdminFeedbackReports', () => { expect(screen.getByText('02/28/2025')).toBeInTheDocument() }) - expect(axiosInstance.get).toHaveBeenCalledWith( + expect(get).toHaveBeenCalledWith( expect.stringContaining('/reports/report-sources/'), expect.objectContaining({ params: { year: '2025' }, - withCredentials: true, }) ) }) it('displays empty state when no history exists', async () => { - axiosInstance.get.mockResolvedValue({ data: { results: [] } }) + get.mockResolvedValue({ + data: { results: [] }, + ok: true, + status: 200, + error: null, + }) renderComponent() @@ -410,7 +435,12 @@ describe('AdminFeedbackReports', () => { }) it('displays error alert when history fetch fails', async () => { - axiosInstance.get.mockRejectedValue(new Error('Failed to fetch')) + get.mockResolvedValue({ + data: null, + ok: false, + status: 500, + error: new Error('Failed to fetch'), + }) renderComponent() @@ -451,15 +481,30 @@ describe('AdminFeedbackReports', () => { ] // Use mockImplementation to handle different year params - axiosInstance.get.mockImplementation((url, config) => { + get.mockImplementation((url, config) => { const year = config?.params?.year if (year === '2025') { - return Promise.resolve({ data: { results: mockHistory2025 } }) + return Promise.resolve({ + data: { results: mockHistory2025 }, + ok: true, + status: 200, + error: null, + }) } if (year === '2024') { - return Promise.resolve({ data: { results: mockHistory2024 } }) + return Promise.resolve({ + data: { results: mockHistory2024 }, + ok: true, + status: 200, + error: null, + }) } - return Promise.resolve({ data: { results: [] } }) + return Promise.resolve({ + data: { results: [] }, + ok: true, + status: 200, + error: null, + }) }) renderComponent() @@ -480,8 +525,11 @@ describe('AdminFeedbackReports', () => { }) it('refreshes history after successful upload', async () => { - axiosInstance.post.mockResolvedValue({ + post.mockResolvedValue({ data: { id: 1, status: 'PENDING' }, + ok: true, + status: 200, + error: null, }) const mockHistoryAfterUpload = [ @@ -496,9 +544,19 @@ describe('AdminFeedbackReports', () => { }, ] - axiosInstance.get - .mockResolvedValueOnce({ data: { results: [] } }) - .mockResolvedValue({ data: { results: mockHistoryAfterUpload } }) + get + .mockResolvedValueOnce({ + data: { results: [] }, + ok: true, + status: 200, + error: null, + }) + .mockResolvedValue({ + data: { results: mockHistoryAfterUpload }, + ok: true, + status: 200, + error: null, + }) renderComponent() @@ -538,7 +596,12 @@ describe('AdminFeedbackReports', () => { }, ] - axiosInstance.get.mockResolvedValue({ data: { results: mockHistory } }) + get.mockResolvedValue({ + data: { results: mockHistory }, + ok: true, + status: 200, + error: null, + }) renderComponent() @@ -565,7 +628,12 @@ describe('AdminFeedbackReports', () => { }, ] - axiosInstance.get.mockResolvedValue({ data: { results: mockHistory } }) + get.mockResolvedValue({ + data: { results: mockHistory }, + ok: true, + status: 200, + error: null, + }) renderComponent() @@ -580,8 +648,11 @@ describe('AdminFeedbackReports', () => { describe('Form Reset', () => { it('clears form after successful upload', async () => { - axiosInstance.post.mockResolvedValue({ + post.mockResolvedValue({ data: { id: 1, status: 'PENDING' }, + ok: true, + status: 200, + error: null, }) renderComponent() diff --git a/tdrs-frontend/src/components/FeedbackReports/FeedbackReportAlert.jsx b/tdrs-frontend/src/components/FeedbackReports/FeedbackReportAlert.jsx index 3e0179022..6b369aee2 100644 --- a/tdrs-frontend/src/components/FeedbackReports/FeedbackReportAlert.jsx +++ b/tdrs-frontend/src/components/FeedbackReports/FeedbackReportAlert.jsx @@ -1,5 +1,5 @@ import { useState, useEffect, useCallback } from 'react' -import axiosInstance from '../../axios-instance' +import { get } from '../../fetch-instance' import { useReportsContext } from '../Reports/ReportsContext' import closeIcon from '@uswds/uswds/img/usa-icons/close.svg' import '../../assets/feedback/Feedback.scss' @@ -40,36 +40,33 @@ const FeedbackReportAlert = () => { return } - try { - const response = await axiosInstance.get( - `${process.env.REACT_APP_BACKEND_URL}/reports/`, - { - params: { - year: yearInputValue, - quarter: quarterInputValue, - latest: 'true', - }, - withCredentials: true, - } - ) + const { data, ok, error } = await get( + `${process.env.REACT_APP_BACKEND_URL}/reports/`, + { + params: { + year: yearInputValue, + quarter: quarterInputValue, + latest: 'true', + }, + } + ) - if (response.data?.results?.length > 0) { - const report = response.data.results[0] - const dismissedTimestamp = getDismissedTimestamp(yearInputValue) + if (ok && data?.results?.length > 0) { + const report = data.results[0] + const dismissedTimestamp = getDismissedTimestamp(yearInputValue) - // Show banner if not dismissed OR if a new report is available - if (dismissedTimestamp && dismissedTimestamp === report.created_at) { - setLatestReportDate(report.created_at) - setIsDismissed(true) - } else { - setLatestReportDate(report.created_at) - setIsDismissed(false) - } + // Show banner if not dismissed OR if a new report is available + if (dismissedTimestamp && dismissedTimestamp === report.created_at) { + setLatestReportDate(report.created_at) + setIsDismissed(true) } else { - setLatestReportDate(null) + setLatestReportDate(report.created_at) setIsDismissed(false) } - } catch (error) { + } else if (ok) { + setLatestReportDate(null) + setIsDismissed(false) + } else { console.error('Error fetching feedback reports:', error) setLatestReportDate(null) setIsDismissed(false) diff --git a/tdrs-frontend/src/components/FeedbackReports/FeedbackReportAlert.test.js b/tdrs-frontend/src/components/FeedbackReports/FeedbackReportAlert.test.js index 67bacb663..e40fb2718 100644 --- a/tdrs-frontend/src/components/FeedbackReports/FeedbackReportAlert.test.js +++ b/tdrs-frontend/src/components/FeedbackReports/FeedbackReportAlert.test.js @@ -2,10 +2,10 @@ import React from 'react' import { render, screen, waitFor, fireEvent } from '@testing-library/react' import { MemoryRouter } from 'react-router-dom' import FeedbackReportAlert from './FeedbackReportAlert' -import axiosInstance from '../../axios-instance' +import { get } from '../../fetch-instance' -// Mock the axios instance -jest.mock('../../axios-instance') +// Mock the fetch instance +jest.mock('../../fetch-instance') // Mock the ReportsContext const mockUseReportsContext = jest.fn() @@ -53,7 +53,7 @@ describe('FeedbackReportAlert', () => { const { container } = renderComponent() expect(container.firstChild).toBeNull() - expect(axiosInstance.get).not.toHaveBeenCalled() + expect(get).not.toHaveBeenCalled() }) it('renders null when quarterInputValue is not set', async () => { @@ -64,7 +64,7 @@ describe('FeedbackReportAlert', () => { const { container } = renderComponent() expect(container.firstChild).toBeNull() - expect(axiosInstance.get).not.toHaveBeenCalled() + expect(get).not.toHaveBeenCalled() }) it('fetches latest report when year and quarter are set', async () => { @@ -73,16 +73,19 @@ describe('FeedbackReportAlert', () => { quarterInputValue: 'Q1', }) - axiosInstance.get.mockResolvedValue({ + get.mockResolvedValue({ data: { results: [{ created_at: '2025-12-01T00:00:00Z' }], }, + ok: true, + status: 200, + error: null, }) renderComponent() await waitFor(() => { - expect(axiosInstance.get).toHaveBeenCalledWith( + expect(get).toHaveBeenCalledWith( expect.stringContaining('/reports/'), expect.objectContaining({ params: { @@ -90,7 +93,6 @@ describe('FeedbackReportAlert', () => { quarter: 'Q1', latest: 'true', }, - withCredentials: true, }) ) }) @@ -102,10 +104,13 @@ describe('FeedbackReportAlert', () => { quarterInputValue: 'Q1', }) - axiosInstance.get.mockResolvedValue({ + get.mockResolvedValue({ data: { results: [{ created_at: '2025-12-01T00:00:00Z' }], }, + ok: true, + status: 200, + error: null, }) renderComponent() @@ -121,16 +126,19 @@ describe('FeedbackReportAlert', () => { quarterInputValue: 'Q1', }) - axiosInstance.get.mockResolvedValue({ + get.mockResolvedValue({ data: { results: [], }, + ok: true, + status: 200, + error: null, }) const { container } = renderComponent() await waitFor(() => { - expect(axiosInstance.get).toHaveBeenCalled() + expect(get).toHaveBeenCalled() }) expect(container.querySelector('.usa-alert')).toBeNull() @@ -142,12 +150,17 @@ describe('FeedbackReportAlert', () => { quarterInputValue: 'Q1', }) - axiosInstance.get.mockRejectedValue(new Error('API error')) + get.mockResolvedValue({ + data: null, + ok: false, + status: 500, + error: new Error('API error'), + }) const { container } = renderComponent() await waitFor(() => { - expect(axiosInstance.get).toHaveBeenCalled() + expect(get).toHaveBeenCalled() }) expect(container.querySelector('.usa-alert')).toBeNull() @@ -159,16 +172,28 @@ describe('FeedbackReportAlert', () => { quarterInputValue: 'Q1', }) - axiosInstance.get.mockResolvedValue({ + get.mockResolvedValue({ data: { results: [{ created_at: '2025-12-01T00:00:00Z' }], }, - }) + ok: true, + status: 200, + error: null, + }) + + const expectedDate = new Date('2025-12-01T00:00:00Z').toLocaleDateString( + 'en-US', + { + month: 'numeric', + day: 'numeric', + year: 'numeric', + } + ) renderComponent() await waitFor(() => { - expect(screen.getByText(/12\/1\/2025/)).toBeInTheDocument() + expect(screen.getByText(new RegExp(expectedDate))).toBeInTheDocument() }) }) @@ -178,10 +203,13 @@ describe('FeedbackReportAlert', () => { quarterInputValue: 'Q1', }) - axiosInstance.get.mockResolvedValue({ + get.mockResolvedValue({ data: { results: [{ created_at: '2025-12-01T00:00:00Z' }], }, + ok: true, + status: 200, + error: null, }) renderComponent() @@ -198,10 +226,13 @@ describe('FeedbackReportAlert', () => { quarterInputValue: 'Q1', }) - axiosInstance.get.mockResolvedValue({ + get.mockResolvedValue({ data: { results: [{ created_at: '2025-12-01T00:00:00Z' }], }, + ok: true, + status: 200, + error: null, }) const { container } = renderComponent() @@ -220,10 +251,13 @@ describe('FeedbackReportAlert', () => { quarterInputValue: 'Q1', }) - axiosInstance.get.mockResolvedValue({ + get.mockResolvedValue({ data: { results: [{ created_at: '2025-12-01T00:00:00Z' }], }, + ok: true, + status: 200, + error: null, }) renderComponent() @@ -241,8 +275,11 @@ describe('FeedbackReportAlert', () => { quarterInputValue: 'Q1', }) - axiosInstance.get.mockResolvedValue({ + get.mockResolvedValue({ data: { results: [{ created_at: '2025-12-01T00:00:00Z' }] }, + ok: true, + status: 200, + error: null, }) const { rerender } = render( @@ -252,7 +289,7 @@ describe('FeedbackReportAlert', () => { ) await waitFor(() => { - expect(axiosInstance.get).toHaveBeenCalledTimes(1) + expect(get).toHaveBeenCalledTimes(1) }) // Change the year @@ -268,7 +305,7 @@ describe('FeedbackReportAlert', () => { ) await waitFor(() => { - expect(axiosInstance.get).toHaveBeenCalledTimes(2) + expect(get).toHaveBeenCalledTimes(2) }) }) @@ -279,10 +316,13 @@ describe('FeedbackReportAlert', () => { quarterInputValue: 'Q1', }) - axiosInstance.get.mockResolvedValue({ + get.mockResolvedValue({ data: { results: [{ created_at: '2025-12-01T00:00:00Z' }], }, + ok: true, + status: 200, + error: null, }) renderComponent() @@ -301,10 +341,13 @@ describe('FeedbackReportAlert', () => { quarterInputValue: 'Q1', }) - axiosInstance.get.mockResolvedValue({ + get.mockResolvedValue({ data: { results: [{ created_at: '2025-12-01T00:00:00Z' }], }, + ok: true, + status: 200, + error: null, }) const { container } = renderComponent() @@ -332,10 +375,13 @@ describe('FeedbackReportAlert', () => { }) const createdAt = '2025-12-01T00:00:00Z' - axiosInstance.get.mockResolvedValue({ + get.mockResolvedValue({ data: { results: [{ created_at: createdAt }], }, + ok: true, + status: 200, + error: null, }) renderComponent() @@ -368,16 +414,19 @@ describe('FeedbackReportAlert', () => { quarterInputValue: 'Q1', }) - axiosInstance.get.mockResolvedValue({ + get.mockResolvedValue({ data: { results: [{ created_at: createdAt }], }, + ok: true, + status: 200, + error: null, }) const { container } = renderComponent() await waitFor(() => { - expect(axiosInstance.get).toHaveBeenCalled() + expect(get).toHaveBeenCalled() }) // Alert should not be visible because it was dismissed @@ -397,10 +446,13 @@ describe('FeedbackReportAlert', () => { }) // API returns a new report with different created_at - axiosInstance.get.mockResolvedValue({ + get.mockResolvedValue({ data: { results: [{ created_at: newCreatedAt }], }, + ok: true, + status: 200, + error: null, }) renderComponent() @@ -427,10 +479,13 @@ describe('FeedbackReportAlert', () => { quarterInputValue: 'Q1', }) - axiosInstance.get.mockResolvedValue({ + get.mockResolvedValue({ data: { results: [{ created_at: '2025-12-01T00:00:00Z' }], }, + ok: true, + status: 200, + error: null, }) renderComponent() @@ -449,10 +504,13 @@ describe('FeedbackReportAlert', () => { quarterInputValue: 'Q1', }) - axiosInstance.get.mockResolvedValue({ + get.mockResolvedValue({ data: { results: [{ created_at: '2025-12-01T00:00:00Z' }], }, + ok: true, + status: 200, + error: null, }) renderComponent() @@ -472,10 +530,13 @@ describe('FeedbackReportAlert', () => { quarterInputValue: 'Q1', }) - axiosInstance.get.mockResolvedValue({ + get.mockResolvedValue({ data: { results: [{ created_at: '2025-12-01T00:00:00Z' }], }, + ok: true, + status: 200, + error: null, }) const { container } = renderComponent() diff --git a/tdrs-frontend/src/components/FeedbackReports/FeedbackReports.test.js b/tdrs-frontend/src/components/FeedbackReports/FeedbackReports.test.js index e7a9d7109..34c3cf2ba 100644 --- a/tdrs-frontend/src/components/FeedbackReports/FeedbackReports.test.js +++ b/tdrs-frontend/src/components/FeedbackReports/FeedbackReports.test.js @@ -5,9 +5,9 @@ import { MemoryRouter } from 'react-router-dom' import configureStore from 'redux-mock-store' import { thunk } from 'redux-thunk' import FeedbackReports from './FeedbackReports' -import axiosInstance from '../../axios-instance' +import { get } from '../../fetch-instance' -jest.mock('../../axios-instance') +jest.mock('../../fetch-instance') jest.mock('../../utils/createFileInputErrorState') jest.mock('@uswds/uswds/src/js/components', () => ({ fileInput: { @@ -23,7 +23,12 @@ const mockStore = configureStore([thunk]) describe('FeedbackReports', () => { beforeEach(() => { jest.clearAllMocks() - axiosInstance.get.mockResolvedValue({ data: { results: [] } }) + get.mockResolvedValue({ + data: { results: [] }, + ok: true, + status: 200, + error: null, + }) // Mock FileReader for AdminFeedbackReports global.FileReader = jest.fn().mockImplementation(() => ({ @@ -244,7 +249,7 @@ describe('FeedbackReports', () => { fireEvent.change(fiscalYearSelect, { target: { value: '2025' } }) await waitFor(() => { - expect(axiosInstance.get).toHaveBeenCalledWith( + expect(get).toHaveBeenCalledWith( expect.stringContaining('/reports/report-sources/'), expect.objectContaining({ params: { year: '2025' }, @@ -279,14 +284,14 @@ describe('FeedbackReports', () => { fireEvent.change(yearSelect, { target: { value: '2025' } }) await waitFor(() => { - expect(axiosInstance.get).toHaveBeenCalledWith( + expect(get).toHaveBeenCalledWith( expect.stringContaining('/reports/'), expect.any(Object) ) }) // Make sure it's not calling the report-sources endpoint - const calls = axiosInstance.get.mock.calls + const calls = get.mock.calls const reportSourcesCall = calls.find((call) => call[0].includes('report-sources') ) diff --git a/tdrs-frontend/src/components/FeedbackReports/STTFeedbackReports.jsx b/tdrs-frontend/src/components/FeedbackReports/STTFeedbackReports.jsx index d4d8752ed..1fff9a8c5 100644 --- a/tdrs-frontend/src/components/FeedbackReports/STTFeedbackReports.jsx +++ b/tdrs-frontend/src/components/FeedbackReports/STTFeedbackReports.jsx @@ -1,7 +1,7 @@ import { useState, useEffect, useCallback } from 'react' import { useSelector } from 'react-redux' import { useSearchParams } from 'react-router-dom' -import axiosInstance from '../../axios-instance' +import { get } from '../../fetch-instance' import { Spinner } from '../Spinner' import { PaginatedComponent } from '../Paginator/Paginator' import STTFeedbackReportsTable from './STTFeedbackReportsTable' @@ -61,25 +61,22 @@ function STTFeedbackReports() { setLoading(true) setAlert({ active: false, type: null, message: null }) - try { - const response = await axiosInstance.get( - `${process.env.REACT_APP_BACKEND_URL}/reports/`, - { - params: { year: selectedYear }, - withCredentials: true, - } - ) - setReports(response.data.results || response.data || []) - } catch (error) { + const { data, ok, error } = await get( + `${process.env.REACT_APP_BACKEND_URL}/reports/`, + { params: { year: selectedYear } } + ) + + if (ok) { + setReports(data?.results || data || []) + } else { console.error('Failed to fetch feedback reports:', error) setAlert({ active: true, type: 'error', message: 'Failed to load feedback reports. Please refresh the page.', }) - } finally { - setLoading(false) } + setLoading(false) }, [selectedYear]) useEffect(() => { diff --git a/tdrs-frontend/src/components/FeedbackReports/STTFeedbackReports.test.js b/tdrs-frontend/src/components/FeedbackReports/STTFeedbackReports.test.js index cbd294b6a..460b3cce9 100644 --- a/tdrs-frontend/src/components/FeedbackReports/STTFeedbackReports.test.js +++ b/tdrs-frontend/src/components/FeedbackReports/STTFeedbackReports.test.js @@ -5,9 +5,9 @@ import { MemoryRouter } from 'react-router-dom' import configureStore from 'redux-mock-store' import { thunk } from 'redux-thunk' import STTFeedbackReports from './STTFeedbackReports' -import axiosInstance from '../../axios-instance' +import { get } from '../../fetch-instance' -jest.mock('../../axios-instance') +jest.mock('../../fetch-instance') const mockStore = configureStore([thunk]) @@ -32,7 +32,12 @@ describe('STTFeedbackReports', () => { jest.clearAllMocks() // Mock successful reports fetch by default - axiosInstance.get.mockResolvedValue({ data: { results: [] } }) + get.mockResolvedValue({ + data: { results: [] }, + ok: true, + status: 200, + error: null, + }) }) const renderComponent = () => { @@ -98,7 +103,12 @@ describe('STTFeedbackReports', () => { }) it('renders the description text with email links when year is selected', async () => { - axiosInstance.get.mockResolvedValue({ data: { results: [] } }) + get.mockResolvedValue({ + data: { results: [] }, + ok: true, + status: 200, + error: null, + }) renderComponent() @@ -116,7 +126,12 @@ describe('STTFeedbackReports', () => { }) it('renders the Knowledge Center link when year is selected', async () => { - axiosInstance.get.mockResolvedValue({ data: { results: [] } }) + get.mockResolvedValue({ + data: { results: [] }, + ok: true, + status: 200, + error: null, + }) renderComponent() @@ -132,7 +147,12 @@ describe('STTFeedbackReports', () => { }) it('renders the H2 header with STT name and fiscal year when year is selected', async () => { - axiosInstance.get.mockResolvedValue({ data: { results: [] } }) + get.mockResolvedValue({ + data: { results: [] }, + ok: true, + status: 200, + error: null, + }) renderComponent() @@ -151,7 +171,12 @@ describe('STTFeedbackReports', () => { }) it('renders the H3 heading as just "Feedback Reports" when year is selected', async () => { - axiosInstance.get.mockResolvedValue({ data: { results: [] } }) + get.mockResolvedValue({ + data: { results: [] }, + ok: true, + status: 200, + error: null, + }) renderComponent() @@ -173,12 +198,17 @@ describe('STTFeedbackReports', () => { // Wait a bit to ensure no fetch happens await waitFor(() => { - expect(axiosInstance.get).not.toHaveBeenCalled() + expect(get).not.toHaveBeenCalled() }) }) it('fetches reports when year is selected', async () => { - axiosInstance.get.mockResolvedValue({ data: { results: [] } }) + get.mockResolvedValue({ + data: { results: [] }, + ok: true, + status: 200, + error: null, + }) renderComponent() @@ -187,19 +217,30 @@ describe('STTFeedbackReports', () => { fireEvent.change(yearSelect, { target: { value: '2025' } }) await waitFor(() => { - expect(axiosInstance.get).toHaveBeenCalledWith( + expect(get).toHaveBeenCalledWith( expect.stringContaining('/reports/'), expect.objectContaining({ params: { year: 2025 }, - withCredentials: true, }) ) }) }) it('displays loading state while fetching', async () => { - axiosInstance.get.mockImplementation( - () => new Promise((resolve) => setTimeout(resolve, 100)) + get.mockImplementation( + () => + new Promise((resolve) => + setTimeout( + () => + resolve({ + data: { results: [] }, + ok: true, + status: 200, + error: null, + }), + 100 + ) + ) ) renderComponent() @@ -216,7 +257,12 @@ describe('STTFeedbackReports', () => { }) it('displays error alert when fetch fails', async () => { - axiosInstance.get.mockRejectedValue(new Error('Failed to fetch')) + get.mockResolvedValue({ + data: null, + ok: false, + status: 500, + error: new Error('Failed to fetch'), + }) renderComponent() @@ -234,7 +280,12 @@ describe('STTFeedbackReports', () => { }) it('displays empty state when no reports exist', async () => { - axiosInstance.get.mockResolvedValue({ data: { results: [] } }) + get.mockResolvedValue({ + data: { results: [] }, + ok: true, + status: 200, + error: null, + }) renderComponent() @@ -262,7 +313,12 @@ describe('STTFeedbackReports', () => { }, ] - axiosInstance.get.mockResolvedValue({ data: { results: mockReports } }) + get.mockResolvedValue({ + data: { results: mockReports }, + ok: true, + status: 200, + error: null, + }) renderComponent() @@ -300,8 +356,11 @@ describe('STTFeedbackReports', () => { ] // Mock first fetch for 2025 - axiosInstance.get.mockResolvedValueOnce({ + get.mockResolvedValueOnce({ data: { results: mock2025Reports }, + ok: true, + status: 200, + error: null, }) renderComponent() @@ -316,7 +375,7 @@ describe('STTFeedbackReports', () => { }) // Verify initial call was made with 2025 - expect(axiosInstance.get).toHaveBeenCalledWith( + expect(get).toHaveBeenCalledWith( expect.stringContaining('/reports/'), expect.objectContaining({ params: { year: 2025 }, @@ -324,8 +383,11 @@ describe('STTFeedbackReports', () => { ) // Mock the next fetch for 2024 - axiosInstance.get.mockResolvedValueOnce({ + get.mockResolvedValueOnce({ data: { results: mock2024Reports }, + ok: true, + status: 200, + error: null, }) // Change to 2024 - should automatically fetch new reports @@ -333,7 +395,7 @@ describe('STTFeedbackReports', () => { // Should fetch with new year param await waitFor(() => { - expect(axiosInstance.get).toHaveBeenCalledWith( + expect(get).toHaveBeenCalledWith( expect.stringContaining('/reports/'), expect.objectContaining({ params: { year: 2024 }, @@ -349,7 +411,12 @@ describe('STTFeedbackReports', () => { }) it('updates the H2 heading when year is changed', async () => { - axiosInstance.get.mockResolvedValue({ data: { results: [] } }) + get.mockResolvedValue({ + data: { results: [] }, + ok: true, + status: 200, + error: null, + }) renderComponent() @@ -394,7 +461,12 @@ describe('STTFeedbackReports', () => { }, ] - axiosInstance.get.mockResolvedValue({ data: { results: mockReports } }) + get.mockResolvedValue({ + data: { results: mockReports }, + ok: true, + status: 200, + error: null, + }) renderComponent() @@ -418,7 +490,12 @@ describe('STTFeedbackReports', () => { }, ] - axiosInstance.get.mockResolvedValue({ data: mockReports }) + get.mockResolvedValue({ + data: mockReports, + ok: true, + status: 200, + error: null, + }) renderComponent() @@ -432,7 +509,7 @@ describe('STTFeedbackReports', () => { }) it('handles response with null/empty data', async () => { - axiosInstance.get.mockResolvedValue({ data: null }) + get.mockResolvedValue({ data: null, ok: true, status: 200, error: null }) renderComponent() @@ -469,7 +546,12 @@ describe('STTFeedbackReports', () => { }, ] - axiosInstance.get.mockResolvedValue({ data: { results: mockReports } }) + get.mockResolvedValue({ + data: { results: mockReports }, + ok: true, + status: 200, + error: null, + }) renderComponent() @@ -496,7 +578,12 @@ describe('STTFeedbackReports', () => { } it('initializes year from URL parameter', async () => { - axiosInstance.get.mockResolvedValue({ data: { results: [] } }) + get.mockResolvedValue({ + data: { results: [] }, + ok: true, + status: 200, + error: null, + }) renderWithUrl('/feedback-reports?year=2024') @@ -507,12 +594,17 @@ describe('STTFeedbackReports', () => { }) it('fetches reports with year from URL parameter', async () => { - axiosInstance.get.mockResolvedValue({ data: { results: [] } }) + get.mockResolvedValue({ + data: { results: [] }, + ok: true, + status: 200, + error: null, + }) renderWithUrl('/feedback-reports?year=2024') await waitFor(() => { - expect(axiosInstance.get).toHaveBeenCalledWith( + expect(get).toHaveBeenCalledWith( expect.stringContaining('/reports/'), expect.objectContaining({ params: { year: 2024 }, @@ -522,7 +614,12 @@ describe('STTFeedbackReports', () => { }) it('shows placeholder for invalid year param', async () => { - axiosInstance.get.mockResolvedValue({ data: { results: [] } }) + get.mockResolvedValue({ + data: { results: [] }, + ok: true, + status: 200, + error: null, + }) renderWithUrl('/feedback-reports?year=invalid') @@ -538,7 +635,12 @@ describe('STTFeedbackReports', () => { }) it('shows placeholder for out-of-range year', async () => { - axiosInstance.get.mockResolvedValue({ data: { results: [] } }) + get.mockResolvedValue({ + data: { results: [] }, + ok: true, + status: 200, + error: null, + }) renderWithUrl('/feedback-reports?year=1999') @@ -554,7 +656,12 @@ describe('STTFeedbackReports', () => { }) it('displays H2 heading with STT name and year from URL parameter', async () => { - axiosInstance.get.mockResolvedValue({ data: { results: [] } }) + get.mockResolvedValue({ + data: { results: [] }, + ok: true, + status: 200, + error: null, + }) renderWithUrl('/feedback-reports?year=2024') diff --git a/tdrs-frontend/src/components/FeedbackReports/STTFeedbackReportsTable.jsx b/tdrs-frontend/src/components/FeedbackReports/STTFeedbackReportsTable.jsx index e6b08a579..3176328c3 100644 --- a/tdrs-frontend/src/components/FeedbackReports/STTFeedbackReportsTable.jsx +++ b/tdrs-frontend/src/components/FeedbackReports/STTFeedbackReportsTable.jsx @@ -1,5 +1,5 @@ import React, { useState } from 'react' -import axiosInstance from '../../axios-instance' +import { get } from '../../fetch-instance' import { downloadBlob } from '../../utils/fileDownload' /** @@ -41,26 +41,22 @@ function STTFeedbackReportsTable({ data, setAlert }) { setDownloadingId(report.id) setAlert({ active: false, type: null, message: null }) - try { - const response = await axiosInstance.get( - `${process.env.REACT_APP_BACKEND_URL}/reports/${report.id}/download/`, - { - responseType: 'blob', - withCredentials: true, - } - ) + const { data, ok, error } = await get( + `${process.env.REACT_APP_BACKEND_URL}/reports/${report.id}/download/`, + { responseType: 'blob' } + ) - downloadBlob(response.data, report.original_filename || 'report.zip') - } catch (error) { + if (ok) { + downloadBlob(data, report.original_filename || 'report.zip') + } else { console.error('Failed to download report:', error) setAlert({ active: true, type: 'error', message: 'Failed to download the report. Please try again.', }) - } finally { - setDownloadingId(null) } + setDownloadingId(null) } return ( diff --git a/tdrs-frontend/src/components/FeedbackReports/STTFeedbackReportsTable.test.js b/tdrs-frontend/src/components/FeedbackReports/STTFeedbackReportsTable.test.js index 660a4fbea..f1f59d251 100644 --- a/tdrs-frontend/src/components/FeedbackReports/STTFeedbackReportsTable.test.js +++ b/tdrs-frontend/src/components/FeedbackReports/STTFeedbackReportsTable.test.js @@ -1,10 +1,10 @@ import React from 'react' import { render, screen, fireEvent, waitFor } from '@testing-library/react' import STTFeedbackReportsTable from './STTFeedbackReportsTable' -import axiosInstance from '../../axios-instance' +import { get } from '../../fetch-instance' import { downloadBlob } from '../../utils/fileDownload' -jest.mock('../../axios-instance') +jest.mock('../../fetch-instance') jest.mock('../../utils/fileDownload') describe('STTFeedbackReportsTable', () => { @@ -207,7 +207,12 @@ describe('STTFeedbackReportsTable', () => { it('triggers download on click', async () => { const mockBlob = new Blob(['test content'], { type: 'application/zip' }) - axiosInstance.get.mockResolvedValue({ data: mockBlob }) + get.mockResolvedValue({ + data: mockBlob, + ok: true, + status: 200, + error: null, + }) const mockData = [ { @@ -226,11 +231,10 @@ describe('STTFeedbackReportsTable', () => { fireEvent.click(downloadButton) await waitFor(() => { - expect(axiosInstance.get).toHaveBeenCalledWith( + expect(get).toHaveBeenCalledWith( expect.stringContaining('/reports/1/download/'), expect.objectContaining({ responseType: 'blob', - withCredentials: true, }) ) }) @@ -241,8 +245,14 @@ describe('STTFeedbackReportsTable', () => { }) it('shows downloading state during download', async () => { - axiosInstance.get.mockImplementation( - () => new Promise((resolve) => setTimeout(resolve, 100)) + get.mockImplementation( + () => + new Promise((resolve) => + setTimeout( + () => resolve({ data: null, ok: true, status: 200, error: null }), + 100 + ) + ) ) const mockData = [ @@ -267,7 +277,12 @@ describe('STTFeedbackReportsTable', () => { }) it('shows error alert when download fails', async () => { - axiosInstance.get.mockRejectedValue(new Error('Download failed')) + get.mockResolvedValue({ + data: null, + ok: false, + status: 500, + error: new Error('Download failed'), + }) const mockData = [ { @@ -295,8 +310,14 @@ describe('STTFeedbackReportsTable', () => { }) it('clears alert before starting download', async () => { - axiosInstance.get.mockImplementation( - () => new Promise((resolve) => setTimeout(resolve, 100)) + get.mockImplementation( + () => + new Promise((resolve) => + setTimeout( + () => resolve({ data: null, ok: true, status: 200, error: null }), + 100 + ) + ) ) const mockData = [ @@ -324,7 +345,12 @@ describe('STTFeedbackReportsTable', () => { it('uses fallback filename "report.zip" when original_filename is missing', async () => { const mockBlob = new Blob(['test content'], { type: 'application/zip' }) - axiosInstance.get.mockResolvedValue({ data: mockBlob }) + get.mockResolvedValue({ + data: mockBlob, + ok: true, + status: 200, + error: null, + }) const mockData = [ { @@ -346,8 +372,14 @@ describe('STTFeedbackReportsTable', () => { }) it('disables button during download', async () => { - axiosInstance.get.mockImplementation( - () => new Promise((resolve) => setTimeout(resolve, 100)) + get.mockImplementation( + () => + new Promise((resolve) => + setTimeout( + () => resolve({ data: null, ok: true, status: 200, error: null }), + 100 + ) + ) ) const mockData = [ @@ -377,8 +409,20 @@ describe('STTFeedbackReportsTable', () => { describe('Multiple Downloads', () => { it('only disables the downloading button, not others', async () => { - axiosInstance.get.mockImplementation( - () => new Promise((resolve) => setTimeout(resolve, 100)) + get.mockImplementation( + () => + new Promise((resolve) => + setTimeout( + () => + resolve({ + data: null, + ok: true, + status: 200, + error: null, + }), + 100 + ) + ) ) const mockData = [ diff --git a/tdrs-frontend/src/components/FileUpload/FileUpload.jsx b/tdrs-frontend/src/components/FileUpload/FileUpload.jsx index 9bd3e2ffd..5a03f3616 100644 --- a/tdrs-frontend/src/components/FileUpload/FileUpload.jsx +++ b/tdrs-frontend/src/components/FileUpload/FileUpload.jsx @@ -116,6 +116,7 @@ function FileUpload({ fileType, label, setLocalAlertState, + setProcessingAlertState, }) { // e.g. 'Aggregate Case Data' => 'aggregate-case-data' // The set of uploaded files in our Redux state @@ -182,6 +183,13 @@ function FileUpload({ type: null, message: null, }) + if (setProcessingAlertState) { + setProcessingAlertState({ + active: false, + type: null, + message: null, + }) + } const { name: section } = event.target const file = event.target.files[0] diff --git a/tdrs-frontend/src/components/FileUploadForms/QuarterFileUploadForm.jsx b/tdrs-frontend/src/components/FileUploadForms/QuarterFileUploadForm.jsx index 3398c9594..aad0db914 100644 --- a/tdrs-frontend/src/components/FileUploadForms/QuarterFileUploadForm.jsx +++ b/tdrs-frontend/src/components/FileUploadForms/QuarterFileUploadForm.jsx @@ -69,12 +69,15 @@ const QuarterFileUploadForm = ({ stt }) => { yearInputValue, fileTypeInputValue, localAlert, + processingAlert, uploadedFiles, isSubmitting, alertRef, + processingAlertRef, onSubmit, handleCancel, setLocalAlertState, + setProcessingAlertState, } = useFileUploadForm({ stt, transformFiles, @@ -84,18 +87,44 @@ const QuarterFileUploadForm = ({ stt }) => { return ( <> + {/* Screen-reader announcer */} +
+
+ {localAlert.active ? localAlert.message : ''} +
+ +
+ {processingAlert.active ? processingAlert.message : ''} +
+
+ + {/* Visible alerts (not in accessibility tree, prevents duplicate screen reads */} {localAlert.active && (