Fix submission cleanup: recover all non-terminal states, not just Running

.env_sample

@@ -116,3 +116,7 @@ ENABLE_SIGN_IN=True
 # -----------------------------------------------------------------------------
 LOG_LEVEL=info
 SERIALIZED=false
+
+
+# Data Upload limit (used in django admin when selecing multiple items, or when adding multiple participants in the Participant Routing feature)
+#DJANGO_DATA_UPLOAD_MAX_NUMBER_FIELDS=2000

src/apps/competitions/admin.py

@@ -2,7 +2,7 @@
 from django import forms
 from django.contrib.admin.widgets import FilteredSelectMultiple
 from django.contrib.auth.models import Group
-
+from competitions.models import Submission
 from django.utils.translation import gettext_lazy as _
 import json
 import csv
@@ -166,6 +166,15 @@ def CompetitionExport_as_json(modeladmin, request, queryset):
     return HttpResponse(json.dumps(email_list), content_type="application/json")
 
 
+# This will export the email of all the selected competition creators, removing duplications and banned users
+@admin.display(description="Fail Submissions")
+def Fail_submissions(modeladmin, request, queryset):
+    for obj in queryset:
+        obj.cancel(status=Submission.FAILED)
+        obj.status_details = "Failed Manually by Site Admin action"
+        obj.save()
+
+
 class QueueFilter(InputFilter):
     # Human-readable title which will be displayed in the
     # right admin sidebar just above the filter options.
@@ -253,6 +262,19 @@ def queryset(self, request, queryset):
             return queryset.filter(phase__competition__created_by__username=value)
 
 
+class StatusDetailsFilter(InputFilter):
+    # Human-readable title which will be displayed in the
+    # right admin sidebar just above the filter options.
+    title = _("Status Details")
+    # Parameter for the filter that will be used in the URL query.
+    parameter_name = "status_details"
+
+    def queryset(self, request, queryset):
+        if self.value() is not None:
+            value = self.value()
+            return queryset.filter(status_details=value)
+
+
 class SubmissionExpansion(admin.ModelAdmin):
     # Raw Id Fields changes the field from displaying everything in a drop down menu into an id fields, which makes the page loads much faster (removes huge SELECT from the database)
     raw_id_fields = [
@@ -270,7 +292,7 @@ class SubmissionExpansion(admin.ModelAdmin):
     ]
     search_fields = ["id", "owner__username", "phase__competition__title", "task__name"]
     ordering = ('-id',)
-    actions = [SubmissionsExport_as_csv]
+    actions = [SubmissionsExport_as_csv, Fail_submissions]
     list_display = [
         "id",
         "owner",
@@ -287,6 +309,7 @@ class SubmissionExpansion(admin.ModelAdmin):
         "status",
         CompetitionOrganizerFilter,
         QueueFilter,
+        StatusDetailsFilter,
     ]
     fieldsets = [
         (

src/apps/competitions/tasks.py

@@ -923,15 +923,86 @@ def update_phase_statuses():
 
 @app.task(queue="site-worker")
 def submission_status_cleanup():
+    # Recover submissions stuck in any non-terminal state
+    # There are multiple special cases that we have to deal with:
+    # - Submission stuck in Submitted, Preparing, Running or Scoring because of a network error or a problem on the compute worker (simple case this code aims to fix)
+    # - RabbitMQ failure (usually means submissions are stuck in Submitting)
+    # - Competitions with Auto Run-Submissions disabled (submissions appear "stuck" in Submitted until an organizer runs them manually)
+    # - Submission that appear to be stuck in Submitted, Preparing, Running, Scoring but are not for multiple reasons :
+    #       - Lack of Compute workers in a queue, the submission is not stuck but waiting its turn
+    #       - Submission file is heavy and/or the compute worker has slow network speeds
+    #       - Combination of few compute workers + long submission execution time (allowed by a high execution time limit in the competition settings)
+    non_terminal_statuses = [
+        Submission.SUBMITTING,
+        Submission.SUBMITTED,
+        Submission.PREPARING,
+        Submission.RUNNING,
+        Submission.SCORING,
+    ]
     submissions = Submission.objects.filter(
-        status=Submission.RUNNING, has_children=False
-    ).select_related("phase", "parent")
+        status__in=non_terminal_statuses,
+        has_children=False,
+    ).select_related('phase', 'parent')
 
     for sub in submissions:
-        if sub.started_when < now() - timedelta(
-            milliseconds=(3600000 * 24) + sub.phase.execution_time_limit
-        ):
-            if sub.parent is not None:
-                sub.parent.cancel(status=Submission.FAILED)
+        # Use started_when for Running submissions, created_when as fallback for others
+        # The deadline waits for 30 minutes after the phase execution time limit before failing submissions (making sure big submissions have time to upload)
+        if sub.started_when is not None:
+            reference_time = sub.started_when
+            deadline = reference_time + timedelta(
+                milliseconds=(60000 * 30) + sub.phase.execution_time_limit
+            )
+
+            if now() > deadline:
+                if sub.parent is not None:
+                    sub.parent.cancel(status=Submission.FAILED)
+                    sub_edit = Submission.objects.get(id=sub.id)
+                    sub_edit.status_details = "Stuck submission was automatically failed"
+                    sub_edit.save()
+                else:
+                    sub.cancel(status=Submission.FAILED)
+                    sub_edit = Submission.objects.get(id=sub.id)
+                    sub_edit.status_details = "Stuck submission was automatically failed"
+                    sub_edit.save()
+        # The first if will filter out Preparing, Running and Scoring submissions. Only Submitting and Submitted are left to deal with
+        else:
+            # If a submission is stuck in SUBMITTING for 10 minutes after its creation
+            # then Rabbit might have failed, in which case the submission is unrecoverable
+            if sub.status == "SUBMITTING":
+                reference_time = sub.created_when
+                deadline = reference_time + timedelta(
+                    milliseconds=(60000 * 10)
+                )
+
+                if now() > deadline:
+                    if sub.parent is not None:
+                        sub.parent.cancel(status=Submission.FAILED)
+                        sub_edit = Submission.objects.get(id=sub.id)
+                        sub_edit.status_details = "Stuck submission was automatically failed (Check RabbitMQ health)"
+                        sub_edit.save()
+                    else:
+                        sub.cancel(status=Submission.FAILED)
+                        sub_edit = Submission.objects.get(id=sub.id)
+                        sub_edit.status_details = "Stuck submission was automatically failed (Check RabbitMQ health)"
+                        sub_edit.save()
             else:
-                sub.cancel(status=Submission.FAILED)
+                # Special case where the competition has Auto Run Submission set to True.
+                # Otherwise, we might fail submissions if the organizer is not fast enough to start the submission
+                # TODO: Better logic to not Fail submissions that are stuck because of low Queue capacity
+                if sub.phase.competition.auto_run_submissions:
+                    reference_time = sub.created_when
+                    deadline = reference_time + timedelta(
+                        milliseconds=(60000 * 30) + sub.phase.execution_time_limit
+                    )
+
+                    if now() > deadline:
+                        if sub.parent is not None:
+                            sub.parent.cancel(status=Submission.FAILED)
+                            sub_edit = Submission.objects.get(id=sub.id)
+                            sub_edit.status_details = "Stuck submission was automatically failed"
+                            sub_edit.save()
+                        else:
+                            sub.cancel(status=Submission.FAILED)
+                            sub_edit = Submission.objects.get(id=sub.id)
+                            sub_edit.status_details = "Stuck submission was automatically failed"
+                            sub_edit.save()

src/apps/competitions/tests/test_submissions.py

@@ -427,6 +427,51 @@ def test_submissions_are_cancelled_if_running_24_hours_past_execution_time_limit
         assert self.submission_pass.status == Submission.RUNNING
         assert self.submission_fail.status == Submission.FAILED
 
+    def test_cleanup_recovers_stuck_submitted_submissions(self):
+        """Submissions stuck in Submitted should be recovered by cleanup."""
+        sub = self.make_submission()
+        sub.status = Submission.SUBMITTED
+        sub.created_when = timezone.now() - timedelta(hours=48)
+        sub.save(ignore_submission_limit=True)
+
+        submission_status_cleanup()
+        sub.refresh_from_db()
+        assert sub.status == Submission.FAILED
+
+    def test_cleanup_recovers_stuck_preparing_submissions(self):
+        """Submissions stuck in Preparing should be recovered by cleanup."""
+        sub = self.make_submission()
+        sub.status = Submission.PREPARING
+        sub.created_when = timezone.now() - timedelta(hours=48)
+        sub.save(ignore_submission_limit=True)
+
+        submission_status_cleanup()
+        sub.refresh_from_db()
+        assert sub.status == Submission.FAILED
+
+    def test_cleanup_recovers_stuck_scoring_submissions(self):
+        """Submissions stuck in Scoring should be recovered by cleanup."""
+        sub = self.make_submission()
+        sub.status = Submission.SCORING
+        sub.created_when = timezone.now() - timedelta(hours=48)
+        sub.save(ignore_submission_limit=True)
+
+        submission_status_cleanup()
+        sub.refresh_from_db()
+        assert sub.status == Submission.FAILED
+
+    def test_cleanup_does_not_touch_recent_non_terminal_submissions(self):
+        """Recent submissions in non-terminal states should NOT be cleaned up."""
+        for status in [Submission.SUBMITTED, Submission.PREPARING, Submission.SCORING]:
+            sub = self.make_submission()
+            sub.status = status
+            sub.created_when = timezone.now()
+            sub.save(ignore_submission_limit=True)
+
+            submission_status_cleanup()
+            sub.refresh_from_db()
+            assert sub.status == status, f"Recent {status} submission should not be cleaned up"
+
     def test_cancelling_parent_submission_cancels_all_children(self):
         self.parent_submission = self.make_submission()
         self.parent_submission.has_children = True

src/settings/base.py

@@ -257,7 +257,7 @@
     },
     'submission_status_cleanup': {
         'task': 'competitions.tasks.submission_status_cleanup',
-        'schedule': timedelta(seconds=3600)
+        'schedule': timedelta(seconds=7200)
     },
     'create_storage_analytics_snapshot': {
         'task': 'analytics.tasks.create_storage_analytics_snapshot',
@@ -584,3 +584,6 @@ def setup_celery_logging(**kwargs):
 # =============================================================================
 ENABLE_SIGN_UP = os.environ.get('ENABLE_SIGN_UP', 'True').lower() == 'true'
 ENABLE_SIGN_IN = os.environ.get('ENABLE_SIGN_IN', 'True').lower() == 'true'
+
+# Data Upload limit (used in django admin when selecing multiple items, or when adding multiple participants in the Participant Routing feature)
+DATA_UPLOAD_MAX_NUMBER_FIELDS = int(os.environ.get('DJANGO_DATA_UPLOAD_MAX_NUMBER_FIELDS', 2000))