From e0b5adb698251d858f59e41f82f17496330ced12 Mon Sep 17 00:00:00 2001 From: Mahi Date: Sat, 24 Jan 2026 17:17:26 +0530 Subject: [PATCH] Refactor sponsorship email to send asynchronously via Celery --- Dockerfile | 8 +-- celerybeat-schedule | Bin 0 -> 12288 bytes celerybeat-schedule-shm | Bin 0 -> 32768 bytes celerybeat-schedule-wal | Bin 0 -> 82432 bytes compose.yml | 55 +++++++++++++++-- portal/__init__.py | 3 + portal/celery.py | 11 ++++ portal/settings.py | 27 ++++++++- requirements-app.txt | 4 +- sponsorship/signals.py | 88 ++++----------------------- sponsorship/tasks.py | 129 ++++++++++++++++++++++++++++++++++++++++ 11 files changed, 233 insertions(+), 92 deletions(-) create mode 100644 celerybeat-schedule create mode 100644 celerybeat-schedule-shm create mode 100644 celerybeat-schedule-wal create mode 100644 portal/celery.py create mode 100644 sponsorship/tasks.py diff --git a/Dockerfile b/Dockerfile index 933ce9d8..6431650a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,11 +19,11 @@ RUN apt-get update && apt-get install -y gettext ############################################################################### FROM base AS dev -ARG USER_ID -ARG GROUP_ID +ARG USER_ID=1000 +ARG GROUP_ID=1000 -RUN groupadd -o -g $GROUP_ID -r usergrp -RUN useradd -o -m -u $USER_ID -g $GROUP_ID user +RUN groupadd -r usergrp -g $GROUP_ID && \ + useradd -r -u $USER_ID -g usergrp user RUN chown user /code COPY requirements-dev.txt /code/ diff --git a/celerybeat-schedule b/celerybeat-schedule new file mode 100644 index 0000000000000000000000000000000000000000..933d58706a7b2421693d29082150104c96cc4546 GIT binary patch literal 12288 zcmeI$O-lkn7zgl~9kn-NB(nA8GI*&V79%@#Da}K$7}a`SOea`0B^{%Lz-}En+O-eS z*9hs{N9a_?j?I`0-K0wp{2!Rzo#%P>?RT3wIdTIf#aX*^UR7d?#EE5*T_Fe|mL5!x zFsQzWgcE%>^J!rbr$!g~C$V^g@CJXGb_41VfB*y_009U<00Izz00bZafqyP=N@wgu zg5IvG>S7mbTs6(D7c3OQ7IoBRZ60K=!tU0bvMS< z%WAVHzw2Q^=hs)Qn7xyrBxu#;RrjK)*Y8^vO9?044rx15#b}k zKlq5h@z=lnJG=`55P$##AOHafKmY;|fB*y_0D)f!*puD`mZp=%eTK6v{jqb9jIlUX MecPme>=C5C0D>%E&Hw-a literal 0 HcmV?d00001 diff --git a/celerybeat-schedule-shm b/celerybeat-schedule-shm new file mode 100644 index 0000000000000000000000000000000000000000..c9977afbd2db902c0c3d4becc3f2f6cf8a5a8ec7 GIT binary patch literal 32768 zcmeI)!A(Lz5C-5M1QbLbCSDVfHW&Zl`^s!jHP4K#Rb009C72oNAZfB*pk1PBlyK!5-N0t5&U qAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&&LEs2|2q@10 literal 0 HcmV?d00001 diff --git a/celerybeat-schedule-wal b/celerybeat-schedule-wal new file mode 100644 index 0000000000000000000000000000000000000000..3b29d5d97842bb6f02eb02bd57d86b661b6eff3d GIT binary patch literal 82432 zcmeI*Uq};i0KoC-bw)+~%&f`$S5qU14j(iL3&Ki^{t#FRMORR8ya2$8r-RE~N9$$XH zyyJ1TmKE=kES7kS@oI1K*lO~N8v75Pf4!!$)z}mpxcyA=m8aeLT^r`=8)t|Xi?}IT zM8D`X_Thy90tg_000IagfB*srAbxXV?|QCe21?f1y_iq$H`XS=IC za>GpDshpB_?pWtl>?x8xM^&pdr>o+yLsj)XW{vD2=PEf$wpS6!l5LUZa62nBmq$M4 zRrDPs<2~}c;8Q_!o%7-8v$OS$Genm~42UlAQM~ef5q}Us009ILKmY**5I_I{1Q0+V z(gdu&6PhF0VtoJfCkx3Q6YxYKDNc%yaaUW7%>mC2QuHH#=>;~GEbMi>EcE(&fuUMJ z+>EqK=6n!9009ILKmY**5I_I{1Q3XRfuOxW>SXo;9c{JtZ7rMU(F;WX-Nh9mfB*sr zAb*AHrSt;-&~3u0B7gt_2q1s}0tg_000Id7-vS}@ z0%^nZ3z+r-x#h>3PHdH5_2q1s}0tg_000IagfWSBeg7gCEzpY0QYA;Z=vL|t% z@6z*tUcgw9fb|H*F)SQ{00IagfB*srAbo+yLsdO;!z>@lDLGfkQL?>?Fq>bXui$=l=KO|}^a9a% zKXFY6AbB?=C*37Z~N`69NbzfB*srAb @@ -49,6 +52,45 @@ services: postgres: condition: service_healthy + celery: + image: pyladiescon-portal-web:docker-compose + command: celery -A portal worker --loglevel=info + volumes: + - .:/code + environment: + DEBUG: "1" + SECRET_KEY: verysecure + DATABASE_URL: postgresql://pyladiescon:pyladiescon@postgres:5432/pyladiescon + CELERY_BROKER_URL: redis://redis:6379/0 + CELERY_RESULT_BACKEND: redis://redis:6379/0 + DJANGO_DEFAULT_FROM_EMAIL: PyLadiesCon + DJANGO_EMAIL_HOST: maildev + DJANGO_EMAIL_PORT: 1025 + DJANGO_ALLOWED_HOSTS: "localhost,127.0.0.1,[::1]" + depends_on: + redis: + condition: service_healthy + postgres: + condition: service_healthy + + celery-beat: + image: pyladiescon-portal-web:docker-compose + command: celery -A portal beat --loglevel=info + volumes: + - .:/code + environment: + DEBUG: "1" + SECRET_KEY: verysecure + DATABASE_URL: postgresql://pyladiescon:pyladiescon@postgres:5432/pyladiescon + CELERY_BROKER_URL: redis://redis:6379/0 + CELERY_RESULT_BACKEND: redis://redis:6379/0 + DJANGO_ALLOWED_HOSTS: "localhost,127.0.0.1,[::1]" + depends_on: + redis: + condition: service_healthy + postgres: + condition: service_healthy + maildev: image: maildev/maildev:2.2.1 ports: @@ -57,3 +99,4 @@ services: volumes: pgdata: + redisdata: diff --git a/portal/__init__.py b/portal/__init__.py index e69de29b..9e0d95fd 100644 --- a/portal/__init__.py +++ b/portal/__init__.py @@ -0,0 +1,3 @@ +from .celery import app as celery_app + +__all__ = ('celery_app',) \ No newline at end of file diff --git a/portal/celery.py b/portal/celery.py new file mode 100644 index 00000000..ae77df9d --- /dev/null +++ b/portal/celery.py @@ -0,0 +1,11 @@ +import os +from celery import Celery + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'portal.settings') +app = Celery('portal') +app.config_from_object('django.conf:settings', namespace='CELERY') +app.autodiscover_tasks() + +@app.task(bind=True, ignore_result=True) +def debug_task(self): + print(f'Request: {self.request!r}') \ No newline at end of file diff --git a/portal/settings.py b/portal/settings.py index 0df1f09f..2f224303 100644 --- a/portal/settings.py +++ b/portal/settings.py @@ -52,9 +52,9 @@ ), ], ) -ALLOWED_HOSTS = os.environ.get("DJANGO_ALLOWED_HOSTS") -if ALLOWED_HOSTS: - ALLOWED_HOSTS = ALLOWED_HOSTS.split(",") +ALLOWED_HOSTS = os.environ.get("DJANGO_ALLOWED_HOSTS", "localhost") +ALLOWED_HOSTS = [host.strip() for host in ALLOWED_HOSTS.split(",")] + # Application definition @@ -335,3 +335,24 @@ PRETIX_API_TOKEN = os.getenv("PRETIX_API_TOKEN") PRETIX_WEBHOOK_SECRET = os.getenv("PRETIX_WEBHOOK_SECRET") + + +CELERY_BROKER_URL = os.environ.get('CELERY_BROKER_URL', 'redis://redis:6379/0') +CELERY_RESULT_BACKEND = os.environ.get('CELERY_RESULT_BACKEND', 'redis://redis:6379/0') + +CELERY_ACCEPT_CONTENT = ['json'] +CELERY_TASK_SERIALIZER = 'json' +CELERY_RESULT_SERIALIZER = 'json' +CELERY_TIMEZONE = 'UTC' + +CELERY_TASK_ROUTES = { + 'sponsorship.tasks.*': {'queue': 'emails'}, + 'volunteer.tasks.*': {'queue': 'emails'}, +} + +CELERY_TASK_TRACK_STARTED = True +CELERY_TASK_TIME_LIMIT = 30 * 60 +CELERY_TASK_SOFT_TIME_LIMIT = 25 * 60 + +CELERY_WORKER_LOG_FORMAT = '[%(asctime)s: %(levelname)s/%(processName)s] %(message)s' +CELERY_WORKER_TASK_LOG_FORMAT = '[%(asctime)s: %(levelname)s/%(processName)s][%(task_name)s(%(task_id)s)] %(message)s' \ No newline at end of file diff --git a/requirements-app.txt b/requirements-app.txt index e74f9e78..96e355ac 100644 --- a/requirements-app.txt +++ b/requirements-app.txt @@ -18,4 +18,6 @@ django-import-export[all]==4.3.12 markdown==3.7 bleach==6.3.0 sentry-sdk[django]==2.43.0 -requests==2.32.5 \ No newline at end of file +requests==2.32.5 +celery==5.4.0 +redis==5.2.1 \ No newline at end of file diff --git a/sponsorship/signals.py b/sponsorship/signals.py index 8dd2e0c4..f504ccf3 100644 --- a/sponsorship/signals.py +++ b/sponsorship/signals.py @@ -5,83 +5,11 @@ from django.dispatch import receiver from common.send_emails import send_email -from volunteer.models import RoleTypes, VolunteerProfile +from volunteer.constants import RoleTypes +from volunteer.models import VolunteerProfile from .models import SponsorshipProfile - -def _send_internal_email( - subject, - *, - markdown_template, - context=None, -): - """Helper function to send an internal email. - - Lookup who the internal team members who should receive the email and then send the emails individually. - Send the email to staff, admin, and sponsorship team members - - Only supports Markdown templates going forward. - """ - - recipients = User.objects.filter( - Q( - id__in=VolunteerProfile.objects.prefetch_related("roles") - .filter(roles__short_name__in=[RoleTypes.ADMIN, RoleTypes.STAFF]) - .values_list("id", flat=True) - ) - | Q(is_superuser=True) - | Q(is_staff=True) - ).distinct() - - if not recipients.exists(): - return - - # send each email individually to each recipient, for privacy reasons - for recipient in recipients: - context["recipient_name"] = recipient.get_full_name() or recipient.username - - send_email( - subject, - [recipient.email], - markdown_template=markdown_template, - context=context, - ) - - -def send_internal_sponsor_onboarding_email(instance): - """Send email to team whenever a new sponsor is created. - - Emails will be sent to team members with the role type Staff or Admin, and to sponsorship team. - Emails will also be sent to users with is_superuser or is_staff set to True. - """ - context = {"profile": instance} - subject = f"{settings.ACCOUNT_EMAIL_SUBJECT_PREFIX} New Sponsorship Tracking: {instance.organization_name}" - - markdown_template = "emails/sponsorship/internal_sponsor_onboarding.md" - - _send_internal_email( - subject, - markdown_template=markdown_template, - context=context, - ) - - -def send_internal_sponsor_progress_update_email(instance): - """Send email to team whenever there is a change in sponsorship progress.""" - - context = {"profile": instance} - subject = f"{settings.ACCOUNT_EMAIL_SUBJECT_PREFIX} Update in Sponsorship Tracking for {instance.organization_name}" - - markdown_template = "emails/sponsorship/internal_sponsor_updated.md" - - _send_internal_email( - subject, - markdown_template=markdown_template, - context=context, - ) - - @receiver(post_save, sender=SponsorshipProfile) def sponsorship_profile_signal(sender, instance, created, **kwargs): """Send emails when sponsorship profile is created or updated. @@ -89,8 +17,12 @@ def sponsorship_profile_signal(sender, instance, created, **kwargs): """ if hasattr(instance, "from_import_export"): return + from sponsorship.tasks import ( + send_internal_sponsor_onboarding_email_task, + send_internal_sponsor_progress_update_email_task + ) + if created: + send_internal_sponsor_onboarding_email_task.delay(instance.id) else: - if created: - send_internal_sponsor_onboarding_email(instance) - else: - send_internal_sponsor_progress_update_email(instance) + send_internal_sponsor_progress_update_email_task.delay(instance.id) + diff --git a/sponsorship/tasks.py b/sponsorship/tasks.py new file mode 100644 index 00000000..82ff5f7b --- /dev/null +++ b/sponsorship/tasks.py @@ -0,0 +1,129 @@ +from celery import shared_task +from django.conf import settings +from django.contrib.auth.models import User +from django.db.models import Q + +from common.send_emails import send_email +from volunteer.constants import RoleTypes +from volunteer.models import VolunteerProfile + + +@shared_task( + name='sponsorship.tasks.send_internal_sponsor_onboarding_email', + bind=True, + max_retries=3, + default_retry_delay=60 # Retry after 60 seconds +) +def send_internal_sponsor_onboarding_email_task(self, profile_id): + """ + Background task to send internal sponsor onboarding email. + + Args: + profile_id: ID of the SponsorshipProfile instance + """ + from sponsorship.models import SponsorshipProfile + + try: + profile = SponsorshipProfile.objects.get(id=profile_id) + + # Get recipients (same logic as before) + recipients = User.objects.filter( + Q( + id__in=VolunteerProfile.objects.prefetch_related("roles") + .filter(roles__short_name__in=[RoleTypes.ADMIN, RoleTypes.STAFF]) + .values_list("id", flat=True) + ) + | Q(is_superuser=True) + | Q(is_staff=True) + ).distinct() + + if not recipients.exists(): + return f"No recipients found for profile {profile_id}" + + subject = f"{settings.ACCOUNT_EMAIL_SUBJECT_PREFIX} New Sponsorship Tracking: {profile.organization_name}" + markdown_template = "emails/sponsorship/internal_sponsor_onboarding.md" + + # Send emails to each recipient + emails_sent = 0 + for recipient in recipients: + context = { + 'profile': profile, + 'recipient_name': recipient.get_full_name() or recipient.username + } + + send_email( + subject, + [recipient.email], + markdown_template=markdown_template, + context=context, + ) + emails_sent += 1 + + return f"Successfully sent {emails_sent} emails for sponsorship {profile.organization_name}" + + except SponsorshipProfile.DoesNotExist: + # Don't retry if profile doesn't exist + return f"SponsorshipProfile {profile_id} not found" + + except Exception as exc: + # Retry on other exceptions + raise self.retry(exc=exc) + + +@shared_task( + name='sponsorship.tasks.send_internal_sponsor_progress_update_email', + bind=True, + max_retries=3, + default_retry_delay=60 +) +def send_internal_sponsor_progress_update_email_task(self, profile_id): + """ + Background task to send internal sponsor progress update email. + + Args: + profile_id: ID of the SponsorshipProfile instance + """ + from sponsorship.models import SponsorshipProfile + + try: + profile = SponsorshipProfile.objects.get(id=profile_id) + + # Get recipients + recipients = User.objects.filter( + Q( + id__in=VolunteerProfile.objects.prefetch_related("roles") + .filter(roles__short_name__in=[RoleTypes.ADMIN, RoleTypes.STAFF]) + .values_list("id", flat=True) + ) + | Q(is_superuser=True) + | Q(is_staff=True) + ).distinct() + + if not recipients.exists(): + return f"No recipients found for profile {profile_id}" + + subject = f"{settings.ACCOUNT_EMAIL_SUBJECT_PREFIX} Update in Sponsorship Tracking for {profile.organization_name}" + markdown_template = "emails/sponsorship/internal_sponsor_updated.md" + + emails_sent = 0 + for recipient in recipients: + context = { + 'profile': profile, + 'recipient_name': recipient.get_full_name() or recipient.username + } + + send_email( + subject, + [recipient.email], + markdown_template=markdown_template, + context=context, + ) + emails_sent += 1 + + return f"Successfully sent {emails_sent} update emails for sponsorship {profile.organization_name}" + + except SponsorshipProfile.DoesNotExist: + return f"SponsorshipProfile {profile_id} not found" + + except Exception as exc: + raise self.retry(exc=exc) \ No newline at end of file