Package updater task

.env.local.example

@@ -7,4 +7,9 @@ POSTGRES_DB=djangopackages
 POSTGRES_PASSWORD=djangopackages
 POSTGRES_USER=djangopackages
 POSTGRES_HOST=postgres
-POSTGRES_PORT=5432
+POSTGRES_PORT=5432
+CELERY_FLOWER_USER=flower_user
+CELERY_FLOWER_PASSWORD=flower_pwd
+MAILGUN_API_KEY=mailgun_api_key
+MAILGUN_SENDER_DOMAIN=mailgun_sender_domain
+SECRET_KEY=secret_key

.github/workflows/actions.yml

@@ -48,6 +48,7 @@ jobs:
           DEBUG: false
           REDIS_URL: 'redis://redis:6379/0'
           SECRET_KEY: 'this-is-for-testing-only'
+          CELERY_BROKER_URL: 'redis://redis:6379/0'
 
         run: |
           python -m pytest .

compose/django/Dockerfile

@@ -14,6 +14,18 @@ COPY ./compose/django/entrypoint.sh /entrypoint.sh
 RUN sed -i 's/\r//' /entrypoint.sh
 RUN chmod +x /entrypoint.sh
 
+COPY ./compose/django/celery/worker/start.sh /start-celeryworker.sh
+RUN sed -i 's/\r//' /start-celeryworker.sh
+RUN chmod +x /start-celeryworker.sh
+
+COPY ./compose/django/celery/beat/start.sh /start-celerybeat.sh
+RUN sed -i 's/\r//' /start-celerybeat.sh
+RUN chmod +x /start-celerybeat.sh
+
+COPY ./compose/django/celery/flower/start.sh /start-flower.sh
+RUN sed -i 's/\r//' /start-flower.sh
+RUN chmod +x /start-flower.sh
+
 COPY . /app
 RUN chown -R django /app
 

compose/django/Dockerfile-dev

@@ -17,6 +17,18 @@ COPY ./compose/django/start-dev.sh /start-dev.sh
 RUN sed -i 's/\r//' /start-dev.sh
 RUN chmod +x /start-dev.sh
 
+COPY ./compose/django/celery/worker/start.sh /start-celeryworker.sh
+RUN sed -i 's/\r//' /start-celeryworker.sh
+RUN chmod +x /start-celeryworker.sh
+
+COPY ./compose/django/celery/beat/start.sh /start-celerybeat.sh
+RUN sed -i 's/\r//' /start-celerybeat.sh
+RUN chmod +x /start-celerybeat.sh
+
+COPY ./compose/django/celery/flower/start.sh /start-flower.sh
+RUN sed -i 's/\r//' /start-flower.sh
+RUN chmod +x /start-flower.sh
+
 
 WORKDIR /app
 

compose/django/celery/beat/start.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -o errexit
+set -o nounset
+
+
+rm -f './celerybeat.pid'
+celery -A settings.celery_app beat -l INFO

compose/django/celery/flower/start.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -o errexit
+set -o nounset
+
+
+celery \
+    -A settings.celery_app \
+    -b "${CELERY_BROKER_URL}" \
+    flower \
+    --basic_auth="${CELERY_FLOWER_USER}:${CELERY_FLOWER_PASSWORD}"

compose/django/celery/worker/start.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -o errexit
+set -o nounset
+
+
+# watchgod celery.__main__.main --args -A celery_app worker -l INFO
+celery -A settings.celery_app worker -l INFO

compose/django/entrypoint.sh

@@ -24,4 +24,5 @@ done
 
 export REDIS_URL=redis://redis:6379/0
 export DATABASE_URL=postgres://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST:$POSTGRES_PORT/$POSTGRES_DB
+export CELERY_BROKER_URL="${REDIS_URL}"
 exec $cmd

dev.yml

@@ -1,7 +1,7 @@
 version: '3.6'
 
 services:
-  django:
+  django: &django
     build:
       context: .
       dockerfile: ./compose/django/Dockerfile-dev
@@ -29,6 +29,34 @@ services:
   redis:
     build: ./compose/redis
 
+  celeryworker:
+    <<: *django
+    image: django_packages_dev_celeryworker
+    container_name: celeryworker
+    depends_on:
+      - redis
+      - postgres
+    ports: []
+    command: /start-celeryworker.sh
+
+  celerybeat:
+    <<: *django
+    image: django_packages_dev_celerybeat
+    container_name: celerybeat
+    depends_on:
+      - redis
+      - postgres
+    ports: []
+    command: /start-celerybeat.sh
+
+  flower:
+    <<: *django
+    image: django_packages_dev_flower
+    container_name: flower
+    ports:
+      - "5555:5555"
+    command: /start-flower.sh
+
 volumes:
   postgres_backup_dev: {}
   postgres_data_dev: {}

package/exceptions.py

@@ -0,0 +1,9 @@
+import logging
+
+class PackageUpdaterException(Exception):
+    def __init__(self, error, title):
+        log_message = "For {title}, {error_type}: {error}".format(
+            title=title, error_type=type(error), error=error
+        )
+        logging.critical(log_message)
+        logging.exception(error)

package/management/commands/package_updater.py

@@ -7,20 +7,12 @@
 from github3 import login as github_login
 
 from package.models import Package
+from package.tasks import update_package_task
 from core.utils import healthcheck
 
 logger = logging.getLogger(__name__)
 
 
-class PackageUpdaterException(Exception):
-    def __init__(self, error, title):
-        log_message = "For {title}, {error_type}: {error}".format(
-            title=title, error_type=type(error), error=error
-        )
-        logging.critical(log_message)
-        logging.exception(error)
-
-
 class Command(BaseCommand):
 
     help = "Updates all the packages in the system. Commands belongs to django-packages.package"
@@ -39,17 +31,7 @@ def handle(self, *args, **options):
                     sleep(120)
                 break
 
-            try:
-                try:
-                    package.fetch_metadata(fetch_pypi=False)
-                    package.fetch_commits()
-                except Exception as e:
-                    logger.error(
-                        f"Error while fetching package details for {package.title}."
-                    )
-                    raise PackageUpdaterException(e, package.title)
-            except PackageUpdaterException:
-                logger.error(f"Unable to update {package.title}", exc_info=True)
+            update_package_task.delay(package.id)
 
             print(f"{__file__}::handle::sleep(5)")
             sleep(5)

package/tasks.py

@@ -0,0 +1,26 @@
+from settings import celery_app
+from celery.utils.log import get_task_logger
+
+from .exceptions import PackageUpdaterException
+from .models import Package
+
+
+logger = get_task_logger(__name__)
+
+
+@celery_app.task()
+def update_package_task(package_id):
+    package = Package.objects.get(id=package_id)
+    logger.info(f'Start updating package {package.id}')
+
+    try:
+        try:
+            package.fetch_metadata(fetch_pypi=False)
+            package.fetch_commits()
+        except Exception as e:
+            logger.error(
+                f"Error while fetching package details for {package.title}."
+            )
+            raise PackageUpdaterException(e, package.title)
+    except PackageUpdaterException:
+        logger.error(f"Unable to update {package.title}", exc_info=True)

requirements.in

@@ -48,3 +48,8 @@ pytest-django
 # utils
 bumpver
 six
+
+# celery
+celery==5.1.2
+django-celery-beat==2.2.1
+flower==1.0.0