diff --git a/.env.local.example b/.env.local.example index d0a30f21f..3093aa430 100644 --- a/.env.local.example +++ b/.env.local.example @@ -7,4 +7,9 @@ POSTGRES_DB=djangopackages POSTGRES_PASSWORD=djangopackages POSTGRES_USER=djangopackages POSTGRES_HOST=postgres -POSTGRES_PORT=5432 \ No newline at end of file +POSTGRES_PORT=5432 +CELERY_FLOWER_USER=flower_user +CELERY_FLOWER_PASSWORD=flower_pwd +MAILGUN_API_KEY=mailgun_api_key +MAILGUN_SENDER_DOMAIN=mailgun_sender_domain +SECRET_KEY=secret_key diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index 8f3991e41..a3e5c5626 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -48,6 +48,7 @@ jobs: DEBUG: false REDIS_URL: 'redis://redis:6379/0' SECRET_KEY: 'this-is-for-testing-only' + CELERY_BROKER_URL: 'redis://redis:6379/0' run: | python -m pytest . diff --git a/compose/django/Dockerfile b/compose/django/Dockerfile index 1b27da47c..1d54725ed 100644 --- a/compose/django/Dockerfile +++ b/compose/django/Dockerfile @@ -14,6 +14,18 @@ COPY ./compose/django/entrypoint.sh /entrypoint.sh RUN sed -i 's/\r//' /entrypoint.sh RUN chmod +x /entrypoint.sh +COPY ./compose/django/celery/worker/start.sh /start-celeryworker.sh +RUN sed -i 's/\r//' /start-celeryworker.sh +RUN chmod +x /start-celeryworker.sh + +COPY ./compose/django/celery/beat/start.sh /start-celerybeat.sh +RUN sed -i 's/\r//' /start-celerybeat.sh +RUN chmod +x /start-celerybeat.sh + +COPY ./compose/django/celery/flower/start.sh /start-flower.sh +RUN sed -i 's/\r//' /start-flower.sh +RUN chmod +x /start-flower.sh + COPY . /app RUN chown -R django /app diff --git a/compose/django/Dockerfile-dev b/compose/django/Dockerfile-dev index 93a825810..f88b0e7c6 100644 --- a/compose/django/Dockerfile-dev +++ b/compose/django/Dockerfile-dev @@ -17,6 +17,18 @@ COPY ./compose/django/start-dev.sh /start-dev.sh RUN sed -i 's/\r//' /start-dev.sh RUN chmod +x /start-dev.sh +COPY ./compose/django/celery/worker/start.sh /start-celeryworker.sh +RUN sed -i 's/\r//' /start-celeryworker.sh +RUN chmod +x /start-celeryworker.sh + +COPY ./compose/django/celery/beat/start.sh /start-celerybeat.sh +RUN sed -i 's/\r//' /start-celerybeat.sh +RUN chmod +x /start-celerybeat.sh + +COPY ./compose/django/celery/flower/start.sh /start-flower.sh +RUN sed -i 's/\r//' /start-flower.sh +RUN chmod +x /start-flower.sh + WORKDIR /app diff --git a/compose/django/celery/beat/start.sh b/compose/django/celery/beat/start.sh new file mode 100644 index 000000000..bcf40c612 --- /dev/null +++ b/compose/django/celery/beat/start.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +set -o errexit +set -o nounset + + +rm -f './celerybeat.pid' +celery -A settings.celery_app beat -l INFO diff --git a/compose/django/celery/flower/start.sh b/compose/django/celery/flower/start.sh new file mode 100644 index 000000000..da7fd548c --- /dev/null +++ b/compose/django/celery/flower/start.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -o errexit +set -o nounset + + +celery \ + -A settings.celery_app \ + -b "${CELERY_BROKER_URL}" \ + flower \ + --basic_auth="${CELERY_FLOWER_USER}:${CELERY_FLOWER_PASSWORD}" diff --git a/compose/django/celery/worker/start.sh b/compose/django/celery/worker/start.sh new file mode 100644 index 000000000..703def09d --- /dev/null +++ b/compose/django/celery/worker/start.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +set -o errexit +set -o nounset + + +# watchgod celery.__main__.main --args -A celery_app worker -l INFO +celery -A settings.celery_app worker -l INFO diff --git a/compose/django/entrypoint.sh b/compose/django/entrypoint.sh index aef43d31a..e065aae3d 100644 --- a/compose/django/entrypoint.sh +++ b/compose/django/entrypoint.sh @@ -24,4 +24,5 @@ done export REDIS_URL=redis://redis:6379/0 export DATABASE_URL=postgres://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST:$POSTGRES_PORT/$POSTGRES_DB +export CELERY_BROKER_URL="${REDIS_URL}" exec $cmd diff --git a/dev.yml b/dev.yml index f9e8fba45..d8830cc65 100644 --- a/dev.yml +++ b/dev.yml @@ -1,7 +1,7 @@ version: '3.6' services: - django: + django: &django build: context: . dockerfile: ./compose/django/Dockerfile-dev @@ -29,6 +29,34 @@ services: redis: build: ./compose/redis + celeryworker: + <<: *django + image: django_packages_dev_celeryworker + container_name: celeryworker + depends_on: + - redis + - postgres + ports: [] + command: /start-celeryworker.sh + + celerybeat: + <<: *django + image: django_packages_dev_celerybeat + container_name: celerybeat + depends_on: + - redis + - postgres + ports: [] + command: /start-celerybeat.sh + + flower: + <<: *django + image: django_packages_dev_flower + container_name: flower + ports: + - "5555:5555" + command: /start-flower.sh + volumes: postgres_backup_dev: {} postgres_data_dev: {} diff --git a/package/exceptions.py b/package/exceptions.py new file mode 100644 index 000000000..fe753c0b5 --- /dev/null +++ b/package/exceptions.py @@ -0,0 +1,9 @@ +import logging + +class PackageUpdaterException(Exception): + def __init__(self, error, title): + log_message = "For {title}, {error_type}: {error}".format( + title=title, error_type=type(error), error=error + ) + logging.critical(log_message) + logging.exception(error) \ No newline at end of file diff --git a/package/management/commands/package_updater.py b/package/management/commands/package_updater.py index fce3b9cd3..e2e156a39 100644 --- a/package/management/commands/package_updater.py +++ b/package/management/commands/package_updater.py @@ -7,20 +7,12 @@ from github3 import login as github_login from package.models import Package +from package.tasks import update_package_task from core.utils import healthcheck logger = logging.getLogger(__name__) -class PackageUpdaterException(Exception): - def __init__(self, error, title): - log_message = "For {title}, {error_type}: {error}".format( - title=title, error_type=type(error), error=error - ) - logging.critical(log_message) - logging.exception(error) - - class Command(BaseCommand): help = "Updates all the packages in the system. Commands belongs to django-packages.package" @@ -39,17 +31,7 @@ def handle(self, *args, **options): sleep(120) break - try: - try: - package.fetch_metadata(fetch_pypi=False) - package.fetch_commits() - except Exception as e: - logger.error( - f"Error while fetching package details for {package.title}." - ) - raise PackageUpdaterException(e, package.title) - except PackageUpdaterException: - logger.error(f"Unable to update {package.title}", exc_info=True) + update_package_task.delay(package.id) print(f"{__file__}::handle::sleep(5)") sleep(5) diff --git a/package/tasks.py b/package/tasks.py new file mode 100644 index 000000000..1ab14078e --- /dev/null +++ b/package/tasks.py @@ -0,0 +1,26 @@ +from settings import celery_app +from celery.utils.log import get_task_logger + +from .exceptions import PackageUpdaterException +from .models import Package + + +logger = get_task_logger(__name__) + + +@celery_app.task() +def update_package_task(package_id): + package = Package.objects.get(id=package_id) + logger.info(f'Start updating package {package.id}') + + try: + try: + package.fetch_metadata(fetch_pypi=False) + package.fetch_commits() + except Exception as e: + logger.error( + f"Error while fetching package details for {package.title}." + ) + raise PackageUpdaterException(e, package.title) + except PackageUpdaterException: + logger.error(f"Unable to update {package.title}", exc_info=True) diff --git a/requirements.in b/requirements.in index 7b3e00728..286f6172c 100644 --- a/requirements.in +++ b/requirements.in @@ -48,3 +48,8 @@ pytest-django # utils bumpver six + +# celery +celery==5.1.2 +django-celery-beat==2.2.1 +flower==1.0.0 diff --git a/requirements.txt b/requirements.txt index b6bc29165..8d6467c0b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,15 +1,24 @@ # -# This file is autogenerated by pip-compile +# This file is autogenerated by pip-compile with python 3.9 # To update, run: # -# pip-compile --output-file=./requirements.txt ./requirements.in +# pip-compile --output-file=requirements.txt requirements.in # +amqp==5.0.6 + # via kombu asgiref==3.4.1 # via django attrs==21.2.0 # via pytest +billiard==3.6.4.0 + # via celery bumpver==2021.1113 - # via -r ./requirements.in + # via -r requirements.in +celery==5.1.2 + # via + # -r requirements.in + # django-celery-beat + # flower certifi==2021.5.30 # via # requests @@ -20,10 +29,20 @@ cffi==1.14.6 # pynacl charset-normalizer==2.0.6 # via requests -click==8.0.1 +click==7.1.2 # via # bumpver + # celery + # click-didyoumean + # click-plugins + # click-repl # django-click +click-didyoumean==0.0.3 + # via celery +click-plugins==1.1.1 + # via celery +click-repl==0.2.0 + # via celery colorama==0.4.4 # via bumpver coverage==5.5 @@ -37,64 +56,71 @@ defusedxml==0.7.1 deprecated==1.2.13 # via pygithub dj-pagination==2.5.0 - # via -r ./requirements.in + # via -r requirements.in +django==3.2.7 + # via + # -r requirements.in + # django-anymail + # django-celery-beat + # django-debug-toolbar + # django-extensions + # django-redis + # django-reversion + # django-secure + # django-structlog + # django-timezone-field + # djangorestframework + # webstack-django-sorting django-anymail==8.4 - # via -r ./requirements.in + # via -r requirements.in +django-celery-beat==2.2.1 + # via -r requirements.in django-click==2.3.0 - # via -r ./requirements.in + # via -r requirements.in django-crispy-forms==1.13.0 - # via -r ./requirements.in + # via -r requirements.in django-debug-toolbar==3.2.2 - # via -r ./requirements.in + # via -r requirements.in django-environ==0.7.0 - # via -r ./requirements.in + # via -r requirements.in django-extensions==3.1.3 - # via -r ./requirements.in + # via -r requirements.in django-ipware==4.0.0 # via django-structlog django-jsonview==2.0.0 - # via -r ./requirements.in + # via -r requirements.in django-redis==5.0.0 - # via -r ./requirements.in + # via -r requirements.in django-reversion==4.0.0 - # via -r ./requirements.in + # via -r requirements.in django-secure==1.0.2 - # via -r ./requirements.in + # via -r requirements.in django-structlog==2.1.2 - # via -r ./requirements.in + # via -r requirements.in django-test-plus==2.2.0 - # via -r ./requirements.in + # via -r requirements.in +django-timezone-field==4.2.1 + # via django-celery-beat django-waffle==2.2.1 - # via -r ./requirements.in -django==3.2.7 - # via - # -r ./requirements.in - # django-anymail - # django-debug-toolbar - # django-extensions - # django-redis - # django-reversion - # django-secure - # django-structlog - # djangorestframework - # webstack-django-sorting + # via -r requirements.in djangorestframework==3.12.4 - # via -r ./requirements.in + # via -r requirements.in feedparser==6.0.8 - # via -r ./requirements.in + # via -r requirements.in +flower==1.0.0 + # via -r requirements.in github3.py==0.9.6 - # via -r ./requirements.in + # via -r requirements.in +humanize==3.11.0 + # via flower idna==3.2 # via requests -importlib-metadata==4.8.1 - # via - # click - # pluggy - # pytest iniconfig==1.1.1 # via pytest jinja2==3.0.1 - # via -r ./requirements.in + # via -r requirements.in +kombu==5.1.0 + # via celery lexid==2021.1006 # via bumpver markupsafe==2.0.1 @@ -105,20 +131,24 @@ oauthlib==3.1.1 # social-auth-core packaging==21.0 # via - # -r ./requirements.in + # -r requirements.in # pytest pathlib2==2.3.6 # via bumpver pluggy==1.0.0 # via pytest +prometheus-client==0.11.0 + # via flower +prompt-toolkit==3.0.20 + # via click-repl psycopg2-binary==2.9.1 - # via -r ./requirements.in + # via -r requirements.in py==1.10.0 # via pytest pycparser==2.20 # via cffi pygithub==1.55 - # via -r ./requirements.in + # via -r requirements.in pyjwt==2.1.0 # via # pygithub @@ -127,36 +157,38 @@ pynacl==1.4.0 # via pygithub pyparsing==2.4.7 # via packaging -pytest-cov==2.12.1 - # via -r ./requirements.in -pytest-django==4.4.0 - # via -r ./requirements.in pytest==6.2.5 # via - # -r ./requirements.in + # -r requirements.in # pytest-cov # pytest-django +pytest-cov==2.12.1 + # via -r requirements.in +pytest-django==4.4.0 + # via -r requirements.in +python-crontab==2.5.1 + # via django-celery-beat python-dateutil==2.8.2 - # via -r ./requirements.in + # via + # -r requirements.in + # python-crontab python-gitlab==2.10.1 - # via -r ./requirements.in + # via -r requirements.in python3-openid==3.2.0 # via social-auth-core pytz==2021.1 - # via django + # via + # celery + # django + # django-timezone-field + # flower redis==3.5.3 # via - # -r ./requirements.in + # -r requirements.in # django-redis -requests-mock==1.9.3 - # via -r ./requirements.in -requests-oauthlib==1.3.0 - # via social-auth-core -requests-toolbelt==0.9.1 - # via python-gitlab requests==2.26.0 # via - # -r ./requirements.in + # -r requirements.in # django-anymail # github3.py # pygithub @@ -165,22 +197,29 @@ requests==2.26.0 # requests-oauthlib # requests-toolbelt # social-auth-core +requests-mock==1.9.3 + # via -r requirements.in +requests-oauthlib==1.3.0 + # via social-auth-core +requests-toolbelt==0.9.1 + # via python-gitlab sentry-sdk==1.4.1 - # via -r ./requirements.in + # via -r requirements.in sgmllib3k==1.0.0 # via feedparser six==1.16.0 # via - # -r ./requirements.in + # -r requirements.in + # click-repl # pathlib2 # pynacl # python-dateutil # requests-mock social-auth-app-django==5.0.0 - # via -r ./requirements.in + # via -r requirements.in social-auth-core==4.1.0 # via - # -r ./requirements.in + # -r requirements.in # social-auth-app-django sqlparse==0.4.2 # via @@ -188,38 +227,40 @@ sqlparse==0.4.2 # django-debug-toolbar structlog==21.1.0 # via - # -r ./requirements.in + # -r requirements.in # django-structlog toml==0.10.2 # via # bumpver # pytest # pytest-cov +tornado==6.1 + # via flower trove-classifiers==2021.9.23 - # via -r ./requirements.in -typing-extensions==3.10.0.2 - # via - # asgiref - # importlib-metadata - # structlog -uritemplate.py==3.0.2 - # via github3.py + # via -r requirements.in uritemplate==3.0.1 # via uritemplate.py +uritemplate.py==3.0.2 + # via github3.py urllib3==1.26.7 # via # requests # sentry-sdk uwsgi==2.0.19.1 - # via -r ./requirements.in + # via -r requirements.in +vine==5.0.0 + # via + # amqp + # celery + # kombu +wcwidth==0.2.5 + # via prompt-toolkit webstack-django-sorting==1.0.2 - # via -r ./requirements.in + # via -r requirements.in whitenoise==5.3.0 - # via -r ./requirements.in + # via -r requirements.in wrapt==1.12.1 # via deprecated -zipp==3.5.0 - # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/settings/__init__.py b/settings/__init__.py index e69de29bb..53f4ccb1d 100644 --- a/settings/__init__.py +++ b/settings/__init__.py @@ -0,0 +1,3 @@ +from .celery import app as celery_app + +__all__ = ("celery_app",) diff --git a/settings/base.py b/settings/base.py index 0fae89de7..7f2edcca6 100644 --- a/settings/base.py +++ b/settings/base.py @@ -154,6 +154,7 @@ "social_django", "waffle", "webstack_django_sorting", + "django_celery_beat", ] INSTALLED_APPS = PREREQ_APPS + PROJECT_APPS @@ -344,3 +345,12 @@ } WAFFLE_CREATE_MISSING_SWITCHES = True + + +# Celery +CELERY_BROKER_URL = env("CELERY_BROKER_URL") +CELERY_RESULT_BACKEND = CELERY_BROKER_URL +CELERY_ACCEPT_CONTENT = ["json"] +CELERY_TASK_SERIALIZER = "json" +CELERY_RESULT_SERIALIZER = "json" +CELERY_BEAT_SCHEDULER = "django_celery_beat.schedulers:DatabaseScheduler" diff --git a/settings/celery.py b/settings/celery.py new file mode 100644 index 000000000..5c962eea0 --- /dev/null +++ b/settings/celery.py @@ -0,0 +1,17 @@ +import os + +from celery import Celery + +# set the default Django settings module for the 'celery' program. +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings.docker") + +app = Celery("django_packages") + +# Using a string here means the worker doesn't have to serialize +# the configuration object to child processes. +# - namespace='CELERY' means all celery-related configuration keys +# should have a `CELERY_` prefix. +app.config_from_object("django.conf:settings", namespace="CELERY") + +# Load task modules from all registered Django app configs. +app.autodiscover_tasks()