From fd125748161a8dffb9603e53c311fb0fcbab6b5b Mon Sep 17 00:00:00 2001 From: Nicolle Cysneiros Date: Fri, 24 Sep 2021 18:19:42 -0300 Subject: [PATCH 1/5] Configs celery in the project --- .env.local.example | 7 +++- compose/django/Dockerfile | 12 ++++++ compose/django/Dockerfile-dev | 12 ++++++ compose/django/celery/beat/start.sh | 8 ++++ compose/django/celery/flower/start.sh | 11 ++++++ compose/django/celery/worker/start.sh | 8 ++++ compose/django/entrypoint.sh | 1 + dev.yml | 30 +++++++++++++- requirements.in | 5 +++ requirements.txt | 56 +++++++++++++++++++++++---- settings/__init__.py | 3 ++ settings/base.py | 10 +++++ settings/celery.py | 17 ++++++++ 13 files changed, 171 insertions(+), 9 deletions(-) create mode 100644 compose/django/celery/beat/start.sh create mode 100644 compose/django/celery/flower/start.sh create mode 100644 compose/django/celery/worker/start.sh create mode 100644 settings/celery.py diff --git a/.env.local.example b/.env.local.example index d0a30f21f..3093aa430 100644 --- a/.env.local.example +++ b/.env.local.example @@ -7,4 +7,9 @@ POSTGRES_DB=djangopackages POSTGRES_PASSWORD=djangopackages POSTGRES_USER=djangopackages POSTGRES_HOST=postgres -POSTGRES_PORT=5432 \ No newline at end of file +POSTGRES_PORT=5432 +CELERY_FLOWER_USER=flower_user +CELERY_FLOWER_PASSWORD=flower_pwd +MAILGUN_API_KEY=mailgun_api_key +MAILGUN_SENDER_DOMAIN=mailgun_sender_domain +SECRET_KEY=secret_key diff --git a/compose/django/Dockerfile b/compose/django/Dockerfile index 1b27da47c..1d54725ed 100644 --- a/compose/django/Dockerfile +++ b/compose/django/Dockerfile @@ -14,6 +14,18 @@ COPY ./compose/django/entrypoint.sh /entrypoint.sh RUN sed -i 's/\r//' /entrypoint.sh RUN chmod +x /entrypoint.sh +COPY ./compose/django/celery/worker/start.sh /start-celeryworker.sh +RUN sed -i 's/\r//' /start-celeryworker.sh +RUN chmod +x /start-celeryworker.sh + +COPY ./compose/django/celery/beat/start.sh /start-celerybeat.sh +RUN sed -i 's/\r//' /start-celerybeat.sh +RUN chmod +x /start-celerybeat.sh + +COPY ./compose/django/celery/flower/start.sh /start-flower.sh +RUN sed -i 's/\r//' /start-flower.sh +RUN chmod +x /start-flower.sh + COPY . /app RUN chown -R django /app diff --git a/compose/django/Dockerfile-dev b/compose/django/Dockerfile-dev index 93a825810..f88b0e7c6 100644 --- a/compose/django/Dockerfile-dev +++ b/compose/django/Dockerfile-dev @@ -17,6 +17,18 @@ COPY ./compose/django/start-dev.sh /start-dev.sh RUN sed -i 's/\r//' /start-dev.sh RUN chmod +x /start-dev.sh +COPY ./compose/django/celery/worker/start.sh /start-celeryworker.sh +RUN sed -i 's/\r//' /start-celeryworker.sh +RUN chmod +x /start-celeryworker.sh + +COPY ./compose/django/celery/beat/start.sh /start-celerybeat.sh +RUN sed -i 's/\r//' /start-celerybeat.sh +RUN chmod +x /start-celerybeat.sh + +COPY ./compose/django/celery/flower/start.sh /start-flower.sh +RUN sed -i 's/\r//' /start-flower.sh +RUN chmod +x /start-flower.sh + WORKDIR /app diff --git a/compose/django/celery/beat/start.sh b/compose/django/celery/beat/start.sh new file mode 100644 index 000000000..bcf40c612 --- /dev/null +++ b/compose/django/celery/beat/start.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +set -o errexit +set -o nounset + + +rm -f './celerybeat.pid' +celery -A settings.celery_app beat -l INFO diff --git a/compose/django/celery/flower/start.sh b/compose/django/celery/flower/start.sh new file mode 100644 index 000000000..da7fd548c --- /dev/null +++ b/compose/django/celery/flower/start.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -o errexit +set -o nounset + + +celery \ + -A settings.celery_app \ + -b "${CELERY_BROKER_URL}" \ + flower \ + --basic_auth="${CELERY_FLOWER_USER}:${CELERY_FLOWER_PASSWORD}" diff --git a/compose/django/celery/worker/start.sh b/compose/django/celery/worker/start.sh new file mode 100644 index 000000000..703def09d --- /dev/null +++ b/compose/django/celery/worker/start.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +set -o errexit +set -o nounset + + +# watchgod celery.__main__.main --args -A celery_app worker -l INFO +celery -A settings.celery_app worker -l INFO diff --git a/compose/django/entrypoint.sh b/compose/django/entrypoint.sh index aef43d31a..e065aae3d 100644 --- a/compose/django/entrypoint.sh +++ b/compose/django/entrypoint.sh @@ -24,4 +24,5 @@ done export REDIS_URL=redis://redis:6379/0 export DATABASE_URL=postgres://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST:$POSTGRES_PORT/$POSTGRES_DB +export CELERY_BROKER_URL="${REDIS_URL}" exec $cmd diff --git a/dev.yml b/dev.yml index f9e8fba45..d8830cc65 100644 --- a/dev.yml +++ b/dev.yml @@ -1,7 +1,7 @@ version: '3.6' services: - django: + django: &django build: context: . dockerfile: ./compose/django/Dockerfile-dev @@ -29,6 +29,34 @@ services: redis: build: ./compose/redis + celeryworker: + <<: *django + image: django_packages_dev_celeryworker + container_name: celeryworker + depends_on: + - redis + - postgres + ports: [] + command: /start-celeryworker.sh + + celerybeat: + <<: *django + image: django_packages_dev_celerybeat + container_name: celerybeat + depends_on: + - redis + - postgres + ports: [] + command: /start-celerybeat.sh + + flower: + <<: *django + image: django_packages_dev_flower + container_name: flower + ports: + - "5555:5555" + command: /start-flower.sh + volumes: postgres_backup_dev: {} postgres_data_dev: {} diff --git a/requirements.in b/requirements.in index 7b3e00728..286f6172c 100644 --- a/requirements.in +++ b/requirements.in @@ -48,3 +48,8 @@ pytest-django # utils bumpver six + +# celery +celery==5.1.2 +django-celery-beat==2.2.1 +flower==1.0.0 diff --git a/requirements.txt b/requirements.txt index b6bc29165..7f4017d2e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,12 +4,21 @@ # # pip-compile --output-file=./requirements.txt ./requirements.in # +amqp==5.0.6 + # via kombu asgiref==3.4.1 # via django attrs==21.2.0 # via pytest +billiard==3.6.4.0 + # via celery bumpver==2021.1113 # via -r ./requirements.in +celery==5.1.2 + # via + # -r ./requirements.in + # django-celery-beat + # flower certifi==2021.5.30 # via # requests @@ -24,6 +33,12 @@ click==8.0.1 # via # bumpver # django-click +click-didyoumean==0.0.3 + # via celery +click-plugins==1.1.1 + # via celery +click-repl==0.2.0 + # via celery colorama==0.4.4 # via bumpver coverage==5.5 @@ -64,37 +79,42 @@ django-structlog==2.1.2 # via -r ./requirements.in django-test-plus==2.2.0 # via -r ./requirements.in +django-timezone-field==4.2.1 + # via django-celery-beat django-waffle==2.2.1 # via -r ./requirements.in django==3.2.7 # via # -r ./requirements.in # django-anymail + # django-celery-beat # django-debug-toolbar # django-extensions # django-redis # django-reversion # django-secure # django-structlog + # django-timezone-field # djangorestframework # webstack-django-sorting djangorestframework==3.12.4 # via -r ./requirements.in feedparser==6.0.8 # via -r ./requirements.in +flower==1.0.0 + # via -r ./requirements.in github3.py==0.9.6 # via -r ./requirements.in +humanize==3.11.0 + # via flower idna==3.2 # via requests -importlib-metadata==4.8.1 - # via - # click - # pluggy - # pytest iniconfig==1.1.1 # via pytest jinja2==3.0.1 # via -r ./requirements.in +kombu==5.1.0 + # via celery lexid==2021.1006 # via bumpver markupsafe==2.0.1 @@ -111,6 +131,10 @@ pathlib2==2.3.6 # via bumpver pluggy==1.0.0 # via pytest +prometheus-client==0.11.0 + # via flower +prompt-toolkit==3.0.20 + # via click-repl psycopg2-binary==2.9.1 # via -r ./requirements.in py==1.10.0 @@ -136,14 +160,22 @@ pytest==6.2.5 # -r ./requirements.in # pytest-cov # pytest-django +python-crontab==2.5.1 + # via django-celery-beat python-dateutil==2.8.2 - # via -r ./requirements.in + # via + # -r ./requirements.in + # python-crontab python-gitlab==2.10.1 # via -r ./requirements.in python3-openid==3.2.0 # via social-auth-core pytz==2021.1 - # via django + # via + # celery + # django + # django-timezone-field + # flower redis==3.5.3 # via # -r ./requirements.in @@ -172,6 +204,7 @@ sgmllib3k==1.0.0 six==1.16.0 # via # -r ./requirements.in + # click-repl # pathlib2 # pynacl # python-dateutil @@ -195,6 +228,8 @@ toml==0.10.2 # bumpver # pytest # pytest-cov +tornado==6.1 + # via flower trove-classifiers==2021.9.23 # via -r ./requirements.in typing-extensions==3.10.0.2 @@ -212,6 +247,13 @@ urllib3==1.26.7 # sentry-sdk uwsgi==2.0.19.1 # via -r ./requirements.in +vine==5.0.0 + # via + # amqp + # celery + # kombu +wcwidth==0.2.5 + # via prompt-toolkit webstack-django-sorting==1.0.2 # via -r ./requirements.in whitenoise==5.3.0 diff --git a/settings/__init__.py b/settings/__init__.py index e69de29bb..53f4ccb1d 100644 --- a/settings/__init__.py +++ b/settings/__init__.py @@ -0,0 +1,3 @@ +from .celery import app as celery_app + +__all__ = ("celery_app",) diff --git a/settings/base.py b/settings/base.py index 0fae89de7..7f2edcca6 100644 --- a/settings/base.py +++ b/settings/base.py @@ -154,6 +154,7 @@ "social_django", "waffle", "webstack_django_sorting", + "django_celery_beat", ] INSTALLED_APPS = PREREQ_APPS + PROJECT_APPS @@ -344,3 +345,12 @@ } WAFFLE_CREATE_MISSING_SWITCHES = True + + +# Celery +CELERY_BROKER_URL = env("CELERY_BROKER_URL") +CELERY_RESULT_BACKEND = CELERY_BROKER_URL +CELERY_ACCEPT_CONTENT = ["json"] +CELERY_TASK_SERIALIZER = "json" +CELERY_RESULT_SERIALIZER = "json" +CELERY_BEAT_SCHEDULER = "django_celery_beat.schedulers:DatabaseScheduler" diff --git a/settings/celery.py b/settings/celery.py new file mode 100644 index 000000000..5c962eea0 --- /dev/null +++ b/settings/celery.py @@ -0,0 +1,17 @@ +import os + +from celery import Celery + +# set the default Django settings module for the 'celery' program. +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings.docker") + +app = Celery("django_packages") + +# Using a string here means the worker doesn't have to serialize +# the configuration object to child processes. +# - namespace='CELERY' means all celery-related configuration keys +# should have a `CELERY_` prefix. +app.config_from_object("django.conf:settings", namespace="CELERY") + +# Load task modules from all registered Django app configs. +app.autodiscover_tasks() From 57ec69b783913c3b6a44268cb0bea487b00c6ff3 Mon Sep 17 00:00:00 2001 From: Nicolle Cysneiros Date: Fri, 24 Sep 2021 18:20:27 -0300 Subject: [PATCH 2/5] [WIP] Creates task to update single package --- package/management/commands/package_updater.py | 2 ++ package/tasks.py | 14 ++++++++++++++ 2 files changed, 16 insertions(+) create mode 100644 package/tasks.py diff --git a/package/management/commands/package_updater.py b/package/management/commands/package_updater.py index fce3b9cd3..1e2007c81 100644 --- a/package/management/commands/package_updater.py +++ b/package/management/commands/package_updater.py @@ -7,6 +7,7 @@ from github3 import login as github_login from package.models import Package +from package.tasks import update_package_task from core.utils import healthcheck logger = logging.getLogger(__name__) @@ -41,6 +42,7 @@ def handle(self, *args, **options): try: try: + update_package_task.delay(package.id) package.fetch_metadata(fetch_pypi=False) package.fetch_commits() except Exception as e: diff --git a/package/tasks.py b/package/tasks.py new file mode 100644 index 000000000..50bca5b25 --- /dev/null +++ b/package/tasks.py @@ -0,0 +1,14 @@ +from settings import celery_app +from celery.utils.log import get_task_logger + +from .models import Package + + +logger = get_task_logger(__name__) + + +@celery_app.task() +def update_package_task(package_id): + package = Package.objects.get(id=package_id) + logger.info(f'Start updating package {package.id}') + From 969c8406ec403894d49e768545d18f220f930e11 Mon Sep 17 00:00:00 2001 From: Nicolle Cysneiros Date: Fri, 29 Oct 2021 15:31:42 -0300 Subject: [PATCH 3/5] Adds Celery broker URL on github actions env --- .github/workflows/actions.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index 8f3991e41..a3e5c5626 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -48,6 +48,7 @@ jobs: DEBUG: false REDIS_URL: 'redis://redis:6379/0' SECRET_KEY: 'this-is-for-testing-only' + CELERY_BROKER_URL: 'redis://redis:6379/0' run: | python -m pytest . From dfc2e7ae0273e88692132647d31c14b390c1aeee Mon Sep 17 00:00:00 2001 From: Nicolle Cysneiros Date: Fri, 29 Oct 2021 15:45:08 -0300 Subject: [PATCH 4/5] Updates requirements --- requirements.txt | 147 +++++++++++++++++++++++------------------------ 1 file changed, 73 insertions(+), 74 deletions(-) diff --git a/requirements.txt b/requirements.txt index 7f4017d2e..8d6467c0b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ # -# This file is autogenerated by pip-compile +# This file is autogenerated by pip-compile with python 3.9 # To update, run: # -# pip-compile --output-file=./requirements.txt ./requirements.in +# pip-compile --output-file=requirements.txt requirements.in # amqp==5.0.6 # via kombu @@ -13,10 +13,10 @@ attrs==21.2.0 billiard==3.6.4.0 # via celery bumpver==2021.1113 - # via -r ./requirements.in + # via -r requirements.in celery==5.1.2 # via - # -r ./requirements.in + # -r requirements.in # django-celery-beat # flower certifi==2021.5.30 @@ -29,9 +29,13 @@ cffi==1.14.6 # pynacl charset-normalizer==2.0.6 # via requests -click==8.0.1 +click==7.1.2 # via # bumpver + # celery + # click-didyoumean + # click-plugins + # click-repl # django-click click-didyoumean==0.0.3 # via celery @@ -52,59 +56,61 @@ defusedxml==0.7.1 deprecated==1.2.13 # via pygithub dj-pagination==2.5.0 - # via -r ./requirements.in + # via -r requirements.in +django==3.2.7 + # via + # -r requirements.in + # django-anymail + # django-celery-beat + # django-debug-toolbar + # django-extensions + # django-redis + # django-reversion + # django-secure + # django-structlog + # django-timezone-field + # djangorestframework + # webstack-django-sorting django-anymail==8.4 - # via -r ./requirements.in + # via -r requirements.in +django-celery-beat==2.2.1 + # via -r requirements.in django-click==2.3.0 - # via -r ./requirements.in + # via -r requirements.in django-crispy-forms==1.13.0 - # via -r ./requirements.in + # via -r requirements.in django-debug-toolbar==3.2.2 - # via -r ./requirements.in + # via -r requirements.in django-environ==0.7.0 - # via -r ./requirements.in + # via -r requirements.in django-extensions==3.1.3 - # via -r ./requirements.in + # via -r requirements.in django-ipware==4.0.0 # via django-structlog django-jsonview==2.0.0 - # via -r ./requirements.in + # via -r requirements.in django-redis==5.0.0 - # via -r ./requirements.in + # via -r requirements.in django-reversion==4.0.0 - # via -r ./requirements.in + # via -r requirements.in django-secure==1.0.2 - # via -r ./requirements.in + # via -r requirements.in django-structlog==2.1.2 - # via -r ./requirements.in + # via -r requirements.in django-test-plus==2.2.0 - # via -r ./requirements.in + # via -r requirements.in django-timezone-field==4.2.1 # via django-celery-beat django-waffle==2.2.1 - # via -r ./requirements.in -django==3.2.7 - # via - # -r ./requirements.in - # django-anymail - # django-celery-beat - # django-debug-toolbar - # django-extensions - # django-redis - # django-reversion - # django-secure - # django-structlog - # django-timezone-field - # djangorestframework - # webstack-django-sorting + # via -r requirements.in djangorestframework==3.12.4 - # via -r ./requirements.in + # via -r requirements.in feedparser==6.0.8 - # via -r ./requirements.in + # via -r requirements.in flower==1.0.0 - # via -r ./requirements.in + # via -r requirements.in github3.py==0.9.6 - # via -r ./requirements.in + # via -r requirements.in humanize==3.11.0 # via flower idna==3.2 @@ -112,7 +118,7 @@ idna==3.2 iniconfig==1.1.1 # via pytest jinja2==3.0.1 - # via -r ./requirements.in + # via -r requirements.in kombu==5.1.0 # via celery lexid==2021.1006 @@ -125,7 +131,7 @@ oauthlib==3.1.1 # social-auth-core packaging==21.0 # via - # -r ./requirements.in + # -r requirements.in # pytest pathlib2==2.3.6 # via bumpver @@ -136,13 +142,13 @@ prometheus-client==0.11.0 prompt-toolkit==3.0.20 # via click-repl psycopg2-binary==2.9.1 - # via -r ./requirements.in + # via -r requirements.in py==1.10.0 # via pytest pycparser==2.20 # via cffi pygithub==1.55 - # via -r ./requirements.in + # via -r requirements.in pyjwt==2.1.0 # via # pygithub @@ -151,23 +157,23 @@ pynacl==1.4.0 # via pygithub pyparsing==2.4.7 # via packaging -pytest-cov==2.12.1 - # via -r ./requirements.in -pytest-django==4.4.0 - # via -r ./requirements.in pytest==6.2.5 # via - # -r ./requirements.in + # -r requirements.in # pytest-cov # pytest-django +pytest-cov==2.12.1 + # via -r requirements.in +pytest-django==4.4.0 + # via -r requirements.in python-crontab==2.5.1 # via django-celery-beat python-dateutil==2.8.2 # via - # -r ./requirements.in + # -r requirements.in # python-crontab python-gitlab==2.10.1 - # via -r ./requirements.in + # via -r requirements.in python3-openid==3.2.0 # via social-auth-core pytz==2021.1 @@ -178,17 +184,11 @@ pytz==2021.1 # flower redis==3.5.3 # via - # -r ./requirements.in + # -r requirements.in # django-redis -requests-mock==1.9.3 - # via -r ./requirements.in -requests-oauthlib==1.3.0 - # via social-auth-core -requests-toolbelt==0.9.1 - # via python-gitlab requests==2.26.0 # via - # -r ./requirements.in + # -r requirements.in # django-anymail # github3.py # pygithub @@ -197,23 +197,29 @@ requests==2.26.0 # requests-oauthlib # requests-toolbelt # social-auth-core +requests-mock==1.9.3 + # via -r requirements.in +requests-oauthlib==1.3.0 + # via social-auth-core +requests-toolbelt==0.9.1 + # via python-gitlab sentry-sdk==1.4.1 - # via -r ./requirements.in + # via -r requirements.in sgmllib3k==1.0.0 # via feedparser six==1.16.0 # via - # -r ./requirements.in + # -r requirements.in # click-repl # pathlib2 # pynacl # python-dateutil # requests-mock social-auth-app-django==5.0.0 - # via -r ./requirements.in + # via -r requirements.in social-auth-core==4.1.0 # via - # -r ./requirements.in + # -r requirements.in # social-auth-app-django sqlparse==0.4.2 # via @@ -221,7 +227,7 @@ sqlparse==0.4.2 # django-debug-toolbar structlog==21.1.0 # via - # -r ./requirements.in + # -r requirements.in # django-structlog toml==0.10.2 # via @@ -231,22 +237,17 @@ toml==0.10.2 tornado==6.1 # via flower trove-classifiers==2021.9.23 - # via -r ./requirements.in -typing-extensions==3.10.0.2 - # via - # asgiref - # importlib-metadata - # structlog -uritemplate.py==3.0.2 - # via github3.py + # via -r requirements.in uritemplate==3.0.1 # via uritemplate.py +uritemplate.py==3.0.2 + # via github3.py urllib3==1.26.7 # via # requests # sentry-sdk uwsgi==2.0.19.1 - # via -r ./requirements.in + # via -r requirements.in vine==5.0.0 # via # amqp @@ -255,13 +256,11 @@ vine==5.0.0 wcwidth==0.2.5 # via prompt-toolkit webstack-django-sorting==1.0.2 - # via -r ./requirements.in + # via -r requirements.in whitenoise==5.3.0 - # via -r ./requirements.in + # via -r requirements.in wrapt==1.12.1 # via deprecated -zipp==3.5.0 - # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools From b4567824923e30f25c5f7ddca13e2bd085b09a81 Mon Sep 17 00:00:00 2001 From: Nicolle Cysneiros Date: Fri, 29 Oct 2021 16:22:18 -0300 Subject: [PATCH 5/5] Transfers package update logic to update_package_task method --- package/exceptions.py | 9 ++++++++ .../management/commands/package_updater.py | 22 +------------------ package/tasks.py | 12 ++++++++++ 3 files changed, 22 insertions(+), 21 deletions(-) create mode 100644 package/exceptions.py diff --git a/package/exceptions.py b/package/exceptions.py new file mode 100644 index 000000000..fe753c0b5 --- /dev/null +++ b/package/exceptions.py @@ -0,0 +1,9 @@ +import logging + +class PackageUpdaterException(Exception): + def __init__(self, error, title): + log_message = "For {title}, {error_type}: {error}".format( + title=title, error_type=type(error), error=error + ) + logging.critical(log_message) + logging.exception(error) \ No newline at end of file diff --git a/package/management/commands/package_updater.py b/package/management/commands/package_updater.py index 1e2007c81..e2e156a39 100644 --- a/package/management/commands/package_updater.py +++ b/package/management/commands/package_updater.py @@ -13,15 +13,6 @@ logger = logging.getLogger(__name__) -class PackageUpdaterException(Exception): - def __init__(self, error, title): - log_message = "For {title}, {error_type}: {error}".format( - title=title, error_type=type(error), error=error - ) - logging.critical(log_message) - logging.exception(error) - - class Command(BaseCommand): help = "Updates all the packages in the system. Commands belongs to django-packages.package" @@ -40,18 +31,7 @@ def handle(self, *args, **options): sleep(120) break - try: - try: - update_package_task.delay(package.id) - package.fetch_metadata(fetch_pypi=False) - package.fetch_commits() - except Exception as e: - logger.error( - f"Error while fetching package details for {package.title}." - ) - raise PackageUpdaterException(e, package.title) - except PackageUpdaterException: - logger.error(f"Unable to update {package.title}", exc_info=True) + update_package_task.delay(package.id) print(f"{__file__}::handle::sleep(5)") sleep(5) diff --git a/package/tasks.py b/package/tasks.py index 50bca5b25..1ab14078e 100644 --- a/package/tasks.py +++ b/package/tasks.py @@ -1,6 +1,7 @@ from settings import celery_app from celery.utils.log import get_task_logger +from .exceptions import PackageUpdaterException from .models import Package @@ -12,3 +13,14 @@ def update_package_task(package_id): package = Package.objects.get(id=package_id) logger.info(f'Start updating package {package.id}') + try: + try: + package.fetch_metadata(fetch_pypi=False) + package.fetch_commits() + except Exception as e: + logger.error( + f"Error while fetching package details for {package.title}." + ) + raise PackageUpdaterException(e, package.title) + except PackageUpdaterException: + logger.error(f"Unable to update {package.title}", exc_info=True)