diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..0d92162 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,19 @@ +[run] +source = scraper +omit = + */venv/* + */virtualenv/* + */tests/* + */site-packages/* + +[report] +exclude_lines = + pragma: no cover + def __repr__ + raise NotImplementedError + if __name__ == .__main__.: + pass + raise ImportError + +[html] +directory = coverage_html_report \ No newline at end of file diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index a34f836..4c647db 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -5,13 +5,12 @@ name: Python package on: push: - branches: [ "main" ] + branches: ["main"] pull_request: - branches: [ "main" ] + branches: ["main"] jobs: build: - runs-on: ubuntu-latest strategy: fail-fast: false @@ -19,22 +18,29 @@ jobs: python-version: ["3.11", "3.12"] steps: - - uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - python -m pip install flake8 pytest - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - - name: Lint with flake8 - run: | - # stop the build if there are Python syntax errors or undefined names - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - name: Test with pytest - run: | - pytest + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -r requirements-dev.txt + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + - name: Lint with flake8 + run: | + # stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + - name: Test with pytest and coverage + run: | + pytest --cov=scraper --cov-report=xml --cov-report=term-missing --cov-fail-under=70 + - name: Upload coverage report to Codecov + uses: codecov/codecov-action@v5 + with: + file: ./coverage.xml + fail_ci_if_error: true + token: ${{ secrets.CODECOV_TOKEN }} + verbose: true diff --git a/README.md b/README.md index 6f4ce92..a63fe4a 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,8 @@ # Scraper +[![Python Tests](https://github.com/johnburbridge/scraper/actions/workflows/python-package.yml/badge.svg)](https://github.com/johnburbridge/scraper/actions/workflows/python-package.yml) +[![Coverage](https://codecov.io/gh/johnburbridge/scraper/branch/main/graph/badge.svg)](https://codecov.io/gh/johnburbridge/scraper) + ## Objectives * Given a URL, recursively crawl its links * Store the response diff --git a/main.py b/main.py index 3ecc169..9f4058b 100755 --- a/main.py +++ b/main.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 import argparse import logging import sys diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..9851efd --- /dev/null +++ b/pytest.ini @@ -0,0 +1,6 @@ +[pytest] +addopts = --cov=scraper --cov-report=term-missing +testpaths = tests +python_files = test_*.py +python_classes = Test* +python_functions = test_* \ No newline at end of file diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..e38f41d --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,6 @@ +pytest>=7.0.0 +pytest-cov>=4.1.0 +pytest-asyncio>=0.21.0 +flake8>=6.0.0 +black>=23.0.0 +mypy>=1.0.0 \ No newline at end of file