diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..ee8e4fbec --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,79 @@ +name: Build and Test + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + +permissions: + contents: read + +jobs: + php-integration-tests: + runs-on: ubuntu-latest + steps: + - name: Setup Helioviewer Docker environment + uses: Helioviewer-Project/helioviewer.org-docker/.github/actions/helioviewer-docker@main + with: + api-ref: ${{ github.ref }} + + - name: Disable movie builder (required for tests) + working-directory: helioviewer.org-docker + run: docker compose down movies + + # Run tests inside the api container + - name: Run phpunit tests + working-directory: helioviewer.org-docker + run: docker compose exec -T api composer run-script test + + - name: Run python tests + uses: Helioviewer-Project/helioviewer.org-docker/.github/actions/pytest-api@main + with: + working-directory: helioviewer.org-docker + + - name: Print container logs + if: always() + working-directory: helioviewer.org-docker + run: | + docker compose logs + + - name: Print API error logs + if: always() + working-directory: helioviewer.org-docker + run: cat data/log/* + + playwright-e2e-tests: + strategy: + matrix: + shardIndex: [1, 2, 3, 4, 5] + shardTotal: [5] + # If one of the shards fails, continue running the remaining tests + fail-fast: false + + timeout-minutes: 60 + runs-on: ubuntu-latest + steps: + - name: Run Playwright tests + uses: Helioviewer-Project/helioviewer.org-tests/.github/actions/playwright-test@main + with: + api-ref: ${{ github.sha }} + shard-index: ${{ matrix.shardIndex }} + shard-total: ${{ matrix.shardTotal }} + merge-reports: + # Merge reports after playwright-tests, even if some shards have failed + if: ${{ !cancelled() }} + needs: [playwright-e2e-tests] + + runs-on: ubuntu-latest + steps: + - name: Checkout test code + uses: actions/checkout@v4 + with: + repository: 'Helioviewer-Project/helioviewer.org-tests' + path: 'helioviewer.org-tests' + + - name: Merge Playwright reports + uses: Helioviewer-Project/helioviewer.org-tests/.github/actions/playwright-merge-reports@main + with: + test-repo-path: 'helioviewer.org-tests' diff --git a/.github/workflows/php.yml b/.github/workflows/php.yml deleted file mode 100644 index 4e1f45ab4..000000000 --- a/.github/workflows/php.yml +++ /dev/null @@ -1,155 +0,0 @@ -name: Build and Test - -on: - push: - branches: [ "main" ] - pull_request: - branches: [ "main" ] - -permissions: - contents: read - -jobs: - php-integration-tests: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - name: Checkout the Dockerfile for local Helioviewer - uses: actions/checkout@v4 - with: - repository: 'Helioviewer-Project/helioviewer.org-docker' - path: 'compose' - sparse-checkout: | - compose.yaml - .env.example - sparse-checkout-cone-mode: false - - name: Setup environment file - run: mv compose/.env.example ../.env - - - name: Start local Helioviewer environment - id: docker - run: | - mv compose/compose.yaml .. - cd .. - # I quit. This keeps failing saying coordinator is unhealthy. - # It is healthy. The next step 'Print container logs' says it is healthy - # Just ignore. docker compose up is already tested that it goes "up" - # in order to be on the main branch. - docker compose up --wait api || true - - - name: Print container logs - if: always() - run: | - cd .. - docker container list - docker compose logs - - # Run the tests inside the api container - - name: Run phpunit tests - run: docker exec -t helioviewer-api-1 composer run-script test - - - name: Run python tests - run: docker exec -t helioviewer-api-1 composer run-script test-python - playwright-e2e-tests: - strategy: - matrix: - shardIndex: [1, 2, 3, 4, 5] - shardTotal: [5] - # If one of the shards fails, continue running the remaining tests - fail-fast: false - - timeout-minutes: 60 - runs-on: ubuntu-latest - steps: - - name: Checkout the code under test - uses: actions/checkout@v4 - with: - submodules: true - path: 'api' - - name: Checkout the latest Helioviewer Front End - uses: actions/checkout@v4 - with: - repository: 'Helioviewer-Project/helioviewer.org' - path: 'helioviewer.org' - - name: Checkout test code - uses: actions/checkout@v4 - with: - repository: 'Helioviewer-Project/helioviewer.org-tests' - path: 'tests' - # See https://github.com/actions/checkout?tab=readme-ov-file#fetch-only-a-single-file - - name: Checkout the Dockerfile for local Helioviewer - uses: actions/checkout@v4 - with: - repository: 'Helioviewer-Project/helioviewer.org-docker' - path: 'compose' - sparse-checkout: | - compose.yaml - .env.example - sparse-checkout-cone-mode: false - - name: Setup environment file - run: mv compose/.env.example .env - - name: Start local Helioviewer environment - id: docker - run: | - mv compose/compose.yaml . - docker compose up -d --wait - - name: Show docker compose logs - if: always() - run: | - docker compose logs - cat compose.yaml - - name: Fix cache directory permissions - if: always() - run: | - docker compose exec web chown 33:33 cache - - uses: actions/setup-node@v4 - with: - node-version: lts/* - - name: Install Playwright Browsers - run: | - cd tests - npm ci - npx playwright install --with-deps - - name: Run Playwright tests - run: | - cd tests - npx playwright test --shard=${{matrix.shardIndex}}/${{matrix.shardTotal}} - - uses: actions/upload-artifact@v4 - if: always() - with: - name: blob-report-${{matrix.shardIndex}} - path: tests/blob-report/ - retention-days: 1 - merge-reports: - # Merge reports after playwright-tests, even if some shards have failed - if: ${{ !cancelled() }} - needs: [playwright-e2e-tests] - - runs-on: ubuntu-latest - steps: - - name: Checkout test code - uses: actions/checkout@v4 - with: - repository: 'Helioviewer-Project/helioviewer.org-tests' - - uses: actions/setup-node@v4 - with: - node-version: lts/* - - name: Install dependencies - run: npm ci - - name: Download blob reports from GitHub Actions Artifacts - uses: actions/download-artifact@v4 - with: - path: all-blob-reports - pattern: blob-report-* - merge-multiple: true - - - name: Merge into HTML Report - run: npx playwright merge-reports --reporter html ./all-blob-reports - - - name: Upload HTML report - uses: actions/upload-artifact@v4 - with: - name: html-report--attempt-${{ github.run_attempt }} - path: playwright-report - retention-days: 5 diff --git a/install/__test__/test_vso_sunpy_download.py b/install/__test__/test_vso_sunpy_download.py index 48bb2a441..7f4c4db8a 100644 --- a/install/__test__/test_vso_sunpy_download.py +++ b/install/__test__/test_vso_sunpy_download.py @@ -1,40 +1,42 @@ -from unittest.mock import patch -import responses -import unittest -import requests -import os - -class TestVsoDownload(unittest.TestCase): - - @patch("sunpy.net.Fido.fetch") - @responses.activate - def test_getImageGroup(self, fido_fetch_response): - # Allow requests to localhost to go through. - responses.add_passthru("http://localhost") - URL = "http://localhost/?action=getSciDataScript&imageScale=4.84088176&sourceIds=[13,10]&startDate=2021-06-01T00:01:00Z&endDate=2021-06-01T00:01:15Z&lang=sunpy&provider=vso"; - - response = requests.get(URL) - - self.assertEqual(200, response.status_code) - self.assertEqual("OK", response.reason) - - script_with_paths = response.content.replace(b'os.path.expanduser(\'~/\')', bytes('\'/tmp/\'', encoding='utf-8')) - - try: - script_for_compile = compile(script_with_paths, '', 'exec', flags=0, dont_inherit=True) - except SyntaxError: - self.fail("Test Fail: Could not compile downloaded sunpy vso script , possible error in script") - - test_dir = os.path.dirname(os.path.abspath(__file__)) - request_path = os.path.join(test_dir, 'mocked_requests', 'vso_sunpy_download.yaml') - responses._add_from_file(file_path=request_path) - - fido_fetch_response.return_value = ['theoretical_file.fits'] - locals = {}; - exec(script_for_compile,globals(), locals) - - self.assertEqual(len(locals['data_aia_304']), 1) - self.assertEqual(len(locals['data_aia_171']), 1) - -if __name__ == '__main__': - unittest.main() +from unittest.mock import patch +import responses +import unittest +import requests +import os + +class TestVsoDownload(unittest.TestCase): + + @patch("sunpy.net.Fido.fetch") + @responses.activate + def test_getImageGroup(self, fido_fetch_response): + # Get API host from environment variable, default to localhost + host = os.environ.get('PYTEST_API_HOST', 'localhost') + # Allow requests to the configured host to go through. + responses.add_passthru(f"http://{host}") + URL = f"http://{host}/?action=getSciDataScript&imageScale=4.84088176&sourceIds=[13,10]&startDate=2021-06-01T00:01:00Z&endDate=2021-06-01T00:01:15Z&lang=sunpy&provider=vso" + + response = requests.get(URL) + + self.assertEqual(200, response.status_code) + self.assertEqual("OK", response.reason) + + script_with_paths = response.content.replace(b'os.path.expanduser(\'~/\')', bytes('\'/tmp/\'', encoding='utf-8')) + + try: + script_for_compile = compile(script_with_paths, '', 'exec', flags=0, dont_inherit=True) + except SyntaxError: + self.fail("Test Fail: Could not compile downloaded sunpy vso script , possible error in script") + + test_dir = os.path.dirname(os.path.abspath(__file__)) + request_path = os.path.join(test_dir, 'mocked_requests', 'vso_sunpy_download.yaml') + responses._add_from_file(file_path=request_path) + + fido_fetch_response.return_value = ['theoretical_file.fits'] + locals = {}; + exec(script_for_compile,globals(), locals) + + self.assertEqual(len(locals['data_aia_304']), 1) + self.assertEqual(len(locals['data_aia_171']), 1) + +if __name__ == '__main__': + unittest.main() diff --git a/src/Movie/FFMPEGEncoder.php b/src/Movie/FFMPEGEncoder.php index 08d989209..909746881 100644 --- a/src/Movie/FFMPEGEncoder.php +++ b/src/Movie/FFMPEGEncoder.php @@ -55,6 +55,17 @@ public function __construct($directory, $filename, $frameRate, $width, $height, $this->_log = fopen($directory . 'ffmpeg.log', 'a'); } + private function checkVideoOutput($outputFile) { + // If FFmpeg segfaults, an empty movie container may still be produced, + if (!file_exists($outputFile)) { + throw new Exception("FFMpeg error encountered - movie file $outputFile does not exist"); + } + $fsize = filesize($outputFile); + if ($fsize < 1000) { + throw new Exception("FFmpeg error encountered - Expected movie to be at least 1000 bytes, got $fsize ($outputFile)", 43); + } + } + /** * Creates a medium quality video */ @@ -72,9 +83,7 @@ public function createVideo($convertHQ = false) $this->_createWebMVideo($outputFile); } - // If FFmpeg segfaults, an empty movie container may still be produced, - if (!file_exists($outputFile) || filesize($outputFile) < 1000) - throw new Exception("FFmpeg error encountered.", 43); + $this->checkVideoOutput($outputFile); return $outputFile; } @@ -93,9 +102,8 @@ public function createHQVideo() $this->_createWebMVideo($outputFile, 1); } - // If FFmpeg segfaults, an empty movie container may still be produced - if (!file_exists($outputFile) || filesize($outputFile) < 1000) - throw new Exception("FFmpeg error encountered.", 43); + // If FFmpeg segfaults, an empty movie container may still be produced, + $this->checkVideoOutput($outputFile); return $outputFile; } diff --git a/src/Movie/HelioviewerMovie.php b/src/Movie/HelioviewerMovie.php index cef0b9d57..d8fb0cdca 100644 --- a/src/Movie/HelioviewerMovie.php +++ b/src/Movie/HelioviewerMovie.php @@ -275,7 +275,7 @@ public function build() { } catch (Exception $e) { Sentry::capture($e); - $this->_abort('Error encountered during movie frame compilation: ' . $e->getFile() . ":" . $e->getLine() . " - " . $e->getMessage() ); + $this->_abort('Error encountered during movie frame compilation: ' . $e->getFile() . ":" . $e->getLine() . " - " . $e->getMessage() . "\n" . $e->getTraceAsString()); } $t3 = time(); @@ -289,7 +289,8 @@ public function build() { $t4 = time(); $this->_abort('Error encountered during video encoding. ' . 'This may be caused by an FFmpeg configuration issue, ' . - 'or by insufficient permissions in the cache.', $t4 - $t3); + 'or by insufficient permissions in the cache.' . "\n" . + $e->getFile() . ":" . $e->getLine() . " - " . $e->getMessage() . "\n" . $e->getTraceAsString(), $t4 - $t3); } // Log buildMovie in statistics table