diff --git a/.github/workflows/end2end_run.yml b/.github/workflows/end2end_run.yml index 36a00745..9a2b3d91 100644 --- a/.github/workflows/end2end_run.yml +++ b/.github/workflows/end2end_run.yml @@ -42,6 +42,8 @@ jobs: python -m pip install --upgrade pip python -m pip install -e .[dev] if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + - name: show environment + run: python -m pip freeze - name: end2end test with pytest run: | pytest -p no:warnings --doctest-rst -m "end2end" --pyargs stixcore --remote-data=any diff --git a/docs/pipelineconfiguration.rst b/docs/pipelineconfiguration.rst index 86006101..83a6f4ec 100644 --- a/docs/pipelineconfiguration.rst +++ b/docs/pipelineconfiguration.rst @@ -44,7 +44,7 @@ SOLMOC TM data is synced from pub026 via rsync by cron every 15min. .. code-block:: # sync TM data from GFTS to local "wait for processing" folder - */15 * * * * rsync -av stixcore@147.86.8.26:'/home/solmoc/from_edds/tm/*PktTmRaw*.xml' /data/stix/SOLSOC/from_edds/tm/incoming > /dev/null + */15 * * * * rsync -av stixcore@147.86.8.26:'/home/solmoc/from_edds/tm/*PktTmRaw*.xml' /data/stix/SOLSOC/from_edds/tm/incomming > /dev/null The very latest SPICE kernels are required for accurate pointing information of the satellite. We may receive TM files via GFTS that contain data for times that are not available in the latest SPICE kernels, as the latest SPICE kernels have not yet been updated and delivered. @@ -53,7 +53,7 @@ Therefore, a stage and delay step is added via cron (every 30 min) that only pub .. code-block:: # move the TM data from the wait into the processing folder - that will trigger the pipeline to "get started": the wait period is meanwhile short - */30 * * * * find /data/stix/SOLSOC/from_edds/tm/incoming/ -type f -mmin +2 -exec rsync -a {} /data/stix/SOLSOC/from_edds/tm/processing/ \; + */30 * * * * find /data/stix/SOLSOC/from_edds/tm/incomming/ -type f -mmin +2 -exec rsync -a {} /data/stix/SOLSOC/from_edds/tm/processing/ \; Sync STIX-CONF repo in all used instances ***************************************** diff --git a/stixcore/util/scripts/end2end_testing.py b/stixcore/util/scripts/end2end_testing.py index dc0c134e..cf8cfa49 100644 --- a/stixcore/util/scripts/end2end_testing.py +++ b/stixcore/util/scripts/end2end_testing.py @@ -188,7 +188,7 @@ def end2end_pipeline(indir, fitsdir): splits=1, outdir=datapath, socdir=Path( - "/data/stix/SOLSOC/from_edds/tm/incoming/", + "/data/stix/SOLSOC/from_edds/tm/incomming/", ), ) diff --git a/stixcore/util/scripts/find_unprocessed_tm.py b/stixcore/util/scripts/find_unprocessed_tm.py index d59e0c57..7034c7d5 100644 --- a/stixcore/util/scripts/find_unprocessed_tm.py +++ b/stixcore/util/scripts/find_unprocessed_tm.py @@ -12,7 +12,7 @@ TM_REGEX = re.compile(r".*PktTmRaw.*.xml$") logging_dir = Path(CONFIG.get("Pipeline", "log_dir")) tm_dir = Path(CONFIG.get("Paths", "tm_archive")) -tm_dir = Path("/data/stix/SOLSOC/from_edds/tm/incoming") +tm_dir = Path("/data/stix/SOLSOC/from_edds/tm/incomming") list_of_log_files = logging_dir.glob("*.xml.out") latest_log_file = ( logging_dir / "TM_BatchRequest.PktTmRaw.SOL.0.2023.305.15.33.13.852.bBWz@2023.352.21.00.01.039.1.xml.out"