From b26123690ce8a1a459dd275b093e10f103710657 Mon Sep 17 00:00:00 2001 From: grallewellyn Date: Tue, 29 Jul 2025 14:16:59 -0700 Subject: [PATCH 01/17] added ogc endpoints --- maap/config_reader.py | 3 + maap/maap.py | 235 ++++++++++++++++++++++++++++++++++++++++++ test_ogc_functions.py | 74 +++++++++++++ 3 files changed, 312 insertions(+) create mode 100644 test_ogc_functions.py diff --git a/maap/config_reader.py b/maap/config_reader.py index e8a2fbd..c8d902b 100644 --- a/maap/config_reader.py +++ b/maap/config_reader.py @@ -81,6 +81,9 @@ def __init__(self, maap_host): self.algorithm_build = self._get_api_endpoint("algorithm_build") self.mas_algo = self._get_api_endpoint("mas_algo") self.dps_job = self._get_api_endpoint("dps_job") + self.processes_ogc = self._get_api_endpoint("processes_ogc") + self.deployment_jobs_ogc = self._get_api_endpoint("deployment_jobs_ogc") + self.jobs_ogc = self._get_api_endpoint("jobs_org") self.member_dps_token = self._get_api_endpoint("member_dps_token") self.requester_pays = self._get_api_endpoint("requester_pays") self.edc_credentials = self._get_api_endpoint("edc_credentials") diff --git a/maap/maap.py b/maap/maap.py index f8683eb..1efc65a 100644 --- a/maap/maap.py +++ b/maap/maap.py @@ -410,6 +410,241 @@ def show(self, granule, display_config={}): ) viz.show() + # OGC-compliant endpoint functions + def search_processes_ogc(self): + """ + Search all OGC processes + :return: Response object with all deployed processes + """ + headers = self._get_api_header() + logger.debug('GET request sent to {}'.format(self.config.processes_ogc)) + print('GET request sent to {}'.format(self.config.processes_ogc)) + + response = requests.get( + url=self.config.processes_ogc, + headers=headers + ) + return response + + def deploy_process_ogc(self, execution_unit_href): + """ + Deploy a new OGC process + :param execution_unit_href: URL to the CWL file + :return: Response object with deployment information + """ + headers = self._get_api_header(content_type='application/json') + data = { + "executionUnit": { + "href": execution_unit_href + } + } + logger.debug('POST request sent to {}'.format(self.config.processes_ogc)) + response = requests.post( + url=self.config.processes_ogc, + headers=headers, + json=data + ) + return response + + def get_deployment_status_ogc(self, deployment_id): + """ + Query the current status of an algorithm being deployed + :param deployment_id: The deployment job ID + :return: Response object with deployment status + """ + url = os.path.join(self.config.deployment_jobs_ogc, deployment_id) + headers = self._get_api_header() + logger.debug('GET request sent to {}'.format(url)) + response = requests.get( + url=url, + headers=headers + ) + return response + + def describe_process_ogc(self, process_id): + """ + Get detailed information about a specific OGC process + :param process_id: The process ID to describe + :return: Response object with process details + """ + url = os.path.join(self.config.processes_ogc, process_id) + headers = self._get_api_header() + response = requests.get( + url=url, + headers=headers + ) + return response + + def update_process_ogc(self, process_id, execution_unit_href): + """ + Replace an existing OGC process (must be the original deployer) + :param process_id: The process ID to update + :param execution_unit_href: URL to the new CWL file + :return: Response object with update information + """ + url = os.path.join(self.config.processes_ogc, process_id) + headers = self._get_api_header(content_type='application/json') + data = { + "executionUnit": { + "href": execution_unit_href + } + } + logger.debug('PUT request sent to {}'.format(url)) + response = requests.put( + url=url, + headers=headers, + json=data + ) + return response + + def delete_process_ogc(self, process_id): + """ + Delete an existing OGC process (must be the original deployer) + :param process_id: The process ID to delete + :return: Response object with deletion confirmation + """ + url = os.path.join(self.config.processes_ogc, process_id) + headers = self._get_api_header() + logger.debug('DELETE request sent to {}'.format(url)) + response = requests.delete( + url=url, + headers=headers + ) + return response + + def get_process_package_ogc(self, process_id): + """ + Access the formal description that can be used to deploy an OGC process + :param process_id: The process ID + :return: Response object with process package description + """ + url = os.path.join(self.config.processes_ogc, process_id, 'package') + headers = self._get_api_header() + logger.debug('GET request sent to {}'.format(url)) + response = requests.get( + url=url, + headers=headers + ) + return response + + def execute_process_ogc(self, process_id, inputs, queue, dedup=None, tag=None): + """ + Execute an OGC process job + :param process_id: The process ID to execute + :param inputs: Dictionary of input parameters for the process + :param queue: Queue to run the job on + :param dedup: Optional deduplication flag + :param tag: Optional user-defined tag for the job + :return: Response object with job execution information + """ + url = os.path.join(self.config.processes_ogc, process_id, 'execution') + headers = self._get_api_header(content_type='application/json') + data = { + "inputs": inputs, + "queue": queue + } + if dedup is not None: + data["dedup"] = dedup + if tag is not None: + data["tag"] = tag + + logger.debug('POST request sent to {}'.format(url)) + + response = requests.post( + url=url, + headers=headers, + json=data + ) + return response + + def get_job_status_ogc(self, job_id): + """ + Get the status of an OGC job + :param job_id: The job ID to check status for + :return: Response object with job status + """ + url = os.path.join(self.config.jobs_ogc, job_id) + headers = self._get_api_header() + logger.debug('GET request sent to {}'.format(url)) + + response = requests.get( + url=url, + headers=headers + ) + return response + + def cancel_job_ogc(self, job_id, wait_for_completion=False): + """ + Cancel a running OGC job or delete a queued job + :param job_id: The job ID to cancel + :param wait_for_completion: Whether to wait for the cancellation to complete + :return: Response object with cancellation status + """ + url = os.path.join(self.config.jobs_ogc, job_id) + params = {} + if wait_for_completion: + params['wait_for_completion'] = str(wait_for_completion).lower() + + headers = self._get_api_header() + logger.debug('DELETE request sent to {}'.format(url)) + + response = requests.delete( + url=url, + headers=headers, + params=params + ) + return response + + def get_job_results_ogc(self, job_id): + """ + Get the results of a completed OGC job + :param job_id: The job ID to get results for + :return: Response object with job results + """ + url = os.path.join(self.config.jobs_ogc, job_id, 'results') + headers = self._get_api_header() + logger.debug('GET request sent to {}'.format(url)) + response = requests.get( + url=url, + headers=headers + ) + return response + + def list_jobs_ogc(self, **kwargs): + """ + Get a list of OGC jobs with optional filtering + :param kwargs: Optional query parameters for filtering jobs + :return: Response object with list of jobs + """ + url = os.path.join(self.config.jobs_ogc) + headers = self._get_api_header() + + # Filter out None values from kwargs + params = {k: v for k, v in kwargs.items() if v is not None} + + logger.debug('GET request sent to {}'.format(url)) + response = requests.get( + url=url, + headers=headers, + params=params + ) + return response + + def get_job_metrics_ogc(self, job_id): + """ + Get metrics for an OGC job + :param job_id: The job ID to get metrics for + :return: Response object with job metrics + """ + url = os.path.join(self.config.jobs_ogc, job_id, 'metrics') + headers = self._get_api_header() + logger.debug('GET request sent to {}'.format(url)) + response = requests.get( + url=url, + headers=headers + ) + return response + if __name__ == "__main__": print("initialized") \ No newline at end of file diff --git a/test_ogc_functions.py b/test_ogc_functions.py new file mode 100644 index 0000000..765e320 --- /dev/null +++ b/test_ogc_functions.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +""" +Simple test script to verify OGC function signatures and basic functionality +""" + +import sys +import os +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '.')) + +from maap.maap import MAAP + +def test_ogc_functions(): + """Test that all OGC functions exist and have correct signatures""" + + # Initialize MAAP instance + maap = MAAP() + + # Test function existence and basic attributes + ogc_functions = [ + 'search_processes_ogc', + 'deploy_process_ogc', + 'get_deployment_status_ogc', + 'describe_process_ogc', + 'update_process_ogc', + 'delete_process_ogc', + 'get_process_package_ogc', + 'execute_process_ogc', + 'get_job_status_ogc', + 'cancel_job_ogc', + 'get_job_results_ogc', + 'list_jobs_ogc', + 'get_job_metrics_ogc' + ] + + print("Testing OGC function existence...") + for func_name in ogc_functions: + assert hasattr(maap, func_name), f"Function {func_name} not found" + func = getattr(maap, func_name) + assert callable(func), f"Function {func_name} is not callable" + print(f"✓ {func_name} exists and is callable") + + print("\nTesting function signatures...") + + # Test functions that don't require parameters + try: + # These should work without throwing signature errors (though may fail on network call) + search_func = getattr(maap, 'search_processes_ogc') + print(f"✓ search_processes_ogc has correct signature") + except Exception as e: + print(f"✗ search_processes_ogc signature error: {e}") + + # Test functions with required parameters + try: + deploy_func = getattr(maap, 'deploy_process_ogc') + # This should not throw a signature error + print(f"✓ deploy_process_ogc has correct signature") + except Exception as e: + print(f"✗ deploy_process_ogc signature error: {e}") + + try: + execute_func = getattr(maap, 'execute_process_ogc') + # This should not throw a signature error + print(f"✓ execute_process_ogc has correct signature") + except Exception as e: + print(f"✗ execute_process_ogc signature error: {e}") + + print("\n✓ All OGC functions successfully added to maap-py!") + print("\nAvailable OGC functions:") + for func_name in ogc_functions: + func = getattr(maap, func_name) + print(f" - {func_name}: {func.__doc__.strip().split(':')[0] if func.__doc__ else 'No description'}") + +if __name__ == "__main__": + test_ogc_functions() \ No newline at end of file From 6c60c72009e132a18aef57a2f304775f62468f3b Mon Sep 17 00:00:00 2001 From: grallewellyn Date: Tue, 29 Jul 2025 14:58:29 -0700 Subject: [PATCH 02/17] added string casting and changed to list_processes wording --- maap/maap.py | 22 +++++++++++----------- test_ogc_functions.py | 8 ++++---- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/maap/maap.py b/maap/maap.py index 1efc65a..d8cc759 100644 --- a/maap/maap.py +++ b/maap/maap.py @@ -411,7 +411,7 @@ def show(self, granule, display_config={}): viz.show() # OGC-compliant endpoint functions - def search_processes_ogc(self): + def list_processes_ogc(self): """ Search all OGC processes :return: Response object with all deployed processes @@ -452,7 +452,7 @@ def get_deployment_status_ogc(self, deployment_id): :param deployment_id: The deployment job ID :return: Response object with deployment status """ - url = os.path.join(self.config.deployment_jobs_ogc, deployment_id) + url = os.path.join(self.config.deployment_jobs_ogc, str(deployment_id)) headers = self._get_api_header() logger.debug('GET request sent to {}'.format(url)) response = requests.get( @@ -467,7 +467,7 @@ def describe_process_ogc(self, process_id): :param process_id: The process ID to describe :return: Response object with process details """ - url = os.path.join(self.config.processes_ogc, process_id) + url = os.path.join(self.config.processes_ogc, str(process_id)) headers = self._get_api_header() response = requests.get( url=url, @@ -482,7 +482,7 @@ def update_process_ogc(self, process_id, execution_unit_href): :param execution_unit_href: URL to the new CWL file :return: Response object with update information """ - url = os.path.join(self.config.processes_ogc, process_id) + url = os.path.join(self.config.processes_ogc, str(process_id)) headers = self._get_api_header(content_type='application/json') data = { "executionUnit": { @@ -503,7 +503,7 @@ def delete_process_ogc(self, process_id): :param process_id: The process ID to delete :return: Response object with deletion confirmation """ - url = os.path.join(self.config.processes_ogc, process_id) + url = os.path.join(self.config.processes_ogc, str(process_id)) headers = self._get_api_header() logger.debug('DELETE request sent to {}'.format(url)) response = requests.delete( @@ -518,7 +518,7 @@ def get_process_package_ogc(self, process_id): :param process_id: The process ID :return: Response object with process package description """ - url = os.path.join(self.config.processes_ogc, process_id, 'package') + url = os.path.join(self.config.processes_ogc, str(process_id), 'package') headers = self._get_api_header() logger.debug('GET request sent to {}'.format(url)) response = requests.get( @@ -537,7 +537,7 @@ def execute_process_ogc(self, process_id, inputs, queue, dedup=None, tag=None): :param tag: Optional user-defined tag for the job :return: Response object with job execution information """ - url = os.path.join(self.config.processes_ogc, process_id, 'execution') + url = os.path.join(self.config.processes_ogc, str(process_id), 'execution') headers = self._get_api_header(content_type='application/json') data = { "inputs": inputs, @@ -563,7 +563,7 @@ def get_job_status_ogc(self, job_id): :param job_id: The job ID to check status for :return: Response object with job status """ - url = os.path.join(self.config.jobs_ogc, job_id) + url = os.path.join(self.config.jobs_ogc, str(job_id)) headers = self._get_api_header() logger.debug('GET request sent to {}'.format(url)) @@ -580,7 +580,7 @@ def cancel_job_ogc(self, job_id, wait_for_completion=False): :param wait_for_completion: Whether to wait for the cancellation to complete :return: Response object with cancellation status """ - url = os.path.join(self.config.jobs_ogc, job_id) + url = os.path.join(self.config.jobs_ogc, str(job_id)) params = {} if wait_for_completion: params['wait_for_completion'] = str(wait_for_completion).lower() @@ -601,7 +601,7 @@ def get_job_results_ogc(self, job_id): :param job_id: The job ID to get results for :return: Response object with job results """ - url = os.path.join(self.config.jobs_ogc, job_id, 'results') + url = os.path.join(self.config.jobs_ogc, str(job_id), 'results') headers = self._get_api_header() logger.debug('GET request sent to {}'.format(url)) response = requests.get( @@ -636,7 +636,7 @@ def get_job_metrics_ogc(self, job_id): :param job_id: The job ID to get metrics for :return: Response object with job metrics """ - url = os.path.join(self.config.jobs_ogc, job_id, 'metrics') + url = os.path.join(self.config.jobs_ogc, str(job_id), 'metrics') headers = self._get_api_header() logger.debug('GET request sent to {}'.format(url)) response = requests.get( diff --git a/test_ogc_functions.py b/test_ogc_functions.py index 765e320..d6bef5d 100644 --- a/test_ogc_functions.py +++ b/test_ogc_functions.py @@ -17,7 +17,7 @@ def test_ogc_functions(): # Test function existence and basic attributes ogc_functions = [ - 'search_processes_ogc', + 'list_processes_ogc', 'deploy_process_ogc', 'get_deployment_status_ogc', 'describe_process_ogc', @@ -44,10 +44,10 @@ def test_ogc_functions(): # Test functions that don't require parameters try: # These should work without throwing signature errors (though may fail on network call) - search_func = getattr(maap, 'search_processes_ogc') - print(f"✓ search_processes_ogc has correct signature") + search_func = getattr(maap, 'list_processes_ogc') + print(f"✓ list_processes_ogc has correct signature") except Exception as e: - print(f"✗ search_processes_ogc signature error: {e}") + print(f"✗ list_processes_ogc signature error: {e}") # Test functions with required parameters try: From 591d7572e5fb41fef1ef5c6a1994c80ccbb0cf10 Mon Sep 17 00:00:00 2001 From: grallewellyn Date: Wed, 30 Jul 2025 10:31:00 -0700 Subject: [PATCH 03/17] fixed typo jobs_ogc --- maap/config_reader.py | 2 +- maap/maap.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/maap/config_reader.py b/maap/config_reader.py index c8d902b..8329755 100644 --- a/maap/config_reader.py +++ b/maap/config_reader.py @@ -83,7 +83,7 @@ def __init__(self, maap_host): self.dps_job = self._get_api_endpoint("dps_job") self.processes_ogc = self._get_api_endpoint("processes_ogc") self.deployment_jobs_ogc = self._get_api_endpoint("deployment_jobs_ogc") - self.jobs_ogc = self._get_api_endpoint("jobs_org") + self.jobs_ogc = self._get_api_endpoint("jobs_ogc") self.member_dps_token = self._get_api_endpoint("member_dps_token") self.requester_pays = self._get_api_endpoint("requester_pays") self.edc_credentials = self._get_api_endpoint("edc_credentials") diff --git a/maap/maap.py b/maap/maap.py index d8cc759..fd19944 100644 --- a/maap/maap.py +++ b/maap/maap.py @@ -623,6 +623,7 @@ def list_jobs_ogc(self, **kwargs): params = {k: v for k, v in kwargs.items() if v is not None} logger.debug('GET request sent to {}'.format(url)) + print('GET request sent to {}'.format(url)) response = requests.get( url=url, headers=headers, From 4eec87bd88ab31e62757a125b63241289c21ea2b Mon Sep 17 00:00:00 2001 From: grallewellyn Date: Wed, 30 Jul 2025 11:02:32 -0700 Subject: [PATCH 04/17] added tests and updated testing instructions --- README.md | 4 +- maap/maap.py | 2 - test/test_ogc_functions.py | 199 +++++++++++++++++++++++++++++++++++++ test_ogc_functions.py | 74 -------------- 4 files changed, 202 insertions(+), 77 deletions(-) create mode 100644 test/test_ogc_functions.py delete mode 100644 test_ogc_functions.py diff --git a/README.md b/README.md index 2e840ad..b632fa1 100644 --- a/README.md +++ b/README.md @@ -76,5 +76,7 @@ lidarGranule = maap.searchGranule(instrument='lvis', site_name='lope') ## Test ```bash -python setup.py test +poetry install +poetry run pytest --cov=maap +poetry run pytest test/specific_test.py ``` diff --git a/maap/maap.py b/maap/maap.py index fd19944..f1d5da9 100644 --- a/maap/maap.py +++ b/maap/maap.py @@ -418,7 +418,6 @@ def list_processes_ogc(self): """ headers = self._get_api_header() logger.debug('GET request sent to {}'.format(self.config.processes_ogc)) - print('GET request sent to {}'.format(self.config.processes_ogc)) response = requests.get( url=self.config.processes_ogc, @@ -623,7 +622,6 @@ def list_jobs_ogc(self, **kwargs): params = {k: v for k, v in kwargs.items() if v is not None} logger.debug('GET request sent to {}'.format(url)) - print('GET request sent to {}'.format(url)) response = requests.get( url=url, headers=headers, diff --git a/test/test_ogc_functions.py b/test/test_ogc_functions.py new file mode 100644 index 0000000..b27b5a5 --- /dev/null +++ b/test/test_ogc_functions.py @@ -0,0 +1,199 @@ +""" +Test module for OGC functions in maap.py +""" + +import pytest +from maap.maap import MAAP + + +def test_list_processes_ogc(): + """Test list_processes_ogc function calls OGC processes endpoint and returns 200 with JSON""" + maap = MAAP(maap_host='api.dit.maap-project.org') + + response = maap.list_processes_ogc() + + # Check that we get a 200 status code + assert response.status_code == 200, f"Expected 200, got {response.status_code}" + + # Check that response is valid JSON + try: + json_data = response.json() + assert isinstance(json_data, (dict, list)), "Response should be valid JSON (dict or list)" + except ValueError as e: + pytest.fail(f"Response is not valid JSON: {e}") + + +def test_deploy_process_ogc(): + """Test deploy_process_ogc function""" + pass + + +def test_get_deployment_status_ogc(): + """Test get_deployment_status_ogc function""" + pass + + +def test_describe_process_ogc(): + """Test describe_process_ogc function by getting process list and describing first process""" + maap = MAAP(maap_host='api.dit.maap-project.org') + + # First get the list of processes + list_response = maap.list_processes_ogc() + assert list_response.status_code == 200, f"Failed to get process list: {list_response.status_code}" + + try: + processes_data = list_response.json() + except ValueError as e: + pytest.fail(f"Process list response is not valid JSON: {e}") + + # Check if there are any processes + if not processes_data or (isinstance(processes_data, dict) and not processes_data.get('processes')): + pytest.skip("No processes available to test describe_process_ogc") + + # Get the first process + if isinstance(processes_data, dict) and 'processes' in processes_data: + processes = processes_data['processes'] + else: + processes = processes_data + + if not processes or len(processes) == 0: + pytest.skip("No processes available to test describe_process_ogc") + + first_process = processes[0] + + # Find the self link or use process ID + process_id = None + if 'links' in first_process: + for link in first_process['links']: + if link.get('rel') == 'self': + href = link.get('href', '') + # Extract process ID from href like /ogc/processes/3 + if '/ogc/processes/' in href: + process_id = href.split('/ogc/processes/')[-1] + break + + # Fall back to process ID field if no self link found + if not process_id and 'id' in first_process: + process_id = first_process['id'] + + if not process_id: + pytest.skip("Could not determine process ID to test describe_process_ogc") + + # Now test the describe_process_ogc function + describe_response = maap.describe_process_ogc(process_id) + + # Check that we get a successful response + assert describe_response.status_code == 200, f"Expected 200, got {describe_response.status_code}" + + # Check that response is valid JSON + try: + describe_data = describe_response.json() + assert isinstance(describe_data, dict), "Describe response should be a JSON object" + except ValueError as e: + pytest.fail(f"Describe response is not valid JSON: {e}") + + # Verify the URL called contains the process ID + assert str(process_id) in describe_response.url + + +def test_update_process_ogc(): + """Test update_process_ogc function""" + pass + + +def test_delete_process_ogc(): + """Test delete_process_ogc function""" + pass + + +def test_get_process_package_ogc(): + """Test get_process_package_ogc function""" + maap = MAAP(maap_host='api.dit.maap-project.org') + + # First get the list of processes + list_response = maap.list_processes_ogc() + assert list_response.status_code == 200, f"Failed to get process list: {list_response.status_code}" + + try: + processes_data = list_response.json() + except ValueError as e: + pytest.fail(f"Process list response is not valid JSON: {e}") + + # Check if there are any processes + if not processes_data or (isinstance(processes_data, dict) and not processes_data.get('processes')): + pytest.skip("No processes available to test describe_process_ogc") + + # Get the first process + if isinstance(processes_data, dict) and 'processes' in processes_data: + processes = processes_data['processes'] + else: + processes = processes_data + + if not processes or len(processes) == 0: + pytest.skip("No processes available to test describe_process_ogc") + + first_process = processes[0] + + # Find the self link or use process ID + process_id = None + if 'links' in first_process: + for link in first_process['links']: + if link.get('rel') == 'self': + href = link.get('href', '') + # Extract process ID from href like /ogc/processes/3 + if '/ogc/processes/' in href: + process_id = href.split('/ogc/processes/')[-1] + break + + # Fall back to process ID field if no self link found + if not process_id and 'id' in first_process: + process_id = first_process['id'] + + if not process_id: + pytest.skip("Could not determine process ID to test describe_process_ogc") + + # Now test the package_response function + package_response = maap.get_process_package_ogc(process_id) + + # Check that we get a successful response + assert package_response.status_code == 200, f"Expected 200, got {package_response.status_code}" + + # Check that response is valid JSON + try: + package_data = package_response.json() + assert isinstance(package_data, dict), "Process Package response should be a JSON object" + except ValueError as e: + pytest.fail(f"Process package response is not valid JSON: {e}") + + # Verify the URL called contains the process ID + assert str(process_id) in package_response.url + + +def test_execute_process_ogc(): + """Test execute_process_ogc function""" + pass + + +def test_get_job_status_ogc(): + """Test get_job_status_ogc function""" + pass + + +def test_cancel_job_ogc(): + """Test cancel_job_ogc function""" + pass + + +def test_get_job_results_ogc(): + """Test get_job_results_ogc function""" + pass + + +def test_list_jobs_ogc(): + """Test list_jobs_ogc function""" + pass + + +def test_get_job_metrics_ogc(): + """Test get_job_metrics_ogc function""" + pass \ No newline at end of file diff --git a/test_ogc_functions.py b/test_ogc_functions.py deleted file mode 100644 index d6bef5d..0000000 --- a/test_ogc_functions.py +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env python3 -""" -Simple test script to verify OGC function signatures and basic functionality -""" - -import sys -import os -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '.')) - -from maap.maap import MAAP - -def test_ogc_functions(): - """Test that all OGC functions exist and have correct signatures""" - - # Initialize MAAP instance - maap = MAAP() - - # Test function existence and basic attributes - ogc_functions = [ - 'list_processes_ogc', - 'deploy_process_ogc', - 'get_deployment_status_ogc', - 'describe_process_ogc', - 'update_process_ogc', - 'delete_process_ogc', - 'get_process_package_ogc', - 'execute_process_ogc', - 'get_job_status_ogc', - 'cancel_job_ogc', - 'get_job_results_ogc', - 'list_jobs_ogc', - 'get_job_metrics_ogc' - ] - - print("Testing OGC function existence...") - for func_name in ogc_functions: - assert hasattr(maap, func_name), f"Function {func_name} not found" - func = getattr(maap, func_name) - assert callable(func), f"Function {func_name} is not callable" - print(f"✓ {func_name} exists and is callable") - - print("\nTesting function signatures...") - - # Test functions that don't require parameters - try: - # These should work without throwing signature errors (though may fail on network call) - search_func = getattr(maap, 'list_processes_ogc') - print(f"✓ list_processes_ogc has correct signature") - except Exception as e: - print(f"✗ list_processes_ogc signature error: {e}") - - # Test functions with required parameters - try: - deploy_func = getattr(maap, 'deploy_process_ogc') - # This should not throw a signature error - print(f"✓ deploy_process_ogc has correct signature") - except Exception as e: - print(f"✗ deploy_process_ogc signature error: {e}") - - try: - execute_func = getattr(maap, 'execute_process_ogc') - # This should not throw a signature error - print(f"✓ execute_process_ogc has correct signature") - except Exception as e: - print(f"✗ execute_process_ogc signature error: {e}") - - print("\n✓ All OGC functions successfully added to maap-py!") - print("\nAvailable OGC functions:") - for func_name in ogc_functions: - func = getattr(maap, func_name) - print(f" - {func_name}: {func.__doc__.strip().split(':')[0] if func.__doc__ else 'No description'}") - -if __name__ == "__main__": - test_ogc_functions() \ No newline at end of file From 2625e7ccd5596698e6ab374774a7f923c4dd5d7a Mon Sep 17 00:00:00 2001 From: grallewellyn Date: Wed, 30 Jul 2025 11:41:18 -0700 Subject: [PATCH 05/17] edited changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7262a1c..bb50b41 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] (post 4.2.0 release) ### Added +- Support for new OGC endpoints ### Changed ### Deprecated ### Removed From 664376b3bd9c2d076f34c46a12c232c0cbe1db2f Mon Sep 17 00:00:00 2001 From: grallewellyn Date: Thu, 21 Aug 2025 09:58:39 -0700 Subject: [PATCH 06/17] camelcase and removed _ogc from naming conventions --- maap/maap.py | 287 ++++++++++++++------------------------------------- 1 file changed, 78 insertions(+), 209 deletions(-) diff --git a/maap/maap.py b/maap/maap.py index f1d5da9..e084d59 100644 --- a/maap/maap.py +++ b/maap/maap.py @@ -63,7 +63,7 @@ def _upload_s3(self, filename, bucket, objectKey): """ return s3_client.upload_file(filename, bucket, objectKey) - def searchGranule(self, limit=20, **kwargs): + def search_granule(self, limit=20, **kwargs): """ Search the CMR granules @@ -79,7 +79,7 @@ def searchGranule(self, limit=20, **kwargs): self._get_api_header(), self._DPS) for result in results][:limit] - def downloadGranule(self, online_access_url, destination_path=".", overwrite=False): + def download_granule(self, online_access_url, destination_path=".", overwrite=False): """ Direct download of http Earthdata granule URL (protected or public). @@ -100,7 +100,7 @@ def downloadGranule(self, online_access_url, destination_path=".", overwrite=Fal # noinspection PyProtectedMember return proxy._getHttpData(online_access_url, overwrite, final_destination) - def getCallFromEarthdataQuery(self, query, variable_name='maap', limit=1000): + def get_call_from_earthdata_query(self, query, variable_name='maap', limit=1000): """ Generate a literal string to use for calling the MAAP API @@ -111,7 +111,7 @@ def getCallFromEarthdataQuery(self, query, variable_name='maap', limit=1000): """ return self._CMR.generateGranuleCallFromEarthDataRequest(query, variable_name, limit) - def getCallFromCmrUri(self, search_url, variable_name='maap', limit=1000, search='granule'): + def get_call_from_cmr_uri(self, search_url, variable_name='maap', limit=1000, search='granule'): """ Generate a literal string to use for calling the MAAP API @@ -123,7 +123,7 @@ def getCallFromCmrUri(self, search_url, variable_name='maap', limit=1000, search """ return self._CMR.generateCallFromEarthDataQueryString(search_url, variable_name, limit, search) - def searchCollection(self, limit=100, **kwargs): + def search_collection(self, limit=100, **kwargs): """ Search the CMR collections :param limit: limit of the number of results @@ -133,7 +133,7 @@ def searchCollection(self, limit=100, **kwargs): results = self._CMR.get_search_results(url=self.config.search_collection_url, limit=limit, **kwargs) return [Collection(result, self.config.maap_host) for result in results][:limit] - def getQueues(self): + def get_queues(self): url = os.path.join(self.config.algorithm_register, 'resource') headers = self._get_api_header() logger.debug('GET request sent to {}'.format(self.config.algorithm_register)) @@ -145,20 +145,9 @@ def getQueues(self): ) return response - def registerAlgorithm(self, arg): - logger.debug('Registering algorithm with args ') - if type(arg) is dict: - arg = json.dumps(arg) - logger.debug(arg) - response = requests_utils.make_request(url=self.config.algorithm_register, config=self.config, - content_type='application/json', request_type=requests_utils.POST, - data=arg) - logger.debug('POST request sent to {}'.format(self.config.algorithm_register)) - return response - def register_algorithm_from_yaml_file(self, file_path): algo_config = algorithm_utils.read_yaml_file(file_path) - return self.registerAlgorithm(algo_config) + return self.register_algorithm(algo_config) def register_algorithm_from_yaml_file_backwards_compatible(self, file_path): algo_yaml = algorithm_utils.read_yaml_file(file_path) @@ -178,184 +167,15 @@ def register_algorithm_from_yaml_file_backwards_compatible(self, file_path): else: output_config.update({key: value}) logger.debug("Registering with config %s " % json.dumps(output_config)) - return self.registerAlgorithm(json.dumps(output_config)) - - def listAlgorithms(self): - url = self.config.mas_algo - headers = self._get_api_header() - logger.debug('GET request sent to {}'.format(url)) - logger.debug('headers:') - logger.debug(headers) - response = requests.get( - url=url, - headers=headers - ) - return response - - def describeAlgorithm(self, algoid): - url = os.path.join(self.config.mas_algo, algoid) - headers = self._get_api_header() - logger.debug('GET request sent to {}'.format(url)) - logger.debug('headers:') - logger.debug(headers) - response = requests.get( - url=url, - headers=headers - ) - return response - - def publishAlgorithm(self, algoid): - url = self.config.mas_algo.replace('algorithm', 'publish') - headers = self._get_api_header() - body = { "algo_id": algoid} - logger.debug('POST request sent to {}'.format(url)) - logger.debug('headers:') - logger.debug(headers) - logger.debug('body:') - logger.debug(body) - response = requests.post( - url=url, - headers=headers, - data=body - ) - return response + return self.register_algorithm(json.dumps(output_config)) - def deleteAlgorithm(self, algoid): - url = os.path.join(self.config.mas_algo, algoid) - headers = self._get_api_header() - logger.debug('DELETE request sent to {}'.format(url)) - logger.debug('headers:') - logger.debug(headers) - response = requests.delete( - url=url, - headers=headers - ) - return response - - - def getJob(self, jobid): + def get_job(self, jobid): job = DPSJob(self.config) job.id = jobid job.retrieve_attributes() return job - def getJobStatus(self, jobid): - job = DPSJob(self.config) - job.id = jobid - return job.retrieve_status() - - def getJobResult(self, jobid): - job = DPSJob(self.config) - job.id = jobid - return job.retrieve_result() - - def getJobMetrics(self, jobid): - job = DPSJob(self.config) - job.id = jobid - return job.retrieve_metrics() - - def cancelJob(self, jobid): - job = DPSJob(self.config) - job.id = jobid - return job.cancel_job() - - def listJobs(self, *, - algo_id=None, - end_time=None, - get_job_details=True, - offset=0, - page_size=10, - queue=None, - start_time=None, - status=None, - tag=None, - version=None): - """ - Returns a list of jobs for a given user that matches query params provided. - - Args: - algo_id (str, optional): Algorithm type. - end_time (str, optional): Specifying this parameter will return all jobs that have completed from the provided end time to now. e.g. 2024-01-01 or 2024-01-01T00:00:00.000000Z. - get_job_details (bool, optional): Flag that determines whether to return a detailed job list or a compact list containing just the job ids and their associated job tags. Default is True. - offset (int, optional): Offset for pagination. Default is 0. - page_size (int, optional): Page size for pagination. Default is 10. - queue (str, optional): Job processing resource. - start_time (str, optional): Specifying this parameter will return all jobs that have started from the provided start time to now. e.g. 2024-01-01 or 2024-01-01T00:00:00.000000Z. - status (str, optional): Job status, e.g. job-completed, job-failed, job-started, job-queued. - tag (str, optional): User job tag/identifier. - version (str, optional): Algorithm version, e.g. GitHub branch or tag. - - Returns: - list: List of jobs for a given user that matches query params provided. - - Raises: - ValueError: If either algo_id or version is provided, but not both. - """ - url = "/".join( - segment.strip("/") - for segment in (self.config.dps_job, endpoints.DPS_JOB_LIST) - ) - - params = { - k: v - for k, v in ( - ("algo_id", algo_id), - ("end_time", end_time), - ("get_job_details", get_job_details), - ("offset", offset), - ("page_size", page_size), - ("queue", queue), - ("start_time", start_time), - ("status", status), - ("tag", tag), - ("version", version), - ) - if v is not None - } - - if (not algo_id) != (not version): - # Either algo_id or version was supplied as a non-empty string, but not both. - # Either both must be non-empty strings or both must be None. - raise ValueError("Either supply non-empty strings for both algo_id and version, or supply neither.") - - # DPS requests use 'job_type', which is a concatenation of 'algo_id' and 'version' - if algo_id and version: - params['job_type'] = f"{algo_id}:{version}" - - algo_id = params.pop('algo_id', None) - version = params.pop('version', None) - - if status is not None: - params['status'] = job.validate_job_status(status) - - headers = self._get_api_header() - logger.debug('GET request sent to {}'.format(url)) - logger.debug('headers:') - logger.debug(headers) - response = requests.get( - url=url, - headers=headers, - params=params, - ) - return response - - def submitJob(self, identifier, algo_id, version, queue, retrieve_attributes=False, **kwargs): - # Note that this is temporary and will be removed when we remove the API not requiring username to submit a job - # Also this now overrides passing someone else's username into submitJob since we don't want to allow that - if self.profile is not None and self.profile.account_info() is not None and 'username' in self.profile.account_info().keys(): - kwargs['username'] = self.profile.account_info()['username'] - response = self._DPS.submit_job(request_url=self.config.dps_job, - identifier=identifier, algo_id=algo_id, version=version, queue=queue, **kwargs) - job = DPSJob(self.config) - job.set_submitted_job_result(response) - try: - if retrieve_attributes: - job.retrieve_attributes() - except: - logger.debug(f"Unable to retrieve attributes for job: {job}") - return job - - def uploadFiles(self, filenames): + def upload_files(self, filenames): """ Uploads files to a user-added staging directory. Enables users of maap-py to potentially share files generated on the MAAP. @@ -411,7 +231,7 @@ def show(self, granule, display_config={}): viz.show() # OGC-compliant endpoint functions - def list_processes_ogc(self): + def list_algorithms(self): """ Search all OGC processes :return: Response object with all deployed processes @@ -425,7 +245,7 @@ def list_processes_ogc(self): ) return response - def deploy_process_ogc(self, execution_unit_href): + def register_algorithm(self, execution_unit_href): """ Deploy a new OGC process :param execution_unit_href: URL to the CWL file @@ -445,7 +265,7 @@ def deploy_process_ogc(self, execution_unit_href): ) return response - def get_deployment_status_ogc(self, deployment_id): + def get_deployment_status(self, deployment_id): """ Query the current status of an algorithm being deployed :param deployment_id: The deployment job ID @@ -460,7 +280,7 @@ def get_deployment_status_ogc(self, deployment_id): ) return response - def describe_process_ogc(self, process_id): + def describe_algorithm(self, process_id): """ Get detailed information about a specific OGC process :param process_id: The process ID to describe @@ -474,7 +294,7 @@ def describe_process_ogc(self, process_id): ) return response - def update_process_ogc(self, process_id, execution_unit_href): + def update_algorithm(self, process_id, execution_unit_href): """ Replace an existing OGC process (must be the original deployer) :param process_id: The process ID to update @@ -496,7 +316,7 @@ def update_process_ogc(self, process_id, execution_unit_href): ) return response - def delete_process_ogc(self, process_id): + def delete_algorithm(self, process_id): """ Delete an existing OGC process (must be the original deployer) :param process_id: The process ID to delete @@ -511,7 +331,7 @@ def delete_process_ogc(self, process_id): ) return response - def get_process_package_ogc(self, process_id): + def get_algorithm_package(self, process_id): """ Access the formal description that can be used to deploy an OGC process :param process_id: The process ID @@ -526,7 +346,7 @@ def get_process_package_ogc(self, process_id): ) return response - def execute_process_ogc(self, process_id, inputs, queue, dedup=None, tag=None): + def submit_job(self, process_id, inputs, queue, dedup=None, tag=None): """ Execute an OGC process job :param process_id: The process ID to execute @@ -556,7 +376,7 @@ def execute_process_ogc(self, process_id, inputs, queue, dedup=None, tag=None): ) return response - def get_job_status_ogc(self, job_id): + def get_job_status(self, job_id): """ Get the status of an OGC job :param job_id: The job ID to check status for @@ -572,7 +392,7 @@ def get_job_status_ogc(self, job_id): ) return response - def cancel_job_ogc(self, job_id, wait_for_completion=False): + def cancel_job(self, job_id, wait_for_completion=False): """ Cancel a running OGC job or delete a queued job :param job_id: The job ID to cancel @@ -594,7 +414,7 @@ def cancel_job_ogc(self, job_id, wait_for_completion=False): ) return response - def get_job_results_ogc(self, job_id): + def get_job_result(self, job_id): """ Get the results of a completed OGC job :param job_id: The job ID to get results for @@ -609,18 +429,67 @@ def get_job_results_ogc(self, job_id): ) return response - def list_jobs_ogc(self, **kwargs): + def list_jobs(self, *, + algorithm_id=None, + limit=None, + get_job_details=True, + offset=0, + page_size=10, + queue=None, + status=None, + tag=None, + min_duration=None, + max_duration=None, + type=None, + datetime=None, + priority=None): """ - Get a list of OGC jobs with optional filtering - :param kwargs: Optional query parameters for filtering jobs - :return: Response object with list of jobs + Returns a list of jobs for a given user that matches query params provided. + + Args: + algorithm_id (str, optional): Algorithm ID to only show jobs submitted for this algorithm + limit (int, optional): Limit of jobs to send back + get_job_details (bool, optional): Flag that determines whether to return a detailed job list or a compact list containing just the job ids and their associated job tags. Default is True. + offset (int, optional): Offset for pagination. Default is 0. + page_size (int, optional): Page size for pagination. Default is 10. + queue (str, optional): Job processing resource. + status (str, optional): Job status, e.g. job-completed, job-failed, job-started, job-queued. + tag (str, optional): User job tag/identifier. + min_duration (int, optional): Minimum duration in seconds + max_duration (int, optional): Maximum duration in seconds + type (str, optional): Type, available values: process + datetime (str, optional): Either a date-time or an interval, half-bounded or bounded. Date and time expressions adhere to RFC 3339. Half-bounded intervals are expressed using double-dots. + priority (int, optional): Job priority, 0-9 + + Returns: + list: List of jobs for a given user that matches query params provided. + + Raises: + ValueError: If either algo_id or version is provided, but not both. """ + params = { + k: v + for k, v in ( + ("processID", algorithm_id), + ("limit", limit), + ("getJobDetails", get_job_details), + ("offset", offset), + ("pageSize", page_size), + ("queue", queue), + ("status", status), + ("tag", tag), + ("minDuration", min_duration), + ("maxDuration", max_duration), + ("type", type), + ("datetime", datetime), + ("priority", priority), + ) + if v is not None + } + url = os.path.join(self.config.jobs_ogc) headers = self._get_api_header() - # Filter out None values from kwargs - params = {k: v for k, v in kwargs.items() if v is not None} - logger.debug('GET request sent to {}'.format(url)) response = requests.get( url=url, @@ -629,7 +498,7 @@ def list_jobs_ogc(self, **kwargs): ) return response - def get_job_metrics_ogc(self, job_id): + def get_job_metrics(self, job_id): """ Get metrics for an OGC job :param job_id: The job ID to get metrics for From 77b88f7488efb29cc805212ac65e7d1cd7d990c5 Mon Sep 17 00:00:00 2001 From: grallewellyn Date: Thu, 21 Aug 2025 12:31:04 -0700 Subject: [PATCH 07/17] Updated changelog and tests, tests are not very helpful but api is already testing all the returns --- CHANGELOG.md | 1 + test/test_ogc.py | 472 +++++++++++++++++++++++++++++++++++++ test/test_ogc_functions.py | 199 ---------------- 3 files changed, 473 insertions(+), 199 deletions(-) create mode 100644 test/test_ogc.py delete mode 100644 test/test_ogc_functions.py diff --git a/CHANGELOG.md b/CHANGELOG.md index bb50b41..7937006 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed ### Deprecated ### Removed +- Removed functions calling old WPST endpoints ### Fixed ### Security diff --git a/test/test_ogc.py b/test/test_ogc.py new file mode 100644 index 0000000..84d3252 --- /dev/null +++ b/test/test_ogc.py @@ -0,0 +1,472 @@ +""" +Test module for algorithm and job functions in maap.py +""" + +import pytest +from maap.maap import MAAP + + +def test_list_algorithms(): + """Test list_algorithms function calls OGC algorithms endpoint and returns 200 with JSON""" + maap = MAAP(maap_host='api.dit.maap-project.org') + + response = maap.list_algorithms() + + # Check that we get a 200 status code + assert response.status_code == 200, f"Expected 200, got {response.status_code}" + + # Check that response is valid JSON + try: + json_data = response.json() + assert isinstance(json_data, (dict, list)), "Response should be valid JSON (dict or list)" + except ValueError as e: + pytest.fail(f"Response is not valid JSON: {e}") + + +def test_register_algorithm(): + """Test register_algorithm function with a valid CWL URL""" + maap = MAAP() + + # Skip test if we can't authenticate + try: + # Test that list_algorithms works first to ensure we have proper authentication + list_response = maap.list_algorithms() + if list_response.status_code != 200: + pytest.skip("Authentication required - skipping register_algorithm test") + except Exception: + pytest.skip("Cannot connect to MAAP API - skipping register_algorithm test") + + # Use a real CWL example URL that should work + sample_cwl_url = "https://raw.githubusercontent.com/MAAP-Project/maap-algorithms/master/examples/hello-world/hello-world.cwl" + + response = maap.register_algorithm(sample_cwl_url) + + # Should get a successful response or a meaningful error + assert response.status_code in [200, 201], f"Expected successful registration, got {response.status_code}: {response.text}" + + # Should return JSON with deployment info + json_data = response.json() + assert isinstance(json_data, dict), "Registration response should be a JSON object" + + # Should contain deployment information + assert "deploymentID" in json_data or "id" in json_data, "Response should contain deployment ID" + + +def test_get_deployment_status(): + """Test get_deployment_status function""" + maap = MAAP() + + # Skip test if we can't authenticate + try: + list_response = maap.list_algorithms() + if list_response.status_code != 200: + pytest.skip("Authentication required - skipping get_deployment_status test") + except Exception: + pytest.skip("Cannot connect to MAAP API - skipping get_deployment_status test") + + # Since we don't have a real deployment ID, this test will likely return 404 + # which is the expected behavior for a non-existent deployment + sample_deployment_id = "test-deployment-123" + + response = maap.get_deployment_status(sample_deployment_id) + + # Should get a valid response - 200 if found, 404 if not found + assert response.status_code in [200, 404], f"Expected 200 or 404, got {response.status_code}: {response.text}" + + # If deployment exists (200), should return JSON with status info + if response.status_code == 200: + json_data = response.json() + assert isinstance(json_data, dict), "Status response should be a JSON object" + assert "status" in json_data, "Response should contain status information" + + # Verify the URL contains the deployment ID + assert str(sample_deployment_id) in response.url + + +def test_describe_algorithm(): + """Test describe_algorithm function by getting algorithm list and describing first algorithm""" + maap = MAAP(maap_host='api.dit.maap-project.org') + + # First get the list of algorithms + list_response = maap.list_algorithms() + assert list_response.status_code == 200, f"Failed to get algorithm list: {list_response.status_code}" + + try: + processes_data = list_response.json() + except ValueError as e: + pytest.fail(f"Algorithm list response is not valid JSON: {e}") + + # Check if there are any algorithms + if not processes_data or (isinstance(processes_data, dict) and not processes_data.get('processes')): + pytest.skip("No algorithms available to test describe_algorithm") + + # Get the first algorithm + if isinstance(processes_data, dict) and 'processes' in processes_data: + processes = processes_data['processes'] + else: + processes = processes_data + + if not processes or len(processes) == 0: + pytest.skip("No algorithms available to test describe_algorithm") + + first_process = processes[0] + + # Find the self link or use process ID + process_id = None + if 'links' in first_process: + for link in first_process['links']: + if link.get('rel') == 'self': + href = link.get('href', '') + # Extract process ID from href like /ogc/processes/3 + if '/ogc/processes/' in href: + process_id = href.split('/ogc/processes/')[-1] + break + + # Fall back to process ID field if no self link found + if not process_id and 'id' in first_process: + process_id = first_process['id'] + + if not process_id: + pytest.skip("Could not determine algorithm ID to test describe_algorithm") + + # Now test the describe_algorithm function + describe_response = maap.describe_algorithm(process_id) + + # Check that we get a successful response + assert describe_response.status_code == 200, f"Expected 200, got {describe_response.status_code}" + + # Check that response is valid JSON + try: + describe_data = describe_response.json() + assert isinstance(describe_data, dict), "Describe response should be a JSON object" + except ValueError as e: + pytest.fail(f"Describe response is not valid JSON: {e}") + + # Verify the URL called contains the algorithm ID + assert str(process_id) in describe_response.url + + +def test_update_algorithm(): + """Test update_algorithm function""" + maap = MAAP() + + # Skip test if we can't authenticate + try: + list_response = maap.list_algorithms() + if list_response.status_code != 200: + pytest.skip("Authentication required - skipping update_algorithm test") + except Exception: + pytest.skip("Cannot connect to MAAP API - skipping update_algorithm test") + + # Use a non-existent algorithm ID - should return 404 which is expected + sample_process_id = "non-existent-algorithm-123" + sample_cwl_url = "https://raw.githubusercontent.com/MAAP-Project/maap-algorithms/master/examples/hello-world/hello-world.cwl" + + response = maap.update_algorithm(sample_process_id, sample_cwl_url) + + # Should get a valid response - 200 if successful, 404 if not found, 403 if not authorized + assert response.status_code in [200, 404, 403], f"Expected 200, 404, or 403, got {response.status_code}: {response.text}" + + # If successful (200), should return JSON with update info + if response.status_code == 200: + json_data = response.json() + assert isinstance(json_data, dict), "Update response should be a JSON object" + + # Verify the URL contains the process ID + assert str(sample_process_id) in response.url + + +def test_delete_algorithm(): + """Test delete_algorithm function""" + maap = MAAP() + + # Skip test if we can't authenticate + try: + list_response = maap.list_algorithms() + if list_response.status_code != 200: + pytest.skip("Authentication required - skipping delete_algorithm test") + except Exception: + pytest.skip("Cannot connect to MAAP API - skipping delete_algorithm test") + + # Use a non-existent algorithm ID - should return 404 which is expected + sample_process_id = "non-existent-algorithm-123" + + response = maap.delete_algorithm(sample_process_id) + + # Should get a valid response - 200/204 if successful, 404 if not found, 403 if not authorized + assert response.status_code in [200, 204, 404, 403], f"Expected 200, 204, 404, or 403, got {response.status_code}: {response.text}" + + # If successful (200/204), response might be empty or contain JSON + if response.status_code in [200, 204]: + if response.content: # Only check JSON if there's content + json_data = response.json() + assert isinstance(json_data, dict), "Delete response should be a JSON object" + + # Verify the URL contains the process ID + assert str(sample_process_id) in response.url + + +def test_get_algorithm_package(): + """Test get_algorithm_package function""" + maap = MAAP(maap_host='api.dit.maap-project.org') + + # First get the list of algorithms + list_response = maap.list_algorithms() + assert list_response.status_code == 200, f"Failed to get algorithm list: {list_response.status_code}" + + try: + processes_data = list_response.json() + except ValueError as e: + pytest.fail(f"Algorithm list response is not valid JSON: {e}") + + # Check if there are any algorithms + if not processes_data or (isinstance(processes_data, dict) and not processes_data.get('processes')): + pytest.skip("No algorithms available to test get_algorithm_package") + + # Get the first algorithm + if isinstance(processes_data, dict) and 'processes' in processes_data: + processes = processes_data['processes'] + else: + processes = processes_data + + if not processes or len(processes) == 0: + pytest.skip("No algorithms available to test get_algorithm_package") + + first_process = processes[0] + + # Find the self link or use process ID + process_id = None + if 'links' in first_process: + for link in first_process['links']: + if link.get('rel') == 'self': + href = link.get('href', '') + # Extract process ID from href like /ogc/processes/3 + if '/ogc/processes/' in href: + process_id = href.split('/ogc/processes/')[-1] + break + + # Fall back to process ID field if no self link found + if not process_id and 'id' in first_process: + process_id = first_process['id'] + + if not process_id: + pytest.skip("Could not determine algorithm ID to test get_algorithm_package") + + # Now test the package_response function + package_response = maap.get_algorithm_package(process_id) + + # Check that we get a successful response + assert package_response.status_code == 200, f"Expected 200, got {package_response.status_code}" + + # Check that response is valid JSON + try: + package_data = package_response.json() + assert isinstance(package_data, dict), "Algorithm Package response should be a JSON object" + except ValueError as e: + pytest.fail(f"Algorithm package response is not valid JSON: {e}") + + # Verify the URL called contains the algorithm ID + assert str(process_id) in package_response.url + + +def test_submit_job(): + """Test submit_job function by first getting a real algorithm ID""" + maap = MAAP() + + # Skip test if we can't authenticate + try: + list_response = maap.list_algorithms() + if list_response.status_code != 200: + pytest.skip("Authentication required - skipping submit_job test") + except Exception: + pytest.skip("Cannot connect to MAAP API - skipping submit_job test") + + # Get a real algorithm to test with + try: + algorithms_data = list_response.json() + if not algorithms_data or (isinstance(algorithms_data, dict) and not algorithms_data.get('processes')): + pytest.skip("No algorithms available to test submit_job") + + if isinstance(algorithms_data, dict) and 'processes' in algorithms_data: + algorithms = algorithms_data['processes'] + else: + algorithms = algorithms_data + + if not algorithms or len(algorithms) == 0: + pytest.skip("No algorithms available to test submit_job") + + # Get the first algorithm's ID + first_algorithm = algorithms[0] + algorithm_id = first_algorithm.get('id') or first_algorithm.get('processId') + + if not algorithm_id: + pytest.skip("Could not determine algorithm ID to test submit_job") + + except Exception as e: + pytest.skip(f"Could not parse algorithms list: {e}") + + # Test job submission with minimal inputs + sample_inputs = {} # Empty inputs for basic test + sample_queue = "maap-dps-worker-32gb" # Use a real queue name + + response = maap.submit_job(algorithm_id, sample_inputs, sample_queue) + + # Should get a response - 200/201 if successful, 400 if invalid inputs, 404 if algorithm not found + assert response.status_code in [200, 201, 400, 404], f"Expected valid response, got {response.status_code}: {response.text}" + + # If successful (200/201), should return JSON with job info + if response.status_code in [200, 201]: + json_data = response.json() + assert isinstance(json_data, dict), "Job submission response should be a JSON object" + assert "jobID" in json_data or "id" in json_data, "Response should contain job ID" + + # Verify the URL contains the algorithm ID + assert str(algorithm_id) in response.url + + +def test_get_job_status(): + """Test get_job_status function""" + maap = MAAP() + + # Skip test if we can't authenticate + try: + list_response = maap.list_jobs() + if list_response.status_code != 200: + pytest.skip("Authentication required - skipping get_job_status test") + except Exception: + pytest.skip("Cannot connect to MAAP API - skipping get_job_status test") + + # Use a non-existent job ID - should return 404 which is expected + sample_job_id = "non-existent-job-123" + + response = maap.get_job_status(sample_job_id) + + # Should get a valid response - 200 if found, 404 if not found + assert response.status_code in [200, 404], f"Expected 200 or 404, got {response.status_code}: {response.text}" + + # If job exists (200), should return JSON with status info + if response.status_code == 200: + json_data = response.json() + assert isinstance(json_data, dict), "Job status response should be a JSON object" + assert "status" in json_data, "Response should contain status information" + + # Verify the URL contains the job ID + assert str(sample_job_id) in response.url + + +def test_cancel_job(): + """Test cancel_job function""" + maap = MAAP() + + # Skip test if we can't authenticate + try: + list_response = maap.list_jobs() + if list_response.status_code != 200: + pytest.skip("Authentication required - skipping cancel_job test") + except Exception: + pytest.skip("Cannot connect to MAAP API - skipping cancel_job test") + + # Use a non-existent job ID - should return 404 which is expected + sample_job_id = "non-existent-job-123" + + response = maap.cancel_job(sample_job_id) + + # Should get a valid response - 200/204 if successful, 404 if not found, 409 if already completed + assert response.status_code in [200, 204, 404, 409], f"Expected 200, 204, 404, or 409, got {response.status_code}: {response.text}" + + # If successful (200/204), response might be empty or contain JSON + if response.status_code in [200, 204]: + if response.content: # Only check JSON if there's content + json_data = response.json() + assert isinstance(json_data, dict), "Cancel response should be a JSON object" + + # Verify the URL contains the job ID + assert str(sample_job_id) in response.url + + +def test_get_job_result(): + """Test get_job_result function""" + maap = MAAP() + + # Skip test if we can't authenticate + try: + list_response = maap.list_jobs() + if list_response.status_code != 200: + pytest.skip("Authentication required - skipping get_job_result test") + except Exception: + pytest.skip("Cannot connect to MAAP API - skipping get_job_result test") + + # Use a non-existent job ID - should return 404 which is expected + sample_job_id = "non-existent-job-123" + + response = maap.get_job_result(sample_job_id) + + # Should get a valid response - 200 if found, 404 if not found, 425 if not ready + assert response.status_code in [200, 404, 425], f"Expected 200, 404, or 425, got {response.status_code}: {response.text}" + + # If job results exist (200), should return JSON with result info + if response.status_code == 200: + json_data = response.json() + assert isinstance(json_data, dict), "Job result response should be a JSON object" + # Should contain outputs or links to result files + + # Verify the URL contains the job ID and 'results' + assert str(sample_job_id) in response.url + assert 'results' in response.url + + +def test_list_jobs(): + """Test list_jobs function""" + maap = MAAP() + + # Skip test if we can't authenticate + try: + response = maap.list_jobs() + if response.status_code != 200: + pytest.skip("Authentication required - skipping get_job_result test") + except Exception: + pytest.skip("Cannot connect to MAAP API - skipping get_job_result test") + + # Only check JSON content if we get a successful response + if response.status_code == 200: + json_data = response.json() + assert isinstance(json_data, (dict, list)), "Jobs list response should be JSON (dict or list)" + + # If it's a dict, it might have a 'jobs' key or similar + if isinstance(json_data, dict): + # Common structures: {"jobs": [...]} or {"processes": [...]} + assert len(json_data) >= 0, "Jobs response should be valid" + elif isinstance(json_data, list): + # Direct list of jobs + assert len(json_data) >= 0, "Jobs list should be valid" + + +def test_get_job_metrics(): + """Test get_job_metrics function""" + maap = MAAP() + + # Skip test if we can't authenticate + try: + list_response = maap.list_jobs() + if list_response.status_code != 200: + pytest.skip("Authentication required - skipping get_job_metrics test") + except Exception: + pytest.skip("Cannot connect to MAAP API - skipping get_job_metrics test") + + # Use a non-existent job ID - should return 404 which is expected + sample_job_id = "non-existent-job-123" + + response = maap.get_job_metrics(sample_job_id) + + # Should get a valid response - 200 if found, 404 if not found, 425 if not available + assert response.status_code in [200, 404, 425], f"Expected 200, 404, or 425, got {response.status_code}: {response.text}" + + # If job metrics exist (200), should return JSON with metrics info + if response.status_code == 200: + json_data = response.json() + assert isinstance(json_data, dict), "Job metrics response should be a JSON object" + # Should contain metrics like CPU usage, memory usage, duration, etc. + + # Verify the URL contains the job ID and 'metrics' + assert str(sample_job_id) in response.url + assert 'metrics' in response.url \ No newline at end of file diff --git a/test/test_ogc_functions.py b/test/test_ogc_functions.py deleted file mode 100644 index b27b5a5..0000000 --- a/test/test_ogc_functions.py +++ /dev/null @@ -1,199 +0,0 @@ -""" -Test module for OGC functions in maap.py -""" - -import pytest -from maap.maap import MAAP - - -def test_list_processes_ogc(): - """Test list_processes_ogc function calls OGC processes endpoint and returns 200 with JSON""" - maap = MAAP(maap_host='api.dit.maap-project.org') - - response = maap.list_processes_ogc() - - # Check that we get a 200 status code - assert response.status_code == 200, f"Expected 200, got {response.status_code}" - - # Check that response is valid JSON - try: - json_data = response.json() - assert isinstance(json_data, (dict, list)), "Response should be valid JSON (dict or list)" - except ValueError as e: - pytest.fail(f"Response is not valid JSON: {e}") - - -def test_deploy_process_ogc(): - """Test deploy_process_ogc function""" - pass - - -def test_get_deployment_status_ogc(): - """Test get_deployment_status_ogc function""" - pass - - -def test_describe_process_ogc(): - """Test describe_process_ogc function by getting process list and describing first process""" - maap = MAAP(maap_host='api.dit.maap-project.org') - - # First get the list of processes - list_response = maap.list_processes_ogc() - assert list_response.status_code == 200, f"Failed to get process list: {list_response.status_code}" - - try: - processes_data = list_response.json() - except ValueError as e: - pytest.fail(f"Process list response is not valid JSON: {e}") - - # Check if there are any processes - if not processes_data or (isinstance(processes_data, dict) and not processes_data.get('processes')): - pytest.skip("No processes available to test describe_process_ogc") - - # Get the first process - if isinstance(processes_data, dict) and 'processes' in processes_data: - processes = processes_data['processes'] - else: - processes = processes_data - - if not processes or len(processes) == 0: - pytest.skip("No processes available to test describe_process_ogc") - - first_process = processes[0] - - # Find the self link or use process ID - process_id = None - if 'links' in first_process: - for link in first_process['links']: - if link.get('rel') == 'self': - href = link.get('href', '') - # Extract process ID from href like /ogc/processes/3 - if '/ogc/processes/' in href: - process_id = href.split('/ogc/processes/')[-1] - break - - # Fall back to process ID field if no self link found - if not process_id and 'id' in first_process: - process_id = first_process['id'] - - if not process_id: - pytest.skip("Could not determine process ID to test describe_process_ogc") - - # Now test the describe_process_ogc function - describe_response = maap.describe_process_ogc(process_id) - - # Check that we get a successful response - assert describe_response.status_code == 200, f"Expected 200, got {describe_response.status_code}" - - # Check that response is valid JSON - try: - describe_data = describe_response.json() - assert isinstance(describe_data, dict), "Describe response should be a JSON object" - except ValueError as e: - pytest.fail(f"Describe response is not valid JSON: {e}") - - # Verify the URL called contains the process ID - assert str(process_id) in describe_response.url - - -def test_update_process_ogc(): - """Test update_process_ogc function""" - pass - - -def test_delete_process_ogc(): - """Test delete_process_ogc function""" - pass - - -def test_get_process_package_ogc(): - """Test get_process_package_ogc function""" - maap = MAAP(maap_host='api.dit.maap-project.org') - - # First get the list of processes - list_response = maap.list_processes_ogc() - assert list_response.status_code == 200, f"Failed to get process list: {list_response.status_code}" - - try: - processes_data = list_response.json() - except ValueError as e: - pytest.fail(f"Process list response is not valid JSON: {e}") - - # Check if there are any processes - if not processes_data or (isinstance(processes_data, dict) and not processes_data.get('processes')): - pytest.skip("No processes available to test describe_process_ogc") - - # Get the first process - if isinstance(processes_data, dict) and 'processes' in processes_data: - processes = processes_data['processes'] - else: - processes = processes_data - - if not processes or len(processes) == 0: - pytest.skip("No processes available to test describe_process_ogc") - - first_process = processes[0] - - # Find the self link or use process ID - process_id = None - if 'links' in first_process: - for link in first_process['links']: - if link.get('rel') == 'self': - href = link.get('href', '') - # Extract process ID from href like /ogc/processes/3 - if '/ogc/processes/' in href: - process_id = href.split('/ogc/processes/')[-1] - break - - # Fall back to process ID field if no self link found - if not process_id and 'id' in first_process: - process_id = first_process['id'] - - if not process_id: - pytest.skip("Could not determine process ID to test describe_process_ogc") - - # Now test the package_response function - package_response = maap.get_process_package_ogc(process_id) - - # Check that we get a successful response - assert package_response.status_code == 200, f"Expected 200, got {package_response.status_code}" - - # Check that response is valid JSON - try: - package_data = package_response.json() - assert isinstance(package_data, dict), "Process Package response should be a JSON object" - except ValueError as e: - pytest.fail(f"Process package response is not valid JSON: {e}") - - # Verify the URL called contains the process ID - assert str(process_id) in package_response.url - - -def test_execute_process_ogc(): - """Test execute_process_ogc function""" - pass - - -def test_get_job_status_ogc(): - """Test get_job_status_ogc function""" - pass - - -def test_cancel_job_ogc(): - """Test cancel_job_ogc function""" - pass - - -def test_get_job_results_ogc(): - """Test get_job_results_ogc function""" - pass - - -def test_list_jobs_ogc(): - """Test list_jobs_ogc function""" - pass - - -def test_get_job_metrics_ogc(): - """Test get_job_metrics_ogc function""" - pass \ No newline at end of file From d59f5408d78c81acc58d3673547f794c7c9654d6 Mon Sep 17 00:00:00 2001 From: grallewellyn Date: Wed, 24 Sep 2025 14:24:40 -0700 Subject: [PATCH 08/17] corrected list jobs function --- maap/maap.py | 43 +++++++++++++++++-------------------------- 1 file changed, 17 insertions(+), 26 deletions(-) diff --git a/maap/maap.py b/maap/maap.py index e084d59..4cdcba8 100644 --- a/maap/maap.py +++ b/maap/maap.py @@ -145,29 +145,20 @@ def get_queues(self): ) return response - def register_algorithm_from_yaml_file(self, file_path): - algo_config = algorithm_utils.read_yaml_file(file_path) - return self.register_algorithm(algo_config) - - def register_algorithm_from_yaml_file_backwards_compatible(self, file_path): - algo_yaml = algorithm_utils.read_yaml_file(file_path) - key_map = {"algo_name": "algorithm_name", "version": "code_version", "environment": "environment_name", - "description": "algorithm_description", "docker_url": "docker_container_url", - "inputs": "algorithm_params", "run_command": "script_command", "repository_url": "repo_url"} - output_config = {} - for key, value in algo_yaml.items(): - if key in key_map: - if key == "inputs": - inputs = [] - for argument in value: - inputs.append({"field": argument.get("name"), "download": argument.get("download")}) - output_config.update({"algorithm_params": inputs}) - else: - output_config.update({key_map.get(key): value}) - else: - output_config.update({key: value}) - logger.debug("Registering with config %s " % json.dumps(output_config)) - return self.register_algorithm(json.dumps(output_config)) + def register_algorithm_from_cwl_file(self, file_path): + """ + Registers an algorithm from a CWL file + """ + # Read cwl file returns a dict in the format to register an algorithm without a CWL + algo_config = algorithm_utils.read_cwl_file(file_path) + headers = self._get_api_header(content_type='application/json') + logger.debug('POST request sent to {}'.format(self.config.processes_ogc)) + response = requests.post( + url=self.config.processes_ogc, + headers=headers, + json=algo_config + ) + return response def get_job(self, jobid): job = DPSJob(self.config) @@ -430,7 +421,7 @@ def get_job_result(self, job_id): return response def list_jobs(self, *, - algorithm_id=None, + process_id=None, limit=None, get_job_details=True, offset=0, @@ -447,7 +438,7 @@ def list_jobs(self, *, Returns a list of jobs for a given user that matches query params provided. Args: - algorithm_id (str, optional): Algorithm ID to only show jobs submitted for this algorithm + process_id (id, optional): Algorithm ID to only show jobs submitted for this algorithm limit (int, optional): Limit of jobs to send back get_job_details (bool, optional): Flag that determines whether to return a detailed job list or a compact list containing just the job ids and their associated job tags. Default is True. offset (int, optional): Offset for pagination. Default is 0. @@ -470,7 +461,7 @@ def list_jobs(self, *, params = { k: v for k, v in ( - ("processID", algorithm_id), + ("processID", process_id), ("limit", limit), ("getJobDetails", get_job_details), ("offset", offset), From d529a2b2a4ef278cefb4449faa217e8759673163 Mon Sep 17 00:00:00 2001 From: grallewellyn Date: Mon, 6 Oct 2025 15:02:54 -0700 Subject: [PATCH 09/17] commented out register_algorithm_from_cwl_file because its functionality is not implemented yet --- maap/maap.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/maap/maap.py b/maap/maap.py index 4cdcba8..12d985c 100644 --- a/maap/maap.py +++ b/maap/maap.py @@ -145,20 +145,20 @@ def get_queues(self): ) return response - def register_algorithm_from_cwl_file(self, file_path): - """ - Registers an algorithm from a CWL file - """ - # Read cwl file returns a dict in the format to register an algorithm without a CWL - algo_config = algorithm_utils.read_cwl_file(file_path) - headers = self._get_api_header(content_type='application/json') - logger.debug('POST request sent to {}'.format(self.config.processes_ogc)) - response = requests.post( - url=self.config.processes_ogc, - headers=headers, - json=algo_config - ) - return response + # def register_algorithm_from_cwl_file(self, file_path): + # """ + # Registers an algorithm from a CWL file + # """ + # # Read cwl file returns a dict in the format to register an algorithm without a CWL + # algo_config = algorithm_utils.read_cwl_file(file_path) + # headers = self._get_api_header(content_type='application/json') + # logger.debug('POST request sent to {}'.format(self.config.processes_ogc)) + # response = requests.post( + # url=self.config.processes_ogc, + # headers=headers, + # json=algo_config + # ) + # return response def get_job(self, jobid): job = DPSJob(self.config) From c5977d95e24649e9c832bd1344d7fa60599bac7b Mon Sep 17 00:00:00 2001 From: grallewellyn Date: Wed, 12 Nov 2025 13:07:03 -0800 Subject: [PATCH 10/17] added functions for registering an algorithm from a cwl file --- maap/maap.py | 10 +++++----- maap/utils/algorithm_utils.py | 23 +++++++++++++++++++++++ 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/maap/maap.py b/maap/maap.py index 12d985c..d2dac91 100644 --- a/maap/maap.py +++ b/maap/maap.py @@ -160,11 +160,11 @@ def get_queues(self): # ) # return response - def get_job(self, jobid): - job = DPSJob(self.config) - job.id = jobid - job.retrieve_attributes() - return job + # def get_job(self, jobid): + # job = DPSJob(self.config) + # job.id = jobid + # job.retrieve_attributes() + # return job def upload_files(self, filenames): """ diff --git a/maap/utils/algorithm_utils.py b/maap/utils/algorithm_utils.py index db12dd6..1717c84 100644 --- a/maap/utils/algorithm_utils.py +++ b/maap/utils/algorithm_utils.py @@ -18,6 +18,29 @@ def read_yaml_file(algo_yaml): algo_config = yaml_load(fr, Loader=Loader) return validate_algorithm_config(algo_config) +def read_cwl_file(algo_cwl): + """ + Parse through the CWL file and returns the response as the POST to register a + a process for OGC is expecting + https://github.com/MAAP-Project/joint-open-api-specs/blob/nasa-adaptation/ogc-api-processes/openapi-template/schemas/processes-core/postProcess.yaml + """ + try: + with open(algo_cwl, 'r') as f: + cwl_data = yaml.safe_load(f) + print(f"Successfully read and parsed '{algo_cwl}'") + return parse_cwl_data(cwl_data) + except FileNotFoundError: + print(f"Error: The file '{algo_cwl}' was not found.") + return None + except yaml.YAMLError as e: + print(f"Error parsing the YAML in '{algo_cwl}': {e}") + return None + +def parse_cwl_data(cwl_data): + algo_config = dict() + # TODO implement this and return cwl_data as a dictionary with important variables like + # the API is expecting + return algo_config def validate_algorithm_config(algo_config): return algo_config From 462b4dc374e39ae418cabe250959dfb160a0ce87 Mon Sep 17 00:00:00 2001 From: grallewellyn Date: Thu, 8 Jan 2026 10:55:28 -0800 Subject: [PATCH 11/17] converted more function names to snakecase --- CHANGELOG.md | 2 +- README.md | 6 +- docs/api/dps.md | 2 +- docs/api/maap.md | 6 +- docs/api/result.md | 2 +- docs/index.md | 4 +- examples/BrowseExample.ipynb | 4 +- ...earch Collection - Basics-checkpoint.ipynb | 2 +- examples/Search Granule-checkpoint.ipynb | 4 +- maap/AWS.py | 2 +- maap/Profile.py | 2 +- maap/Result.py | 10 +-- maap/__init__.py | 4 +- maap/dps/dps_job.py | 14 ++-- maap/maap.py | 74 +++++++++---------- maap/utils/CMR.py | 8 +- test/functional_test.py | 2 +- test/test_CMR.py | 54 +++++++------- test/test_DPS.py | 61 --------------- test/test_MAAP.py | 8 +- 20 files changed, 105 insertions(+), 166 deletions(-) delete mode 100644 test/test_DPS.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 7937006..bef2003 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,7 +19,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added ### Changed - listJobs no longer takes username as an argument, you can only list jobs for the current `MAAP_PGT` token user -- submitJob gets the username from the `MAAP_PGT` token and not username being submitted as an argument +- submit_job gets the username from the `MAAP_PGT` token and not username being submitted as an argument ### Deprecated ### Removed ### Fixed diff --git a/README.md b/README.md index b632fa1..189036c 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ $ python >>> from maap.maap import MAAP >>> maap = MAAP() ->>> granules = maap.searchGranule(sitename='lope', instrument='uavsar') +>>> granules = maap.search_granule(sitename='lope', instrument='uavsar') >>> for res in granules: print(res.getDownloadUrl()) res.download() @@ -64,13 +64,13 @@ where: With named attribute parameters, this query: ```python -lidarGranule = maap.searchGranule(instrument='lvis', attribute='string,Site Name,lope') +lidarGranule = maap.search_granule(instrument='lvis', attribute='string,Site Name,lope') ``` Simplifies to: ```python -lidarGranule = maap.searchGranule(instrument='lvis', site_name='lope') +lidarGranule = maap.search_granule(instrument='lvis', site_name='lope') ``` ## Test diff --git a/docs/api/dps.md b/docs/api/dps.md index 3a30b6e..33ece6c 100644 --- a/docs/api/dps.md +++ b/docs/api/dps.md @@ -32,7 +32,7 @@ from maap.maap import MAAP maap = MAAP() # Submit a job -job = maap.submitJob( +job = maap.submit_job( identifier='my_analysis', algo_id='my_algorithm', version='main', diff --git a/docs/api/maap.md b/docs/api/maap.md index 5b06c20..2f28189 100644 --- a/docs/api/maap.md +++ b/docs/api/maap.md @@ -19,13 +19,13 @@ from maap.maap import MAAP maap = MAAP() # Search granules -granules = maap.searchGranule(short_name='GEDI02_A', limit=10) +granules = maap.search_granule(short_name='GEDI02_A', limit=10) # Search collections -collections = maap.searchCollection(provider='MAAP') +collections = maap.search_collection(provider='MAAP') # Submit a job -job = maap.submitJob( +job = maap.submit_job( identifier='analysis', algo_id='my_algo', version='main', diff --git a/docs/api/result.md b/docs/api/result.md index 9cbafa3..6a677c2 100644 --- a/docs/api/result.md +++ b/docs/api/result.md @@ -49,7 +49,7 @@ from maap.maap import MAAP maap = MAAP() # Search granules -granules = maap.searchGranule(short_name='GEDI02_A', limit=5) +granules = maap.search_granule(short_name='GEDI02_A', limit=5) for granule in granules: # Get URLs diff --git a/docs/index.md b/docs/index.md index c1bc030..402a72c 100644 --- a/docs/index.md +++ b/docs/index.md @@ -30,7 +30,7 @@ from maap.maap import MAAP maap = MAAP() # Search for granules -granules = maap.searchGranule( +granules = maap.search_granule( short_name='GEDI02_A', bounding_box='-122.5,37.5,-121.5,38.5', limit=10 @@ -42,7 +42,7 @@ for granule in granules: print(f"Downloaded: {local_path}") # Submit a job -job = maap.submitJob( +job = maap.submit_job( identifier='my_analysis', algo_id='my_algorithm', version='main', diff --git a/examples/BrowseExample.ipynb b/examples/BrowseExample.ipynb index a18d022..6f0ca7e 100644 --- a/examples/BrowseExample.ipynb +++ b/examples/BrowseExample.ipynb @@ -386,7 +386,7 @@ } ], "source": [ - "granule = maap.searchGranule(granule_ur='uavsar_AfriSAR_v1_SLC-lopenp_14043_16015_001_160308_L090.vrt')[0]\n", + "granule = maap.search_granule(granule_ur='uavsar_AfriSAR_v1_SLC-lopenp_14043_16015_001_160308_L090.vrt')[0]\n", "maap.show(granule)" ] }, @@ -747,7 +747,7 @@ } ], "source": [ - "granule = maap.searchGranule(granule_ur='ILVIS2_GA2016_0220_R1611_038024')[0]\n", + "granule = maap.search_granule(granule_ur='ILVIS2_GA2016_0220_R1611_038024')[0]\n", "maap.show(granule)" ] } diff --git a/examples/Search Collection - Basics-checkpoint.ipynb b/examples/Search Collection - Basics-checkpoint.ipynb index 49f3ddf..544c247 100644 --- a/examples/Search Collection - Basics-checkpoint.ipynb +++ b/examples/Search Collection - Basics-checkpoint.ipynb @@ -53,7 +53,7 @@ "metadata": {}, "outputs": [], "source": [ - "results = maap.searchCollection(keyword='precipitation')" + "results = maap.search_collection(keyword='precipitation')" ] }, { diff --git a/examples/Search Granule-checkpoint.ipynb b/examples/Search Granule-checkpoint.ipynb index 11c7f47..d3eb1bd 100644 --- a/examples/Search Granule-checkpoint.ipynb +++ b/examples/Search Granule-checkpoint.ipynb @@ -63,7 +63,7 @@ } ], "source": [ - "results = maap.searchCollection(keyword='land',data_center='modaps')\n", + "results = maap.search_collection(keyword='land',data_center='modaps')\n", "print(len(results))\n", "\n", "for res in results:\n", @@ -159,7 +159,7 @@ } ], "source": [ - "results = maap.searchGranule(limit=10,short_name=\"MOD11A1\")\n", + "results = maap.search_granule(limit=10,short_name=\"MOD11A1\")\n", "\n", "print(len(results))\n", "for res in results:\n", diff --git a/maap/AWS.py b/maap/AWS.py index 3803c84..fe2a53f 100644 --- a/maap/AWS.py +++ b/maap/AWS.py @@ -336,7 +336,7 @@ def workspace_bucket_credentials(self): See Also -------- :meth:`requester_pays_credentials` : For accessing external data - :meth:`maap.maap.MAAP.uploadFiles` : Upload files to shared storage + :meth:`maap.maap.MAAP.upload_files` : Upload files to shared storage """ headers = self._api_header headers["Accept"] = "application/json" diff --git a/maap/Profile.py b/maap/Profile.py index 3bc2ce5..eb3a62d 100644 --- a/maap/Profile.py +++ b/maap/Profile.py @@ -100,7 +100,7 @@ def account_info(self, proxy_ticket=None): Notes ----- - This method is used internally by :meth:`~maap.maap.MAAP.submitJob` + This method is used internally by :meth:`~maap.maap.MAAP.submit_job` to automatically include the username with job submissions. See Also diff --git a/maap/Result.py b/maap/Result.py index 2f96012..39e4abb 100644 --- a/maap/Result.py +++ b/maap/Result.py @@ -21,7 +21,7 @@ from maap.maap import MAAP maap = MAAP() - granules = maap.searchGranule(short_name='GEDI02_A', limit=5) + granules = maap.search_granule(short_name='GEDI02_A', limit=5) for granule in granules: # Get download URLs @@ -439,7 +439,7 @@ class Collection(Result): -------- Search for collections:: - >>> collections = maap.searchCollection(short_name='GEDI02_A') + >>> collections = maap.search_collection(short_name='GEDI02_A') >>> for c in collections: ... print(c['Collection']['ShortName']) ... print(c['Collection']['Description']) @@ -458,7 +458,7 @@ class Collection(Result): See Also -------- :class:`Granule` : Individual data file results - :meth:`maap.maap.MAAP.searchCollection` : Search for collections + :meth:`maap.maap.MAAP.search_collection` : Search for collections """ def __init__(self, metaResult, maap_host): @@ -512,7 +512,7 @@ class Granule(Result): -------- Search and access granule metadata:: - >>> granules = maap.searchGranule(short_name='GEDI02_A', limit=5) + >>> granules = maap.search_granule(short_name='GEDI02_A', limit=5) >>> granule = granules[0] >>> print(granule['Granule']['GranuleUR']) @@ -540,7 +540,7 @@ class Granule(Result): See Also -------- :class:`Collection` : Dataset metadata results - :meth:`maap.maap.MAAP.searchGranule` : Search for granules + :meth:`maap.maap.MAAP.search_granule` : Search for granules """ def __init__( diff --git a/maap/__init__.py b/maap/__init__.py index a436ba3..adf9538 100644 --- a/maap/__init__.py +++ b/maap/__init__.py @@ -24,7 +24,7 @@ maap = MAAP() # Search for granules - granules = maap.searchGranule( + granules = maap.search_granule( short_name='GEDI02_A', limit=10 ) @@ -34,7 +34,7 @@ local_path = granule.getData(destpath='/tmp') # Submit a job - job = maap.submitJob( + job = maap.submit_job( identifier='my_job', algo_id='my_algorithm', version='main', diff --git a/maap/dps/dps_job.py b/maap/dps/dps_job.py index 6c395dd..8b1a3d0 100644 --- a/maap/dps/dps_job.py +++ b/maap/dps/dps_job.py @@ -21,7 +21,7 @@ maap = MAAP() # Submit a job - job = maap.submitJob( + job = maap.submit_job( identifier='my_analysis', algo_id='my_algorithm', version='main', @@ -38,8 +38,8 @@ See Also -------- -:meth:`maap.maap.MAAP.submitJob` : Submit a new job -:meth:`maap.maap.MAAP.getJob` : Retrieve an existing job +:meth:`maap.maap.MAAP.submit_job` : Submit a new job +:meth:`maap.maap.MAAP.get_job` : Retrieve an existing job """ import json @@ -131,12 +131,12 @@ class DPSJob: -------- Get job status:: - >>> job = maap.getJob('f3780917-92c0-4440-8a84-9b28c2e64fa8') + >>> job = maap.get_job('f3780917-92c0-4440-8a84-9b28c2e64fa8') >>> print(f"Status: {job.status}") Wait for completion:: - >>> job = maap.submitJob(...) + >>> job = maap.submit_job(...) >>> job.wait_for_completion() >>> print(f"Final status: {job.status}") @@ -159,7 +159,7 @@ class DPSJob: See Also -------- - :meth:`maap.maap.MAAP.submitJob` : Submit new jobs + :meth:`maap.maap.MAAP.submit_job` : Submit new jobs :meth:`maap.maap.MAAP.listJobs` : List all jobs """ @@ -243,7 +243,7 @@ def wait_for_completion(self): -------- :: - >>> job = maap.submitJob(...) + >>> job = maap.submit_job(...) >>> job.wait_for_completion() >>> if job.status == 'Succeeded': ... print("Job completed successfully!") diff --git a/maap/maap.py b/maap/maap.py index 87051b2..7e19240 100644 --- a/maap/maap.py +++ b/maap/maap.py @@ -23,7 +23,7 @@ maap = MAAP() # Search for granules - granules = maap.searchGranule( + granules = maap.search_granule( short_name='GEDI02_A', limit=10 ) @@ -111,7 +111,7 @@ class MAAP(object): Search for granules:: - >>> granules = maap.searchGranule( + >>> granules = maap.search_granule( ... short_name='GEDI02_A', ... bounding_box='-122.5,37.5,-121.5,38.5', ... limit=5 @@ -121,7 +121,7 @@ class MAAP(object): Submit a job:: - >>> job = maap.submitJob( + >>> job = maap.submit_job( ... identifier='my_analysis', ... algo_id='my_algorithm', ... version='main', @@ -226,7 +226,7 @@ def _upload_s3(self, filename, bucket, objectKey): Notes ----- - This is an internal method primarily used by :meth:`uploadFiles`. + This is an internal method primarily used by :meth:`upload_files`. It uses the boto3 S3 client configured at module level. """ return s3_client.upload_file(filename, bucket, objectKey) @@ -275,14 +275,14 @@ def search_granule(self, limit=20, **kwargs): -------- Search by collection name:: - >>> granules = maap.searchGranule( + >>> granules = maap.search_granule( ... short_name='GEDI02_A', ... limit=10 ... ) Search with spatial bounds:: - >>> granules = maap.searchGranule( + >>> granules = maap.search_granule( ... collection_concept_id='C1234567890-MAAP', ... bounding_box='-122.5,37.5,-121.5,38.5', ... limit=5 @@ -290,7 +290,7 @@ def search_granule(self, limit=20, **kwargs): Search with temporal filter:: - >>> granules = maap.searchGranule( + >>> granules = maap.search_granule( ... short_name='AFLVIS2', ... temporal='2019-01-01T00:00:00Z,2019-12-31T23:59:59Z', ... limit=100 @@ -298,7 +298,7 @@ def search_granule(self, limit=20, **kwargs): Search with pattern matching:: - >>> granules = maap.searchGranule( + >>> granules = maap.search_granule( ... readable_granule_name='*2019*', ... short_name='GEDI02_A' ... ) @@ -317,7 +317,7 @@ def search_granule(self, limit=20, **kwargs): See Also -------- - :meth:`searchCollection` : Search for collections + :meth:`search_collection` : Search for collections :class:`~maap.Result.Granule` : Granule result class """ results = self._CMR.get_search_results(url=self.config.search_granule_url, limit=limit, **kwargs) @@ -356,7 +356,7 @@ def download_granule(self, online_access_url, destination_path=".", overwrite=Fa -------- Download a granule by URL:: - >>> local_file = maap.downloadGranule( + >>> local_file = maap.download_granule( ... 'https://data.maap-project.org/file/data.h5', ... destination_path='/tmp/downloads' ... ) @@ -364,7 +364,7 @@ def download_granule(self, online_access_url, destination_path=".", overwrite=Fa Force overwrite of existing files:: - >>> local_file = maap.downloadGranule( + >>> local_file = maap.download_granule( ... url, ... destination_path='/tmp', ... overwrite=True @@ -383,7 +383,7 @@ def download_granule(self, online_access_url, destination_path=".", overwrite=Fa See Also -------- - :meth:`searchGranule` : Search for granules + :meth:`search_granule` : Search for granules :meth:`~maap.Result.Granule.getData` : Download granule data """ filename = os.path.basename(urllib.parse.urlparse(online_access_url).path) @@ -426,9 +426,9 @@ def get_call_from_earthdata_query(self, query, variable_name='maap', limit=1000) Convert an Earthdata query:: >>> query = '{"instrument_h": ["GEDI"], "bounding_box": "-180,-90,180,90"}' - >>> code = maap.getCallFromEarthdataQuery(query) + >>> code = maap.get_call_from_earthdata_query(query) >>> print(code) - maap.searchGranule(instrument="GEDI", bounding_box="-180,-90,180,90", limit=1000) + maap.search_granule(instrument="GEDI", bounding_box="-180,-90,180,90", limit=1000) Notes ----- @@ -438,8 +438,8 @@ def get_call_from_earthdata_query(self, query, variable_name='maap', limit=1000) See Also -------- - :meth:`getCallFromCmrUri` : Generate call from CMR URI - :meth:`searchGranule` : Execute a granule search + :meth:`get_call_from_cmr_uri` : Generate call from CMR URI + :meth:`search_granule` : Execute a granule search """ return self._CMR.generateGranuleCallFromEarthDataRequest(query, variable_name, limit) @@ -475,16 +475,16 @@ def get_call_from_cmr_uri(self, search_url, variable_name='maap', limit=1000, se Convert a CMR granule search URL:: >>> url = 'https://cmr.earthdata.nasa.gov/search/granules?short_name=GEDI02_A' - >>> code = maap.getCallFromCmrUri(url) + >>> code = maap.get_call_from_cmr_uri(url) >>> print(code) - maap.searchGranule(short_name="GEDI02_A", limit=1000) + maap.search_granule(short_name="GEDI02_A", limit=1000) Convert a collection search:: >>> url = 'https://cmr.earthdata.nasa.gov/search/collections?provider=MAAP' - >>> code = maap.getCallFromCmrUri(url, search='collection') + >>> code = maap.get_call_from_cmr_uri(url, search='collection') >>> print(code) - maap.searchCollection(provider="MAAP", limit=1000) + maap.search_collection(provider="MAAP", limit=1000) Notes ----- @@ -494,9 +494,9 @@ def get_call_from_cmr_uri(self, search_url, variable_name='maap', limit=1000, se See Also -------- - :meth:`getCallFromEarthdataQuery` : Generate call from Earthdata query - :meth:`searchGranule` : Execute a granule search - :meth:`searchCollection` : Execute a collection search + :meth:`get_call_from_earthdata_query` : Generate call from Earthdata query + :meth:`search_granule` : Execute a granule search + :meth:`search_collection` : Execute a collection search """ return self._CMR.generateCallFromEarthDataQueryString(search_url, variable_name, limit, search) @@ -541,20 +541,20 @@ def search_collection(self, limit=100, **kwargs): -------- Search by short name:: - >>> collections = maap.searchCollection(short_name='GEDI02_A') + >>> collections = maap.search_collection(short_name='GEDI02_A') >>> for c in collections: ... print(c['Collection']['ShortName']) Search by provider:: - >>> collections = maap.searchCollection( + >>> collections = maap.search_collection( ... provider='MAAP', ... limit=50 ... ) Search by keyword:: - >>> collections = maap.searchCollection( + >>> collections = maap.search_collection( ... keyword='biomass forest', ... limit=20 ... ) @@ -562,12 +562,12 @@ def search_collection(self, limit=100, **kwargs): Notes ----- Collections contain metadata about datasets but not the actual data - files. Use :meth:`searchGranule` to find individual data files within + files. Use :meth:`search_granule` to find individual data files within a collection. See Also -------- - :meth:`searchGranule` : Search for granules within collections + :meth:`search_granule` : Search for granules within collections :class:`~maap.Result.Collection` : Collection result class """ results = self._CMR.get_search_results(url=self.config.search_collection_url, limit=limit, **kwargs) @@ -591,7 +591,7 @@ def get_queues(self): -------- List available queues:: - >>> response = maap.getQueues() + >>> response = maap.get_queues() >>> queues = response.json() >>> for queue in queues: ... print(f"{queue['name']}: {queue['memory']} RAM") @@ -603,8 +603,8 @@ def get_queues(self): See Also -------- - :meth:`submitJob` : Submit a job to a queue - :meth:`registerAlgorithm` : Register an algorithm to run on queues + :meth:`submit_job` : Submit a job to a queue + :meth:`register_algorithm` : Register an algorithm to run on queues """ url = os.path.join(self.config.algorithm_register, 'resource') headers = self._get_api_header() @@ -654,13 +654,13 @@ def upload_files(self, filenames): -------- Upload files to share:: - >>> result = maap.uploadFiles(['data.csv', 'config.json']) + >>> result = maap.upload_files(['data.csv', 'config.json']) >>> print(result) Upload file subdirectory: a1b2c3d4-e5f6-... (keep a record of...) Upload a single file:: - >>> result = maap.uploadFiles(['output.tif']) + >>> result = maap.upload_files(['output.tif']) Notes ----- @@ -670,7 +670,7 @@ def upload_files(self, filenames): See Also -------- - :meth:`submitJob` : Use uploaded files as job inputs + :meth:`submit_job` : Use uploaded files as job inputs """ bucket = self.config.s3_user_upload_bucket prefix = self.config.s3_user_upload_dir @@ -738,7 +738,7 @@ def show(self, granule, display_config={}): ---------- granule : dict A granule result dictionary, typically obtained from - :meth:`searchGranule`. Must contain ``Granule.GranuleUR``. + :meth:`search_granule`. Must contain ``Granule.GranuleUR``. display_config : dict, optional Configuration options for rendering. Common options include: @@ -751,7 +751,7 @@ def show(self, granule, display_config={}): -------- Display a granule on a map:: - >>> granules = maap.searchGranule(short_name='AFLVIS2', limit=1) + >>> granules = maap.search_granule(short_name='AFLVIS2', limit=1) >>> maap.show(granules[0]) Display with custom rendering:: @@ -769,7 +769,7 @@ def show(self, granule, display_config={}): See Also -------- - :meth:`searchGranule` : Search for granules to visualize + :meth:`search_granule` : Search for granules to visualize """ from mapboxgl.viz import RasterTilesViz diff --git a/maap/utils/CMR.py b/maap/utils/CMR.py index 18ebf94..665228f 100644 --- a/maap/utils/CMR.py +++ b/maap/utils/CMR.py @@ -129,7 +129,7 @@ def generateGranuleCallFromEarthDataRequest(self, query, variable_name='maap', l params.append("limit=" + str(limit)) - result = variable_name + ".searchGranule(" + ", ".join(params) + ")" + result = variable_name + ".search_granule(" + ", ".join(params) + ")" return result @@ -157,7 +157,7 @@ def generateCallFromEarthDataQueryString(self, search_url, variable_name='maap', # e.g., # granules?collection_concept_id[]=C1&collection_concept_id[]=C2 # will be converted to - # maap.searchGranule(collection_concept_id="C1|C2") + # maap.search_granule(collection_concept_id="C1|C2") if any(x for x in params if x.startswith(p_key_assignment)): params[i - 1] = params[i - 1].replace(p_key_assignment, p_key_assignment + p_val + "|") else: @@ -167,8 +167,8 @@ def generateCallFromEarthDataQueryString(self, search_url, variable_name='maap', params.append("limit=" + str(limit)) if search == 'granule': - result = variable_name + ".searchGranule(" + ", ".join(params) + ")" + result = variable_name + ".search_granule(" + ", ".join(params) + ")" else: - result = variable_name + ".searchCollection(" + ", ".join(params) + ")" + result = variable_name + ".search_collection(" + ", ".join(params) + ")" return result diff --git a/test/functional_test.py b/test/functional_test.py index de55154..d2ef8b7 100644 --- a/test/functional_test.py +++ b/test/functional_test.py @@ -79,7 +79,7 @@ def submit_job(maap: MAAP, wait_for_completion=False, queue="maap-dps-worker-8gb "output_filename": "output.tif", "outsize": "20" } - job = maap.submitJob(identifier="maap_functional_test", + job = maap.submit_job(identifier="maap_functional_test", algo_id=algo_name, version=algo_version, queue=queue, diff --git a/test/test_CMR.py b/test/test_CMR.py index 3a9ad33..d8878e7 100644 --- a/test/test_CMR.py +++ b/test/test_CMR.py @@ -16,79 +16,79 @@ def setUpClass(cls): cls._test_ur = 'uavsar_AfriSAR_v1-cov_lopenp_14043_16008_140_001_160225-geo_cov_4-4.bin' cls._test_site_name = 'lope' - def test_searchGranuleByInstrumentAndTrackNumber(self): - results = self.maap.searchGranule( + def test_search_granule_by_instrument_and_track_number(self): + results = self.maap.search_granule( instrument=self._test_instrument_name_uavsar, track_number=self._test_track_number, polarization='HH') self.assertTrue('concept-id' in results[0].keys()) - def test_searchGranuleByGranuleUR(self): - results = self.maap.searchGranule( + def test_search_granule_by_granule_ur(self): + results = self.maap.search_granule( granule_ur=self._test_ur) self.assertTrue('concept-id' in results[0].keys()) - def test_granuleDownload(self): - results = self.maap.searchGranule( + def test_granule_download(self): + results = self.maap.search_granule( granule_ur=self._test_ur) - download = results[0].getLocalPath('/Users/satorius/source') + download = results[0].get_local_path('/Users/satorius/source') self.assertTrue(len(download) > 0) - def test_granuleDownloadExternalDAAC(self): - # results = self.maap.searchGranule( + def test_granule_download_external_daac(self): + # results = self.maap.search_granule( # collection_concept_id='C1200231010-NASA_MAAP') - results = self.maap.searchGranule( + results = self.maap.search_granule( cmr_host='cmr.earthdata.nasa.gov', collection_concept_id='C2067521974-ORNL_CLOUD', granule_ur='GEDI_L3_Land_Surface_Metrics.GEDI03_elev_lowestmode_stddev_2019108_2020106_001_08.tif') - download = results[0].getData() + download = results[0].get_data() self.assertTrue(len(download) > 0) - def test_direct_granuleDownload(self): - results = self.maap.downloadGranule( + def test_direct_granule_download(self): + results = self.maap.download_granule( online_access_url='https://datapool.asf.alaska.edu/GRD_HD/SA/S1A_S3_GRDH_1SDH_20140615T034444_20140615T034512_001055_00107C_8977.zip', destination_path='./tmp' ) self.assertTrue(len(results) > 0) - def test_searchGranuleByInstrumentAndSiteName(self): - results = self.maap.searchGranule( + def test_search_granule_by_instrument_and_site_name(self): + results = self.maap.search_granule( instrument=self._test_instrument_name_lvis, site_name=self._test_site_name) self.assertTrue('concept-id' in results[0].keys()) - def test_searchGranuleWithPipeDelimiters(self): - results = self.maap.searchGranule( + def test_search_granule_with_pipe_delimiters(self): + results = self.maap.search_granule( instrument="LVIS|UAVSAR", platform="AIRCRAFT") self.assertTrue('concept-id' in results[0].keys()) - def test_searchFromEarthdata(self): - results = self.maap.searchCollection( + def test_search_from_earthdata(self): + results = self.maap.search_collection( instrument="LVIS|UAVSAR", platform="AIRCRAFT|B-200|COMPUTERS", data_center="MAAP Data Management Team|ORNL_DAAC") self.assertTrue('concept-id' in results[0].keys()) - def test_searchCollection(self): - results = self.maap.searchCollection( + def test_search_collection(self): + results = self.maap.search_collection( instrument=self._test_instrument_name_uavsar) self.assertTrue('concept-id' in results[0].keys()) - def test_searchGranuleWithWildcards(self): - results = self.maap.searchGranule(collection_concept_id="C1200110748-NASA_MAAP", + def test_search_granule_with_wildcards(self): + results = self.maap.search_granule(collection_concept_id="C1200110748-NASA_MAAP", readable_granule_name='*185*') self.assertTrue('concept-id' in results[0].keys()) - def test_getUrl(self): - results = self.maap.searchGranule(page_num="1", concept_id="C1214470488-ASF", sort_key="-start_date", limit=1) + def test_get_url(self): + results = self.maap.search_granule(page_num="1", concept_id="C1214470488-ASF", sort_key="-start_date", limit=1) - url = results[0].getHttpUrl() + url = results[0].get_http_url() self.assertTrue(url.startswith("http")) - url = results[0].getS3Url() + url = results[0].get_s3_url() self.assertTrue(url.startswith("s3")) diff --git a/test/test_DPS.py b/test/test_DPS.py deleted file mode 100644 index cb6dc47..0000000 --- a/test/test_DPS.py +++ /dev/null @@ -1,61 +0,0 @@ -from unittest import TestCase - -import yaml - -from maap.maap import MAAP -import logging -from yaml import load as yaml_load, dump as yaml_dump -try: - from yaml import CLoader as Loader, CDumper as Dumper -except ImportError: - from yaml import Loader, Dumper - -class TestDPS(TestCase): - logging.basicConfig(level=logging.DEBUG) - logger = logging.getLogger(__name__) - - @classmethod - def setUpClass(cls): - cls.logger.debug("Initializing MAAP") - cls.maap = MAAP() - - def test_registerAlgorithm(self): - self.maap.register_algorithm_from_yaml_file("dps_test_algo_config.yaml") - - def test_deleteAlgorithm(self): - pass - - def test_deleteJob(self): - pass - - def test_describeAlgorithm(self): - pass - - def test_dismissJob(self): - pass - - def test_getJobMetrics(self): - pass - - def test_getJobResult(self): - pass - - def test_getJobStatus(self): - pass - - def test_getQueues(self): - pass - - def test_listAlgorithms(self): - pass - - def test_listJobs(self): - pass - - def test_publishAlgorithm(self): - pass - - - def test_submitJob(self): - pass - diff --git a/test/test_MAAP.py b/test/test_MAAP.py index 5dcc082..aef8238 100644 --- a/test/test_MAAP.py +++ b/test/test_MAAP.py @@ -54,18 +54,18 @@ def test_genFromEarthdata(self): """ var_name = 'maapVar' - testResult = self.maap.getCallFromEarthdataQuery(query=input, variable_name=var_name) + testResult = self.maap.get_call_from_earthdata_query(query=input, variable_name=var_name) self.assertTrue( - testResult == var_name + '.searchGranule('\ + testResult == var_name + '.search_granule('\ 'processing_level_id="1A|1B|2|4", '\ 'instrument="LVIS|UAVSAR", '\ 'platform="AIRCRAFT|B-200|COMPUTERS", '\ 'data_center="MAAP Data Management Team", '\ 'bounding_box="-35.4375,-55.6875,-80.4375,37.6875")') - def test_uploadFiles(self): + def test_upload_files(self): self.maap._upload_s3 = MagicMock(return_value=None) - result = self.maap.uploadFiles(['test/s3-upload-testfile1.txt', 'test/s3-upload-testfile2.txt']) + result = self.maap.upload_files(['test/s3-upload-testfile1.txt', 'test/s3-upload-testfile2.txt']) upload_msg_regex = re.compile('Upload file subdirectory: .+ \\(keep a record of this if you want to share these files with other users\\)') self.assertTrue(re.match(upload_msg_regex, result)) From 75b6cf08492e425e92a2e50a2c1ec060bf78bf8b Mon Sep 17 00:00:00 2001 From: grallewellyn Date: Mon, 12 Jan 2026 17:18:30 -0800 Subject: [PATCH 12/17] initial attempt to register alg from cwl file --- maap/maap.py | 30 ++++----- maap/utils/algorithm_utils.py | 115 +++++++++++++++++++++++++++++++--- pyproject.toml | 1 + 3 files changed, 122 insertions(+), 24 deletions(-) diff --git a/maap/maap.py b/maap/maap.py index 7e19240..f8b77c3 100644 --- a/maap/maap.py +++ b/maap/maap.py @@ -617,20 +617,22 @@ def get_queues(self): ) return response - # def register_algorithm_from_cwl_file(self, file_path): - # """ - # Registers an algorithm from a CWL file - # """ - # # Read cwl file returns a dict in the format to register an algorithm without a CWL - # algo_config = algorithm_utils.read_cwl_file(file_path) - # headers = self._get_api_header(content_type='application/json') - # logger.debug('POST request sent to {}'.format(self.config.processes_ogc)) - # response = requests.post( - # url=self.config.processes_ogc, - # headers=headers, - # json=algo_config - # ) - # return response + def register_algorithm_from_cwl_file(self, file_path): + """ + Registers an algorithm from a CWL file + """ + # Read cwl file returns a dict in the format to register an algorithm without a CWL + process_config = algorithm_utils.read_cwl_file(file_path) + print("graceal1 returned JSON of metadata extracted from CWL:") + print(process_config) + headers = self._get_api_header(content_type='application/json') + logger.debug('POST request sent to {}'.format(self.config.processes_ogc)) + response = requests.post( + url=self.config.processes_ogc, + headers=headers, + json=process_config + ) + return response def upload_files(self, filenames): """ diff --git a/maap/utils/algorithm_utils.py b/maap/utils/algorithm_utils.py index 1717c84..4fdc932 100644 --- a/maap/utils/algorithm_utils.py +++ b/maap/utils/algorithm_utils.py @@ -7,7 +7,10 @@ from yaml import CLoader as Loader, CDumper as Dumper except ImportError: from yaml import Loader, Dumper - +from cwl_utils.parser import load_document_by_uri, cwl_v1_2 +import re +import urllib.parse +import os logger = logging.getLogger(__name__) @@ -20,15 +23,22 @@ def read_yaml_file(algo_yaml): def read_cwl_file(algo_cwl): """ - Parse through the CWL file and returns the response as the POST to register a + Parse through the CWL file and returns the response as a JSON for the POST to register a a process for OGC is expecting https://github.com/MAAP-Project/joint-open-api-specs/blob/nasa-adaptation/ogc-api-processes/openapi-template/schemas/processes-core/postProcess.yaml """ try: with open(algo_cwl, 'r') as f: - cwl_data = yaml.safe_load(f) - print(f"Successfully read and parsed '{algo_cwl}'") - return parse_cwl_data(cwl_data) + cwl_text = yaml.safe_load(f) + try: + cwl_obj = load_document_by_uri(algo_cwl, load_all=True) + except Exception as e: + print(f"Failed to parse CWL: {e}") + raise ValueError("CWL file is not in the right format or is invalid.") + print("graceal1 successfully got cwl object and data") + print(cwl_obj) + print(cwl_text) + return parse_cwl_data(cwl_obj, cwl_text) except FileNotFoundError: print(f"Error: The file '{algo_cwl}' was not found.") return None @@ -36,11 +46,96 @@ def read_cwl_file(algo_cwl): print(f"Error parsing the YAML in '{algo_cwl}': {e}") return None -def parse_cwl_data(cwl_data): - algo_config = dict() - # TODO implement this and return cwl_data as a dictionary with important variables like - # the API is expecting - return algo_config + +def parse_cwl_data(cwl_obj, cwl_text): + """ + Pass the cwl object for essential arguments + Assigning of default values is in the API + + :param cwl_obj: Object of CWL file from cwl_utils + :param cwl_text: Plain text of CWL file + """ + workflow = next((obj for obj in cwl_obj if isinstance(obj, cwl_v1_2.Workflow)), None) + if not workflow: + raise ValueError("A valid Workflow object must be defined in the CWL file.") + + cwl_id = workflow.id + version_match = re.search(r"s:version:\s*(\S+)", cwl_text, re.IGNORECASE) + + if not version_match or not cwl_id: + raise ValueError("Required metadata missing: s:version and a top-level id are required.") + + fragment = urllib.parse.urlparse(cwl_id).fragment + cwl_id = os.path.basename(fragment) + process_version = version_match.group(1) + + if ":" in process_version: + raise ValueError("Process version cannot contain a :") + + # Get git information + github_url = re.search(r"s:codeRepository:\s*(\S+)", cwl_text, re.IGNORECASE) + github_url = github_url.group(1) if github_url else None + git_commit_hash = re.search(r"s:commitHash:\s*(\S+)", cwl_text, re.IGNORECASE) + git_commit_hash = git_commit_hash.group(1) if git_commit_hash else None + + keywords_match = re.search(r"s:keywords:\s*(.*)", cwl_text, re.IGNORECASE) + keywords = keywords_match.group(1).replace(" ", "") if keywords_match else None + + try: + author_match = re.search( + r"s:author:.*?s:name:\s*(\S+)", + cwl_text, + re.DOTALL | re.IGNORECASE + ) + author = author_match.group(1) if author_match else None + except Exception as e: + author = None + print(f"Failed to get author name: {e}") + + # Initialize optional variables + ram_min = None + cores_min = None + base_command = None + + # Find the CommandLineTool run by the first step of the workflow + if workflow.steps: + # Get the ID of the tool to run (e.g., '#main') + tool_id_ref = workflow.steps[0].run + # The actual ID is the part after the '#' + tool_id = os.path.basename(tool_id_ref) + + # Find the CommandLineTool object in the parsed CWL graph + command_line_tool = next((obj for obj in cwl_obj if isinstance(obj, cwl_v1_2.CommandLineTool) and obj.id.endswith(tool_id)), None) + + if command_line_tool: + # Extract the baseCommand directly + base_command = command_line_tool.baseCommand + + # Find the ResourceRequirement to extract ramMin and coresMin + if command_line_tool.requirements: + for req in command_line_tool.requirements: + if isinstance(req, cwl_v1_2.ResourceRequirement): + ram_min = req.ramMin if req.ramMin else ram_min + cores_min = req.coresMin if req.coresMin else cores_min + break # Stop after finding the first ResourceRequirement + + # Build dictionary with all extracted variables + process_config = { + 'id': cwl_id, + 'version': process_version, + 'title': workflow.label, + 'description': workflow.doc, + 'keywords': keywords, + 'raw_text': cwl_text, + 'github_url': github_url, + 'git_commit_hash': git_commit_hash, + 'ram_min': ram_min, + 'cores_min': cores_min, + 'base_command': base_command, + 'author': author + } + + return process_config def validate_algorithm_config(algo_config): return algo_config diff --git a/pyproject.toml b/pyproject.toml index 701a5c3..7208acc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,6 +33,7 @@ importlib-resources = "^6.4.0" mapboxgl = "^0.10.2" pyyaml = "^6.0.1" requests = "^2.32.3" +cwl_utils = ">0.10" [tool.poetry.group.dev.dependencies] From 890dc970b5cad4c68fc8fc752c77dc4e96dac2d4 Mon Sep 17 00:00:00 2001 From: grallewellyn Date: Mon, 12 Jan 2026 17:26:26 -0800 Subject: [PATCH 13/17] cwl_text is now a string --- maap/utils/algorithm_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maap/utils/algorithm_utils.py b/maap/utils/algorithm_utils.py index 4fdc932..11ea79e 100644 --- a/maap/utils/algorithm_utils.py +++ b/maap/utils/algorithm_utils.py @@ -29,7 +29,7 @@ def read_cwl_file(algo_cwl): """ try: with open(algo_cwl, 'r') as f: - cwl_text = yaml.safe_load(f) + cwl_text = f.read() try: cwl_obj = load_document_by_uri(algo_cwl, load_all=True) except Exception as e: From 2a7c749a36d909e05aada8757f766010c10b5151 Mon Sep 17 00:00:00 2001 From: grallewellyn Date: Tue, 13 Jan 2026 14:23:55 -0800 Subject: [PATCH 14/17] pass cwl as raw text to API --- maap/maap.py | 14 +++++++++----- maap/utils/algorithm_utils.py | 3 +-- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/maap/maap.py b/maap/maap.py index f8b77c3..efd68b9 100644 --- a/maap/maap.py +++ b/maap/maap.py @@ -621,16 +621,20 @@ def register_algorithm_from_cwl_file(self, file_path): """ Registers an algorithm from a CWL file """ - # Read cwl file returns a dict in the format to register an algorithm without a CWL - process_config = algorithm_utils.read_cwl_file(file_path) - print("graceal1 returned JSON of metadata extracted from CWL:") - print(process_config) + # Read raw text from CWL file + with open(file_path, 'r') as f: + raw_text = f.read() + process = { + "cwlRawText": raw_text + } + print("graceal1 returned raw text from CWL:") + print(raw_text) headers = self._get_api_header(content_type='application/json') logger.debug('POST request sent to {}'.format(self.config.processes_ogc)) response = requests.post( url=self.config.processes_ogc, headers=headers, - json=process_config + json=process ) return response diff --git a/maap/utils/algorithm_utils.py b/maap/utils/algorithm_utils.py index 11ea79e..471c220 100644 --- a/maap/utils/algorithm_utils.py +++ b/maap/utils/algorithm_utils.py @@ -126,8 +126,7 @@ def parse_cwl_data(cwl_obj, cwl_text): 'title': workflow.label, 'description': workflow.doc, 'keywords': keywords, - 'raw_text': cwl_text, - 'github_url': github_url, + 'github_url': {'href': github_url}, 'git_commit_hash': git_commit_hash, 'ram_min': ram_min, 'cores_min': cores_min, From 48b69d34d396c2b9ba7daa4e63a8eeb14b665dd3 Mon Sep 17 00:00:00 2001 From: grallewellyn Date: Wed, 14 Jan 2026 16:41:45 -0800 Subject: [PATCH 15/17] deleted unnecessary code and print statements --- maap/maap.py | 2 - maap/utils/algorithm_utils.py | 115 ---------------------------------- 2 files changed, 117 deletions(-) diff --git a/maap/maap.py b/maap/maap.py index efd68b9..1625f11 100644 --- a/maap/maap.py +++ b/maap/maap.py @@ -627,8 +627,6 @@ def register_algorithm_from_cwl_file(self, file_path): process = { "cwlRawText": raw_text } - print("graceal1 returned raw text from CWL:") - print(raw_text) headers = self._get_api_header(content_type='application/json') logger.debug('POST request sent to {}'.format(self.config.processes_ogc)) response = requests.post( diff --git a/maap/utils/algorithm_utils.py b/maap/utils/algorithm_utils.py index 471c220..0618907 100644 --- a/maap/utils/algorithm_utils.py +++ b/maap/utils/algorithm_utils.py @@ -21,120 +21,5 @@ def read_yaml_file(algo_yaml): algo_config = yaml_load(fr, Loader=Loader) return validate_algorithm_config(algo_config) -def read_cwl_file(algo_cwl): - """ - Parse through the CWL file and returns the response as a JSON for the POST to register a - a process for OGC is expecting - https://github.com/MAAP-Project/joint-open-api-specs/blob/nasa-adaptation/ogc-api-processes/openapi-template/schemas/processes-core/postProcess.yaml - """ - try: - with open(algo_cwl, 'r') as f: - cwl_text = f.read() - try: - cwl_obj = load_document_by_uri(algo_cwl, load_all=True) - except Exception as e: - print(f"Failed to parse CWL: {e}") - raise ValueError("CWL file is not in the right format or is invalid.") - print("graceal1 successfully got cwl object and data") - print(cwl_obj) - print(cwl_text) - return parse_cwl_data(cwl_obj, cwl_text) - except FileNotFoundError: - print(f"Error: The file '{algo_cwl}' was not found.") - return None - except yaml.YAMLError as e: - print(f"Error parsing the YAML in '{algo_cwl}': {e}") - return None - - -def parse_cwl_data(cwl_obj, cwl_text): - """ - Pass the cwl object for essential arguments - Assigning of default values is in the API - - :param cwl_obj: Object of CWL file from cwl_utils - :param cwl_text: Plain text of CWL file - """ - workflow = next((obj for obj in cwl_obj if isinstance(obj, cwl_v1_2.Workflow)), None) - if not workflow: - raise ValueError("A valid Workflow object must be defined in the CWL file.") - - cwl_id = workflow.id - version_match = re.search(r"s:version:\s*(\S+)", cwl_text, re.IGNORECASE) - - if not version_match or not cwl_id: - raise ValueError("Required metadata missing: s:version and a top-level id are required.") - - fragment = urllib.parse.urlparse(cwl_id).fragment - cwl_id = os.path.basename(fragment) - process_version = version_match.group(1) - - if ":" in process_version: - raise ValueError("Process version cannot contain a :") - - # Get git information - github_url = re.search(r"s:codeRepository:\s*(\S+)", cwl_text, re.IGNORECASE) - github_url = github_url.group(1) if github_url else None - git_commit_hash = re.search(r"s:commitHash:\s*(\S+)", cwl_text, re.IGNORECASE) - git_commit_hash = git_commit_hash.group(1) if git_commit_hash else None - - keywords_match = re.search(r"s:keywords:\s*(.*)", cwl_text, re.IGNORECASE) - keywords = keywords_match.group(1).replace(" ", "") if keywords_match else None - - try: - author_match = re.search( - r"s:author:.*?s:name:\s*(\S+)", - cwl_text, - re.DOTALL | re.IGNORECASE - ) - author = author_match.group(1) if author_match else None - except Exception as e: - author = None - print(f"Failed to get author name: {e}") - - # Initialize optional variables - ram_min = None - cores_min = None - base_command = None - - # Find the CommandLineTool run by the first step of the workflow - if workflow.steps: - # Get the ID of the tool to run (e.g., '#main') - tool_id_ref = workflow.steps[0].run - # The actual ID is the part after the '#' - tool_id = os.path.basename(tool_id_ref) - - # Find the CommandLineTool object in the parsed CWL graph - command_line_tool = next((obj for obj in cwl_obj if isinstance(obj, cwl_v1_2.CommandLineTool) and obj.id.endswith(tool_id)), None) - - if command_line_tool: - # Extract the baseCommand directly - base_command = command_line_tool.baseCommand - - # Find the ResourceRequirement to extract ramMin and coresMin - if command_line_tool.requirements: - for req in command_line_tool.requirements: - if isinstance(req, cwl_v1_2.ResourceRequirement): - ram_min = req.ramMin if req.ramMin else ram_min - cores_min = req.coresMin if req.coresMin else cores_min - break # Stop after finding the first ResourceRequirement - - # Build dictionary with all extracted variables - process_config = { - 'id': cwl_id, - 'version': process_version, - 'title': workflow.label, - 'description': workflow.doc, - 'keywords': keywords, - 'github_url': {'href': github_url}, - 'git_commit_hash': git_commit_hash, - 'ram_min': ram_min, - 'cores_min': cores_min, - 'base_command': base_command, - 'author': author - } - - return process_config - def validate_algorithm_config(algo_config): return algo_config From 4762c1685b2a9b12f75610e923f624a43f075226 Mon Sep 17 00:00:00 2001 From: grallewellyn Date: Wed, 14 Jan 2026 16:43:31 -0800 Subject: [PATCH 16/17] deleted unnecessary packages and imports --- maap/utils/algorithm_utils.py | 4 ---- pyproject.toml | 1 - 2 files changed, 5 deletions(-) diff --git a/maap/utils/algorithm_utils.py b/maap/utils/algorithm_utils.py index 0618907..8c652ee 100644 --- a/maap/utils/algorithm_utils.py +++ b/maap/utils/algorithm_utils.py @@ -7,10 +7,6 @@ from yaml import CLoader as Loader, CDumper as Dumper except ImportError: from yaml import Loader, Dumper -from cwl_utils.parser import load_document_by_uri, cwl_v1_2 -import re -import urllib.parse -import os logger = logging.getLogger(__name__) diff --git a/pyproject.toml b/pyproject.toml index 7208acc..701a5c3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,6 @@ importlib-resources = "^6.4.0" mapboxgl = "^0.10.2" pyyaml = "^6.0.1" requests = "^2.32.3" -cwl_utils = ">0.10" [tool.poetry.group.dev.dependencies] From 5d23fa38a7814016299f6a546ff838ea964581f4 Mon Sep 17 00:00:00 2001 From: grallewellyn Date: Wed, 14 Jan 2026 17:11:13 -0800 Subject: [PATCH 17/17] added replace algorithm from cwl function --- maap/maap.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/maap/maap.py b/maap/maap.py index 1625f11..2ac93ce 100644 --- a/maap/maap.py +++ b/maap/maap.py @@ -635,6 +635,26 @@ def register_algorithm_from_cwl_file(self, file_path): json=process ) return response + + def replace_algorithm_from_cwl_file(self, process_id, file_path): + """ + Registers an algorithm from a CWL file + """ + # Read raw text from CWL file + with open(file_path, 'r') as f: + raw_text = f.read() + process = { + "cwlRawText": raw_text + } + url = os.path.join(self.config.processes_ogc, str(process_id)) + headers = self._get_api_header(content_type='application/json') + logger.debug('PUT request sent to {}'.format(url)) + response = requests.put( + url=url, + headers=headers, + json=process + ) + return response def upload_files(self, filenames): """