From 806b4ba127cd738c899de3edafdf43c99418cc7a Mon Sep 17 00:00:00 2001
From: rileykk <rileykk@jpl.nasa.gov>
Date: Fri, 2 Sep 2022 15:13:48 -0700
Subject: [PATCH 01/38] Base framework for CAML rendering

Should add support for CAML output to results and matchup endpoints (notwithstanding the fact it currently does nothing; just links up the code for now)
---
 .../algorithms/doms/BaseDomsHandler.py        |  8 ++++++++
 .../request/renderers/NexusCAMLRenderer.py    | 19 +++++++++++++++++++
 .../request/renderers/NexusRendererFactory.py |  2 +-
 3 files changed, 28 insertions(+), 1 deletion(-)
 create mode 100644 analysis/webservice/nexus_tornado/request/renderers/NexusCAMLRenderer.py

diff --git a/analysis/webservice/algorithms/doms/BaseDomsHandler.py b/analysis/webservice/algorithms/doms/BaseDomsHandler.py
index 2bcb728f..f513a296 100644
--- a/analysis/webservice/algorithms/doms/BaseDomsHandler.py
+++ b/analysis/webservice/algorithms/doms/BaseDomsHandler.py
@@ -106,6 +106,9 @@ def toCSV(self):
     def toNetCDF(self):
         return DomsNetCDFFormatter.create(self.__executionId, self.results(), self.__args, self.__details)
 
+    def toCAML(self):
+        return DomsCAMLFormatter.create(self.__executionId, self.results(), self.__args, self.__details)
+
 
 class DomsCSVFormatter:
     @staticmethod
@@ -529,3 +532,8 @@ def __enrichDepth(var, var_min, var_max):
         var.axis = "Z"
         var.positive = "Down"
 
+
+class DomsCAMLFormatter:
+    @staticmethod
+    def create(executionId, results, params, details):
+        pass
\ No newline at end of file
diff --git a/analysis/webservice/nexus_tornado/request/renderers/NexusCAMLRenderer.py b/analysis/webservice/nexus_tornado/request/renderers/NexusCAMLRenderer.py
new file mode 100644
index 00000000..0ca2653d
--- /dev/null
+++ b/analysis/webservice/nexus_tornado/request/renderers/NexusCAMLRenderer.py
@@ -0,0 +1,19 @@
+import sys
+import traceback
+import json
+
+
+class NexusJSONRenderer(object):
+    def __init__(self, nexus_request):
+        self.request = nexus_request
+
+    def render(self, tornado_handler, result):
+        tornado_handler.set_header("Content-Type", "application/json")
+        try:
+            result_str = result.toCAML()
+            tornado_handler.write(result_str)
+            tornado_handler.finish()
+        except AttributeError:
+            traceback.print_exc(file=sys.stdout)
+            tornado_handler.write(json.dumps(result, indent=4))
+            tornado_handler.finish()
diff --git a/analysis/webservice/nexus_tornado/request/renderers/NexusRendererFactory.py b/analysis/webservice/nexus_tornado/request/renderers/NexusRendererFactory.py
index 9fc06e31..17708ed4 100644
--- a/analysis/webservice/nexus_tornado/request/renderers/NexusRendererFactory.py
+++ b/analysis/webservice/nexus_tornado/request/renderers/NexusRendererFactory.py
@@ -1,5 +1,5 @@
 class NexusRendererFactory(object):
-    content_types = ["CSV", "JSON", "XML", "PNG", "NETCDF", "ZIP"]
+    content_types = ["CSV", "JSON", "XML", "PNG", "NETCDF", "ZIP", "CAML"]
     module = __import__(__name__)
 
     @classmethod

From c7d85e1984d4b6ab86b719ba99cbd0f7b4785460 Mon Sep 17 00:00:00 2001
From: rileykk <rileykk@jpl.nasa.gov>
Date: Wed, 7 Sep 2022 16:34:48 -0700
Subject: [PATCH 02/38] Empty CAML return + utility function

Also a quick test on parameter passing
---
 analysis/webservice/algorithms/doms/BaseDomsHandler.py | 7 ++++++-
 analysis/webservice/webmodel/NexusRequestObject.py     | 3 +++
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/analysis/webservice/algorithms/doms/BaseDomsHandler.py b/analysis/webservice/algorithms/doms/BaseDomsHandler.py
index f513a296..36785b83 100644
--- a/analysis/webservice/algorithms/doms/BaseDomsHandler.py
+++ b/analysis/webservice/algorithms/doms/BaseDomsHandler.py
@@ -536,4 +536,9 @@ def __enrichDepth(var, var_min, var_max):
 class DomsCAMLFormatter:
     @staticmethod
     def create(executionId, results, params, details):
-        pass
\ No newline at end of file
+        query = {}
+        result = {}
+
+        return json.dumps(
+            {'query': query, 'result': result, 'test': params}
+        )
diff --git a/analysis/webservice/webmodel/NexusRequestObject.py b/analysis/webservice/webmodel/NexusRequestObject.py
index 3779cf96..7aa41a37 100644
--- a/analysis/webservice/webmodel/NexusRequestObject.py
+++ b/analysis/webservice/webmodel/NexusRequestObject.py
@@ -23,6 +23,9 @@ def __init__(self, reqHandler):
     def get_argument(self, name, default=None):
         return self.requestHandler.get_argument(name, default=default)
 
+    def get_arguments(self, name):
+        return self.requestHandler.get_arguments(name)
+
     def get_list_int_arg(self, name, default=None):
         arg = self.get_argument(name, default=default)
         return arg.split(',')

From 6609f06a2594d83a0c3ddba47d222bdd486b709c Mon Sep 17 00:00:00 2001
From: rileykk <rileykk@jpl.nasa.gov>
Date: Thu, 8 Sep 2022 12:25:09 -0700
Subject: [PATCH 03/38] Fixed issue to get renderer working with empty output

Temporarily also passing through parameters for testing & dev purposes
---
 analysis/webservice/algorithms/doms/BaseDomsHandler.py        | 4 +++-
 .../nexus_tornado/request/renderers/NexusCAMLRenderer.py      | 2 +-
 .../webservice/nexus_tornado/request/renderers/__init__.py    | 3 ++-
 3 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/analysis/webservice/algorithms/doms/BaseDomsHandler.py b/analysis/webservice/algorithms/doms/BaseDomsHandler.py
index 36785b83..b1f2329e 100644
--- a/analysis/webservice/algorithms/doms/BaseDomsHandler.py
+++ b/analysis/webservice/algorithms/doms/BaseDomsHandler.py
@@ -540,5 +540,7 @@ def create(executionId, results, params, details):
         result = {}
 
         return json.dumps(
-            {'query': query, 'result': result, 'test': params}
+            {'query': query, 'result': result, 'test_params': params},
+            indent=4,
+            cls=DomsEncoder
         )
diff --git a/analysis/webservice/nexus_tornado/request/renderers/NexusCAMLRenderer.py b/analysis/webservice/nexus_tornado/request/renderers/NexusCAMLRenderer.py
index 0ca2653d..7942f81d 100644
--- a/analysis/webservice/nexus_tornado/request/renderers/NexusCAMLRenderer.py
+++ b/analysis/webservice/nexus_tornado/request/renderers/NexusCAMLRenderer.py
@@ -3,7 +3,7 @@
 import json
 
 
-class NexusJSONRenderer(object):
+class NexusCAMLRenderer(object):
     def __init__(self, nexus_request):
         self.request = nexus_request
 
diff --git a/analysis/webservice/nexus_tornado/request/renderers/__init__.py b/analysis/webservice/nexus_tornado/request/renderers/__init__.py
index 807eb7a5..1cd420e7 100644
--- a/analysis/webservice/nexus_tornado/request/renderers/__init__.py
+++ b/analysis/webservice/nexus_tornado/request/renderers/__init__.py
@@ -3,4 +3,5 @@
 from .NexusCSVRenderer import NexusCSVRenderer
 from .NexusNETCDFRenderer import NexusNETCDFRenderer
 from .NexusPNGRenderer import NexusPNGRenderer
-from .NexusZIPRenderer import NexusZIPRenderer
\ No newline at end of file
+from .NexusZIPRenderer import NexusZIPRenderer
+from .NexusCAMLRenderer import NexusCAMLRenderer
\ No newline at end of file

From 3710e8f5cae2a4bcc088b581ab54697ba5f63785 Mon Sep 17 00:00:00 2001
From: rileykk <rileykk@jpl.nasa.gov>
Date: Thu, 8 Sep 2022 13:35:17 -0700
Subject: [PATCH 04/38] Framework for passing params into CAML formatter

---
 analysis/webservice/algorithms_spark/Matchup.py | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/analysis/webservice/algorithms_spark/Matchup.py b/analysis/webservice/algorithms_spark/Matchup.py
index 5dafb313..d435a96b 100644
--- a/analysis/webservice/algorithms_spark/Matchup.py
+++ b/analysis/webservice/algorithms_spark/Matchup.py
@@ -199,10 +199,17 @@ def parse_arguments(self, request):
         start_seconds_from_epoch = int((start_time - EPOCH).total_seconds())
         end_seconds_from_epoch = int((end_time - EPOCH).total_seconds())
 
+        output_type = request.get_argument("output", default='JSON')
+
+        caml_params = {}
+
+        if output_type == 'CAML':
+            pass
+
         return bounding_polygon, primary_ds_name, secondary_ds_names, parameter_s, \
                start_time, start_seconds_from_epoch, end_time, end_seconds_from_epoch, \
                depth_min, depth_max, time_tolerance, radius_tolerance, \
-               platforms, match_once, result_size_limit
+               platforms, match_once, result_size_limit, output_type, caml_params
 
     def calc(self, request, **args):
         start = datetime.utcnow()
@@ -210,7 +217,7 @@ def calc(self, request, **args):
         bounding_polygon, primary_ds_name, secondary_ds_names, parameter_s, \
         start_time, start_seconds_from_epoch, end_time, end_seconds_from_epoch, \
         depth_min, depth_max, time_tolerance, radius_tolerance, \
-        platforms, match_once, result_size_limit = self.parse_arguments(request)
+        platforms, match_once, result_size_limit, output_type, caml_params = self.parse_arguments(request)
 
         with ResultsStorage(self.config) as resultsStorage:
 
@@ -251,6 +258,9 @@ def calc(self, request, **args):
             "parameter": parameter_s
         }
 
+        if output_type == 'CAML':
+            args['caml_params'] = caml_params
+
         if depth_min is not None:
             args["depthMin"] = float(depth_min)
 

From 6026e7dfbdbf406f757cd70ea528617ac6e6d44e Mon Sep 17 00:00:00 2001
From: rileykk <rileykk@jpl.nasa.gov>
Date: Thu, 8 Sep 2022 13:35:54 -0700
Subject: [PATCH 05/38] Query results in CAML output w/ some placeholder data

---
 .../algorithms/doms/BaseDomsHandler.py           | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/analysis/webservice/algorithms/doms/BaseDomsHandler.py b/analysis/webservice/algorithms/doms/BaseDomsHandler.py
index b1f2329e..e55b7cbe 100644
--- a/analysis/webservice/algorithms/doms/BaseDomsHandler.py
+++ b/analysis/webservice/algorithms/doms/BaseDomsHandler.py
@@ -539,6 +539,22 @@ def create(executionId, results, params, details):
         query = {}
         result = {}
 
+        query['apiRequest'] = ''
+        query['analysisName'] = ''
+        query['layerName'] = None
+        query['featureName'] = None
+
+        b = params['bbox'].split(',')
+
+        query['bounds'] = {
+            "lon_min": float(b[0]),
+            "lon_max": float(b[2]),
+            "lat_min": float(b[1]),
+            "lat_max": float(b[3]),
+            "time_start": params['startTime'].isoformat() + "+0000",
+            "time_end": params['endTime'].isoformat() + "+0000"
+        }
+
         return json.dumps(
             {'query': query, 'result': result, 'test_params': params},
             indent=4,

From 88f313694c402e5247b39564c55268de221326f1 Mon Sep 17 00:00:00 2001
From: rileykk <rileykk@jpl.nasa.gov>
Date: Thu, 8 Sep 2022 14:28:27 -0700
Subject: [PATCH 06/38] Prep to fill in CAML result data

---
 .../algorithms/doms/BaseDomsHandler.py        | 21 +++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/analysis/webservice/algorithms/doms/BaseDomsHandler.py b/analysis/webservice/algorithms/doms/BaseDomsHandler.py
index e55b7cbe..7f5178a2 100644
--- a/analysis/webservice/algorithms/doms/BaseDomsHandler.py
+++ b/analysis/webservice/algorithms/doms/BaseDomsHandler.py
@@ -536,6 +536,18 @@ def __enrichDepth(var, var_min, var_max):
 class DomsCAMLFormatter:
     @staticmethod
     def create(executionId, results, params, details):
+        def keyname(name, number):
+            if number == 0:
+                return name
+            else:
+                return f"{name}_{number}"
+
+        def empty_if_none(string):
+            if string is not None:
+                return string
+            else:
+                return ""
+
         query = {}
         result = {}
 
@@ -555,6 +567,15 @@ def create(executionId, results, params, details):
             "time_end": params['endTime'].isoformat() + "+0000"
         }
 
+        n_variable = 0
+        n_chart = 0
+
+        VAR = "variable"
+        CHART = "chart"
+
+        if len(results) > 0:
+            result[keyname(VAR, n_variable)] = {}
+
         return json.dumps(
             {'query': query, 'result': result, 'test_params': params},
             indent=4,

From ceec6c28cd3b18c8fec3403f9a61c3ec9544099e Mon Sep 17 00:00:00 2001
From: rileykk <rileykk@jpl.nasa.gov>
Date: Fri, 9 Sep 2022 14:20:44 -0700
Subject: [PATCH 07/38] First two charts - initial implementation

---
 .../algorithms/doms/BaseDomsHandler.py        | 68 ++++++++++++++++++-
 1 file changed, 65 insertions(+), 3 deletions(-)

diff --git a/analysis/webservice/algorithms/doms/BaseDomsHandler.py b/analysis/webservice/algorithms/doms/BaseDomsHandler.py
index 7f5178a2..2c515b18 100644
--- a/analysis/webservice/algorithms/doms/BaseDomsHandler.py
+++ b/analysis/webservice/algorithms/doms/BaseDomsHandler.py
@@ -536,6 +536,8 @@ def __enrichDepth(var, var_min, var_max):
 class DomsCAMLFormatter:
     @staticmethod
     def create(executionId, results, params, details):
+        import copy
+
         def keyname(name, number):
             if number == 0:
                 return name
@@ -548,6 +550,13 @@ def empty_if_none(string):
             else:
                 return ""
 
+        def datetime_to_iso(dt):
+            return dt.isoformat() + "+0000"
+
+        def get_match_by_variable_name(matches, variable_name):
+            m = [m for m in matches if m['cf_variable_name'] == variable_name]
+            return m[0]
+
         query = {}
         result = {}
 
@@ -563,8 +572,8 @@ def empty_if_none(string):
             "lon_max": float(b[2]),
             "lat_min": float(b[1]),
             "lat_max": float(b[3]),
-            "time_start": params['startTime'].isoformat() + "+0000",
-            "time_end": params['endTime'].isoformat() + "+0000"
+            "time_start": datetime_to_iso(params['startTime']),
+            "time_end": datetime_to_iso(params['endTime'])
         }
 
         n_variable = 0
@@ -574,7 +583,60 @@ def empty_if_none(string):
         CHART = "chart"
 
         if len(results) > 0:
-            result[keyname(VAR, n_variable)] = {}
+            result[keyname(VAR, n_variable)] = {
+                "object": "layer",
+                "name": results[0]["primary"][0]["cf_variable_name"],
+                "units": empty_if_none(results[0]["primary"][0]["variable_unit"])
+            }
+
+            n_variable += 1
+
+            result[keyname(VAR, n_variable)] = {
+                "object": "feature",
+                "name": results[0]['matches'][0]["secondary"][0]["cf_variable_name"],
+                "units": empty_if_none(results[0]['matches'][0]["secondary"][0]["variable_unit"])
+            }
+
+            n_variable += 1
+
+            data = [[], []]
+
+            for r in results:
+                data[0].append([datetime_to_iso(r['time']), r['primary'][0]['variable_value']])
+                data[1].append([datetime_to_iso(r['matches'][0]['time']), r['matches'][0]['secondary'][0]['variable_value']])
+
+            result[keyname(CHART, n_chart)] = {
+                "object": ["layer", "feature"],
+                "type": "xy_line_point",
+                "title": "Time Series",
+                "xAxis_label": f"{result[keyname(VAR, 0)]['name']} ({result[keyname(VAR, 0)]['units']})",
+                "yAxis_label": f"{result[keyname(VAR, 1)]['name']} ({result[keyname(VAR, 1)]['units']})",
+                "xySeries_data": copy.deepcopy(data),
+                "xySeries_labels": [query["layerName"], query["featureName"]]
+            }
+
+            n_chart += 1
+            data.clear()
+
+            for r in results:
+                data.append(
+                    [
+                        r['primary'][0]['variable_value'],
+                        r['matches'][0]['secondary'][0]['variable_value']
+                    ]
+                )
+
+            result[keyname(CHART, n_chart)] = {
+                "object": ["layer", "feature"],
+                "type": "xy_scatter_point",
+                "title": "Scatter Plot",
+                "xAxis_label": 'Time',
+                "yAxis_label": f"{result[keyname(VAR, 1)]['name']} ({result[keyname(VAR, 1)]['units']})",
+                "xySeries_data": copy.deepcopy(data),
+            }
+
+            n_chart += 1
+            data.clear()
 
         return json.dumps(
             {'query': query, 'result': result, 'test_params': params},

From 90cb0eeb59ff0cbd426a45ea5b3bc67049c2f7c2 Mon Sep 17 00:00:00 2001
From: rileykk <rileykk@jpl.nasa.gov>
Date: Fri, 9 Sep 2022 14:54:19 -0700
Subject: [PATCH 08/38] Parameters for CAML rendering in matchup

2 required 1 optional
---
 analysis/webservice/algorithms_spark/Matchup.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/analysis/webservice/algorithms_spark/Matchup.py b/analysis/webservice/algorithms_spark/Matchup.py
index d435a96b..57806d8f 100644
--- a/analysis/webservice/algorithms_spark/Matchup.py
+++ b/analysis/webservice/algorithms_spark/Matchup.py
@@ -204,7 +204,19 @@ def parse_arguments(self, request):
         caml_params = {}
 
         if output_type == 'CAML':
-            pass
+            layer = request.get_argument("camlLayer")
+            if layer is None:
+                raise NexusProcessingException(reason="Layer argument is required when outputting in CAML format", code=400)
+
+            feature = request.get_argument("camlFeature")
+            if feature is None:
+                raise NexusProcessingException(reason="Feature argument is required when outputting in CAML format", code=400)
+
+            raise_if_missing = request.get_boolean_arg("camlRaiseIfMissing")
+
+            caml_params['layer'] = layer
+            caml_params['feature'] = feature
+            caml_params['raise_if_missing'] = raise_if_missing
 
         return bounding_polygon, primary_ds_name, secondary_ds_names, parameter_s, \
                start_time, start_seconds_from_epoch, end_time, end_seconds_from_epoch, \

From f4e180111838613085c73e0fd7e35d4b98217ae2 Mon Sep 17 00:00:00 2001
From: rileykk <rileykk@jpl.nasa.gov>
Date: Fri, 9 Sep 2022 15:40:32 -0700
Subject: [PATCH 09/38] Ensure plotted results have the variable name specified
 in args

---
 .../algorithms/doms/BaseDomsHandler.py        | 45 +++++++++++++++----
 1 file changed, 36 insertions(+), 9 deletions(-)

diff --git a/analysis/webservice/algorithms/doms/BaseDomsHandler.py b/analysis/webservice/algorithms/doms/BaseDomsHandler.py
index 2c515b18..2cae0f27 100644
--- a/analysis/webservice/algorithms/doms/BaseDomsHandler.py
+++ b/analysis/webservice/algorithms/doms/BaseDomsHandler.py
@@ -30,7 +30,7 @@
 from . import config
 from . import geo
 from webservice.algorithms.NexusCalcHandler import NexusCalcHandler
-from webservice.webmodel import NexusResults
+from webservice.webmodel import NexusResults, NexusProcessingException
 
 EPOCH = timezone('UTC').localize(datetime(1970, 1, 1))
 ISO_8601 = '%Y-%m-%dT%H:%M:%S%z'
@@ -559,6 +559,7 @@ def get_match_by_variable_name(matches, variable_name):
 
         query = {}
         result = {}
+        caml_params = params['caml_params']
 
         query['apiRequest'] = ''
         query['analysisName'] = ''
@@ -585,16 +586,16 @@ def get_match_by_variable_name(matches, variable_name):
         if len(results) > 0:
             result[keyname(VAR, n_variable)] = {
                 "object": "layer",
-                "name": results[0]["primary"][0]["cf_variable_name"],
-                "units": empty_if_none(results[0]["primary"][0]["variable_unit"])
+                "name": caml_params['layer'],
+                "units": empty_if_none(get_match_by_variable_name(results[0]['primary'], caml_params['layer'])["variable_unit"])
             }
 
             n_variable += 1
 
             result[keyname(VAR, n_variable)] = {
                 "object": "feature",
-                "name": results[0]['matches'][0]["secondary"][0]["cf_variable_name"],
-                "units": empty_if_none(results[0]['matches'][0]["secondary"][0]["variable_unit"])
+                "name": caml_params['feature'],
+                "units": empty_if_none(get_match_by_variable_name(results[0]['matches'][0]['secondary'], caml_params['feature'])["variable_unit"])
             }
 
             n_variable += 1
@@ -602,8 +603,22 @@ def get_match_by_variable_name(matches, variable_name):
             data = [[], []]
 
             for r in results:
-                data[0].append([datetime_to_iso(r['time']), r['primary'][0]['variable_value']])
-                data[1].append([datetime_to_iso(r['matches'][0]['time']), r['matches'][0]['secondary'][0]['variable_value']])
+                secondary = None
+                secondary_match = None
+
+                for s in r['matches']:
+                    try:
+                        secondary_match = get_match_by_variable_name(s['secondary'], caml_params['feature'])
+                        secondary = s
+                        break
+                    except:
+                        pass
+
+                if secondary is None:
+                    continue
+
+                data[0].append([datetime_to_iso(r['time']), get_match_by_variable_name(r['primary'], caml_params['layer'])['variable_value']])
+                data[1].append([datetime_to_iso(secondary['time']), secondary_match['variable_value']])
 
             result[keyname(CHART, n_chart)] = {
                 "object": ["layer", "feature"],
@@ -619,10 +634,22 @@ def get_match_by_variable_name(matches, variable_name):
             data.clear()
 
             for r in results:
+                secondary_match = None
+
+                for s in r['matches']:
+                    try:
+                        secondary_match = get_match_by_variable_name(s['secondary'], caml_params['feature'])
+                        break
+                    except:
+                        pass
+
+                if secondary_match is None:
+                    continue
+
                 data.append(
                     [
-                        r['primary'][0]['variable_value'],
-                        r['matches'][0]['secondary'][0]['variable_value']
+                        get_match_by_variable_name(r['primary'], caml_params['layer'])['variable_value'],
+                        secondary_match['variable_value']
                     ]
                 )
 

From daf31bf12bcedc83612348eb95388fc0d73563b4 Mon Sep 17 00:00:00 2001
From: rileykk <rileykk@jpl.nasa.gov>
Date: Fri, 9 Sep 2022 16:45:58 -0700
Subject: [PATCH 10/38] Map output

---
 .../algorithms/doms/BaseDomsHandler.py        | 35 +++++++++++++++++--
 1 file changed, 33 insertions(+), 2 deletions(-)

diff --git a/analysis/webservice/algorithms/doms/BaseDomsHandler.py b/analysis/webservice/algorithms/doms/BaseDomsHandler.py
index 2cae0f27..ad922050 100644
--- a/analysis/webservice/algorithms/doms/BaseDomsHandler.py
+++ b/analysis/webservice/algorithms/doms/BaseDomsHandler.py
@@ -624,7 +624,7 @@ def get_match_by_variable_name(matches, variable_name):
                 "object": ["layer", "feature"],
                 "type": "xy_line_point",
                 "title": "Time Series",
-                "xAxis_label": f"{result[keyname(VAR, 0)]['name']} ({result[keyname(VAR, 0)]['units']})",
+                "xAxis_label": 'Time',
                 "yAxis_label": f"{result[keyname(VAR, 1)]['name']} ({result[keyname(VAR, 1)]['units']})",
                 "xySeries_data": copy.deepcopy(data),
                 "xySeries_labels": [query["layerName"], query["featureName"]]
@@ -657,7 +657,7 @@ def get_match_by_variable_name(matches, variable_name):
                 "object": ["layer", "feature"],
                 "type": "xy_scatter_point",
                 "title": "Scatter Plot",
-                "xAxis_label": 'Time',
+                "xAxis_label": f"{result[keyname(VAR, 0)]['name']} ({result[keyname(VAR, 0)]['units']})",
                 "yAxis_label": f"{result[keyname(VAR, 1)]['name']} ({result[keyname(VAR, 1)]['units']})",
                 "xySeries_data": copy.deepcopy(data),
             }
@@ -665,6 +665,37 @@ def get_match_by_variable_name(matches, variable_name):
             n_chart += 1
             data.clear()
 
+            for r in results:
+                secondary = None
+                secondary_match = None
+
+                for s in r['matches']:
+                    try:
+                        secondary_match = get_match_by_variable_name(s['secondary'], caml_params['feature'])
+                        secondary = s
+                        break
+                    except:
+                        pass
+
+                if secondary_match is None:
+                    continue
+
+                primary = get_match_by_variable_name(r['primary'], caml_params['layer'])
+
+                data.append([
+                    datetime_to_iso(secondary['time']),
+                    [secondary['lat'], secondary['lon']],
+                    primary['variable_value'] - secondary_match['variable_value']
+                ])
+
+            result['map'] = {
+                "object": ["feature"],
+                "type": "trajectory",
+                "title": "Along track colocation differences",
+                "colorbar_label": f"{result[keyname(VAR, 1)]['name']} ({result[keyname(VAR, 1)]['units']})",
+                "xySeries_data": copy.deepcopy(data),
+            }
+
         return json.dumps(
             {'query': query, 'result': result, 'test_params': params},
             indent=4,

From 4948ae955abc22ebd63a58fcde14bd0ed9607871 Mon Sep 17 00:00:00 2001
From: rileykk <rileykk@jpl.nasa.gov>
Date: Mon, 12 Sep 2022 12:05:56 -0700
Subject: [PATCH 11/38] Query fields

---
 .../webservice/algorithms/doms/BaseDomsHandler.py  | 14 +++++++-------
 .../request/renderers/NexusCAMLRenderer.py         |  2 +-
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/analysis/webservice/algorithms/doms/BaseDomsHandler.py b/analysis/webservice/algorithms/doms/BaseDomsHandler.py
index ad922050..1c7b5a40 100644
--- a/analysis/webservice/algorithms/doms/BaseDomsHandler.py
+++ b/analysis/webservice/algorithms/doms/BaseDomsHandler.py
@@ -106,8 +106,8 @@ def toCSV(self):
     def toNetCDF(self):
         return DomsNetCDFFormatter.create(self.__executionId, self.results(), self.__args, self.__details)
 
-    def toCAML(self):
-        return DomsCAMLFormatter.create(self.__executionId, self.results(), self.__args, self.__details)
+    def toCAML(self, request):
+        return DomsCAMLFormatter.create(self.__executionId, self.results(), self.__args, self.__details, request)
 
 
 class DomsCSVFormatter:
@@ -535,7 +535,7 @@ def __enrichDepth(var, var_min, var_max):
 
 class DomsCAMLFormatter:
     @staticmethod
-    def create(executionId, results, params, details):
+    def create(executionId, results, params, details, request):
         import copy
 
         def keyname(name, number):
@@ -561,10 +561,10 @@ def get_match_by_variable_name(matches, variable_name):
         result = {}
         caml_params = params['caml_params']
 
-        query['apiRequest'] = ''
-        query['analysisName'] = ''
-        query['layerName'] = None
-        query['featureName'] = None
+        query['apiRequest'] = f"{request.protocol}://{request.host}{request.uri}"
+        query['analysisName'] = 'colocation_trajectory'
+        query['layerName'] = params['primary']
+        query['featureName'] = params['matchup']
 
         b = params['bbox'].split(',')
 
diff --git a/analysis/webservice/nexus_tornado/request/renderers/NexusCAMLRenderer.py b/analysis/webservice/nexus_tornado/request/renderers/NexusCAMLRenderer.py
index 7942f81d..467cdf87 100644
--- a/analysis/webservice/nexus_tornado/request/renderers/NexusCAMLRenderer.py
+++ b/analysis/webservice/nexus_tornado/request/renderers/NexusCAMLRenderer.py
@@ -10,7 +10,7 @@ def __init__(self, nexus_request):
     def render(self, tornado_handler, result):
         tornado_handler.set_header("Content-Type", "application/json")
         try:
-            result_str = result.toCAML()
+            result_str = result.toCAML(tornado_handler.request)
             tornado_handler.write(result_str)
             tornado_handler.finish()
         except AttributeError:

From 4171f4fea98529a440af200c3fee47e32a6fb648 Mon Sep 17 00:00:00 2001
From: rileykk <rileykk@jpl.nasa.gov>
Date: Mon, 12 Sep 2022 12:50:18 -0700
Subject: [PATCH 12/38] TBA Histogram plots

---
 analysis/webservice/algorithms/doms/BaseDomsHandler.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/analysis/webservice/algorithms/doms/BaseDomsHandler.py b/analysis/webservice/algorithms/doms/BaseDomsHandler.py
index 1c7b5a40..859a82b3 100644
--- a/analysis/webservice/algorithms/doms/BaseDomsHandler.py
+++ b/analysis/webservice/algorithms/doms/BaseDomsHandler.py
@@ -665,6 +665,13 @@ def get_match_by_variable_name(matches, variable_name):
             n_chart += 1
             data.clear()
 
+            result[keyname(CHART, n_chart)] = 'TBA - Need to figure out histogram format'
+
+            n_chart += 1
+            data.clear()
+
+            result[keyname(CHART, n_chart)] = 'TBA - Need to figure out histogram format'
+
             for r in results:
                 secondary = None
                 secondary_match = None

From 2db044afa6f3ea91cbc60ab851937019803ccc72 Mon Sep 17 00:00:00 2001
From: Riley Kuttruff <72955101+RKuttruff@users.noreply.github.com>
Date: Mon, 12 Sep 2022 15:10:13 -0700
Subject: [PATCH 13/38] Coverage json hist (#4)

* Binned primary and secondary values by date

* Histogram implementation

* Cleanup

Co-authored-by: rileykk <rileykk@jpl.nasa.gov>
---
 .../algorithms/doms/BaseDomsHandler.py        | 101 +++++++++++++++++-
 1 file changed, 99 insertions(+), 2 deletions(-)

diff --git a/analysis/webservice/algorithms/doms/BaseDomsHandler.py b/analysis/webservice/algorithms/doms/BaseDomsHandler.py
index 859a82b3..5e6cb491 100644
--- a/analysis/webservice/algorithms/doms/BaseDomsHandler.py
+++ b/analysis/webservice/algorithms/doms/BaseDomsHandler.py
@@ -557,6 +557,9 @@ def get_match_by_variable_name(matches, variable_name):
             m = [m for m in matches if m['cf_variable_name'] == variable_name]
             return m[0]
 
+        def round_down_day(dt):
+            return datetime(*dt.timetuple()[:3])
+
         query = {}
         result = {}
         caml_params = params['caml_params']
@@ -665,12 +668,106 @@ def get_match_by_variable_name(matches, variable_name):
             n_chart += 1
             data.clear()
 
-            result[keyname(CHART, n_chart)] = 'TBA - Need to figure out histogram format'
+            primary_histdata = {}
+            secondary_histdata = {}
+
+            for r in results:
+                secondary = None
+                secondary_match = None
+
+                for s in r['matches']:
+                    try:
+                        secondary_match = get_match_by_variable_name(s['secondary'], caml_params['feature'])
+                        secondary = s
+                        break
+                    except:
+                        pass
+
+                if secondary is None:
+                    continue
+
+                pts = datetime_to_iso(round_down_day(r['time']))
+                sts = datetime_to_iso(round_down_day(secondary['time']))
+
+                if pts not in primary_histdata:
+                    primary_histdata[pts] = {
+                        'data': [],
+                        'hist': None
+                    }
+
+                if sts not in secondary_histdata:
+                    secondary_histdata[sts] = {
+                        'data': [],
+                        'hist': None
+                    }
+
+                primary_histdata[pts]['data'].append(get_match_by_variable_name(r['primary'], caml_params['layer'])['variable_value'])
+                secondary_histdata[sts]['data'].append(secondary_match['variable_value'])
+
+            bins = [-5, 0, 5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55]
+
+            for d in primary_histdata:
+                hist, _ = np.histogram(primary_histdata[d]['data'], bins=bins, density=False)
+                h = []
+
+                for i in range(len(hist)):
+                    h.append([bins[i], int(hist[i])])
+
+                primary_histdata[d]['hist'] = copy.deepcopy(h)
+
+            for d in secondary_histdata:
+                hist, _ = np.histogram(secondary_histdata[d]['data'], bins=bins, density=False)
+                h = []
+
+                for i in range(len(hist)):
+                    h.append([bins[i], int(hist[i])])
+
+                secondary_histdata[d]['hist'] = copy.deepcopy(h)
+
+            head = primary_histdata[next(iter(primary_histdata))]['hist']
+
+            data.append(head)
+
+            for d in primary_histdata:
+                d_hist = [d]
+
+                for bin in primary_histdata[d]['hist']:
+                    d_hist.append(bin)
+
+                data.append([d_hist])
+
+            result[keyname(CHART, n_chart)] = {
+                "object": ["layer"],
+                "type": "histogram",
+                "title": "Frequency Distribution over Time",
+                "xAxis_label": f"{result[keyname(VAR, 0)]['name']} ({result[keyname(VAR, 0)]['units']})",
+                "yAxis_label": "frequency (count)",
+                "xySeries_data": copy.deepcopy(data),
+            }
 
             n_chart += 1
             data.clear()
 
-            result[keyname(CHART, n_chart)] = 'TBA - Need to figure out histogram format'
+            data.append(head)
+
+            for d in secondary_histdata:
+                d_hist = [d]
+
+                for bin in secondary_histdata[d]['hist']:
+                    d_hist.append(bin)
+
+                data.append([d_hist])
+
+            result[keyname(CHART, n_chart)] = {
+                "object": ["feature"],
+                "type": "histogram",
+                "title": "Frequency Distribution over Time",
+                "xAxis_label": f"{result[keyname(VAR, 1)]['name']} ({result[keyname(VAR, 1)]['units']})",
+                "yAxis_label": "frequency (count)",
+                "xySeries_data": copy.deepcopy(data),
+            }
+
+            data.clear()
 
             for r in results:
                 secondary = None

From 202b53ccdd1448270c195738f6b0eadee0ebdcb7 Mon Sep 17 00:00:00 2001
From: rileykk <rileykk@jpl.nasa.gov>
Date: Mon, 12 Sep 2022 15:30:50 -0700
Subject: [PATCH 14/38] Processing for chart type arguments

---
 .../webservice/algorithms_spark/Matchup.py    | 39 +++++++++++++++++++
 1 file changed, 39 insertions(+)

diff --git a/analysis/webservice/algorithms_spark/Matchup.py b/analysis/webservice/algorithms_spark/Matchup.py
index 57806d8f..71bdbafb 100644
--- a/analysis/webservice/algorithms_spark/Matchup.py
+++ b/analysis/webservice/algorithms_spark/Matchup.py
@@ -214,9 +214,48 @@ def parse_arguments(self, request):
 
             raise_if_missing = request.get_boolean_arg("camlRaiseIfMissing")
 
+            CHART_TYPES = [
+                'time_series',
+                'scatter',
+                'histogram_primary',
+                'histogram_secondary',
+                'trajectory'
+            ]
+
+            types_arg = request.get_argument("camlChartTypes")
+
+            if types_arg is None:
+                types = {
+                    'time_series': True,
+                    'scatter': True,
+                    'histogram_primary': True,
+                    'histogram_secondary': True,
+                    'trajectory': True
+                }
+            else:
+                types_arg = types_arg.split(',')
+
+                types = {
+                    'time_series': False,
+                    'scatter': False,
+                    'histogram_primary': False,
+                    'histogram_secondary': False,
+                    'trajectory': False
+                }
+
+                for t in types_arg:
+                    if t not in CHART_TYPES:
+                        raise NexusProcessingException(
+                            reason=f"Invalid chart type argument: {t}",
+                            code=500
+                        )
+
+                    types[t] = True
+
             caml_params['layer'] = layer
             caml_params['feature'] = feature
             caml_params['raise_if_missing'] = raise_if_missing
+            caml_params['charts'] = types
 
         return bounding_polygon, primary_ds_name, secondary_ds_names, parameter_s, \
                start_time, start_seconds_from_epoch, end_time, end_seconds_from_epoch, \

From d21cb28e09d622befa15d3ba28c7a2cfcb658527 Mon Sep 17 00:00:00 2001
From: rileykk <rileykk@jpl.nasa.gov>
Date: Mon, 12 Sep 2022 15:54:56 -0700
Subject: [PATCH 15/38] Logic for chart filtering

---
 .../algorithms/doms/BaseDomsHandler.py        | 340 +++++++++---------
 1 file changed, 174 insertions(+), 166 deletions(-)

diff --git a/analysis/webservice/algorithms/doms/BaseDomsHandler.py b/analysis/webservice/algorithms/doms/BaseDomsHandler.py
index 5e6cb491..7445646a 100644
--- a/analysis/webservice/algorithms/doms/BaseDomsHandler.py
+++ b/analysis/webservice/algorithms/doms/BaseDomsHandler.py
@@ -603,202 +603,210 @@ def round_down_day(dt):
 
             n_variable += 1
 
-            data = [[], []]
-
-            for r in results:
-                secondary = None
-                secondary_match = None
-
-                for s in r['matches']:
-                    try:
-                        secondary_match = get_match_by_variable_name(s['secondary'], caml_params['feature'])
-                        secondary = s
-                        break
-                    except:
-                        pass
-
-                if secondary is None:
-                    continue
-
-                data[0].append([datetime_to_iso(r['time']), get_match_by_variable_name(r['primary'], caml_params['layer'])['variable_value']])
-                data[1].append([datetime_to_iso(secondary['time']), secondary_match['variable_value']])
-
-            result[keyname(CHART, n_chart)] = {
-                "object": ["layer", "feature"],
-                "type": "xy_line_point",
-                "title": "Time Series",
-                "xAxis_label": 'Time',
-                "yAxis_label": f"{result[keyname(VAR, 1)]['name']} ({result[keyname(VAR, 1)]['units']})",
-                "xySeries_data": copy.deepcopy(data),
-                "xySeries_labels": [query["layerName"], query["featureName"]]
-            }
+            data = []
 
-            n_chart += 1
-            data.clear()
-
-            for r in results:
-                secondary_match = None
-
-                for s in r['matches']:
-                    try:
-                        secondary_match = get_match_by_variable_name(s['secondary'], caml_params['feature'])
-                        break
-                    except:
-                        pass
-
-                if secondary_match is None:
-                    continue
-
-                data.append(
-                    [
-                        get_match_by_variable_name(r['primary'], caml_params['layer'])['variable_value'],
-                        secondary_match['variable_value']
-                    ]
-                )
-
-            result[keyname(CHART, n_chart)] = {
-                "object": ["layer", "feature"],
-                "type": "xy_scatter_point",
-                "title": "Scatter Plot",
-                "xAxis_label": f"{result[keyname(VAR, 0)]['name']} ({result[keyname(VAR, 0)]['units']})",
-                "yAxis_label": f"{result[keyname(VAR, 1)]['name']} ({result[keyname(VAR, 1)]['units']})",
-                "xySeries_data": copy.deepcopy(data),
-            }
+            if caml_params['charts']['time_series']:
+                data = [[], []]
 
-            n_chart += 1
-            data.clear()
+                for r in results:
+                    secondary = None
+                    secondary_match = None
 
-            primary_histdata = {}
-            secondary_histdata = {}
+                    for s in r['matches']:
+                        try:
+                            secondary_match = get_match_by_variable_name(s['secondary'], caml_params['feature'])
+                            secondary = s
+                            break
+                        except:
+                            pass
 
-            for r in results:
-                secondary = None
-                secondary_match = None
+                    if secondary is None:
+                        continue
 
-                for s in r['matches']:
-                    try:
-                        secondary_match = get_match_by_variable_name(s['secondary'], caml_params['feature'])
-                        secondary = s
-                        break
-                    except:
-                        pass
+                    data[0].append([datetime_to_iso(r['time']), get_match_by_variable_name(r['primary'], caml_params['layer'])['variable_value']])
+                    data[1].append([datetime_to_iso(secondary['time']), secondary_match['variable_value']])
+
+                result[keyname(CHART, n_chart)] = {
+                    "object": ["layer", "feature"],
+                    "type": "xy_line_point",
+                    "title": "Time Series",
+                    "xAxis_label": 'Time',
+                    "yAxis_label": f"{result[keyname(VAR, 1)]['name']} ({result[keyname(VAR, 1)]['units']})",
+                    "xySeries_data": copy.deepcopy(data),
+                    "xySeries_labels": [query["layerName"], query["featureName"]]
+                }
+
+                n_chart += 1
+                data.clear()
+
+            if caml_params['charts']['scatter']:
+                for r in results:
+                    secondary_match = None
+
+                    for s in r['matches']:
+                        try:
+                            secondary_match = get_match_by_variable_name(s['secondary'], caml_params['feature'])
+                            break
+                        except:
+                            pass
+
+                    if secondary_match is None:
+                        continue
 
-                if secondary is None:
-                    continue
+                    data.append(
+                        [
+                            get_match_by_variable_name(r['primary'], caml_params['layer'])['variable_value'],
+                            secondary_match['variable_value']
+                        ]
+                    )
+
+                result[keyname(CHART, n_chart)] = {
+                    "object": ["layer", "feature"],
+                    "type": "xy_scatter_point",
+                    "title": "Scatter Plot",
+                    "xAxis_label": f"{result[keyname(VAR, 0)]['name']} ({result[keyname(VAR, 0)]['units']})",
+                    "yAxis_label": f"{result[keyname(VAR, 1)]['name']} ({result[keyname(VAR, 1)]['units']})",
+                    "xySeries_data": copy.deepcopy(data),
+                }
+
+                n_chart += 1
+                data.clear()
+
+            if caml_params['charts']['histogram_primary'] or caml_params['charts']['histogram_secondary']:
+                primary_histdata = {}
+                secondary_histdata = {}
+
+                for r in results:
+                    secondary = None
+                    secondary_match = None
+
+                    for s in r['matches']:
+                        try:
+                            secondary_match = get_match_by_variable_name(s['secondary'], caml_params['feature'])
+                            secondary = s
+                            break
+                        except:
+                            pass
+
+                    if secondary is None:
+                        continue
 
-                pts = datetime_to_iso(round_down_day(r['time']))
-                sts = datetime_to_iso(round_down_day(secondary['time']))
+                    pts = datetime_to_iso(round_down_day(r['time']))
+                    sts = datetime_to_iso(round_down_day(secondary['time']))
 
-                if pts not in primary_histdata:
-                    primary_histdata[pts] = {
-                        'data': [],
-                        'hist': None
-                    }
+                    if pts not in primary_histdata:
+                        primary_histdata[pts] = {
+                            'data': [],
+                            'hist': None
+                        }
 
-                if sts not in secondary_histdata:
-                    secondary_histdata[sts] = {
-                        'data': [],
-                        'hist': None
-                    }
+                    if sts not in secondary_histdata:
+                        secondary_histdata[sts] = {
+                            'data': [],
+                            'hist': None
+                        }
 
-                primary_histdata[pts]['data'].append(get_match_by_variable_name(r['primary'], caml_params['layer'])['variable_value'])
-                secondary_histdata[sts]['data'].append(secondary_match['variable_value'])
+                    primary_histdata[pts]['data'].append(get_match_by_variable_name(r['primary'], caml_params['layer'])['variable_value'])
+                    secondary_histdata[sts]['data'].append(secondary_match['variable_value'])
 
-            bins = [-5, 0, 5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55]
+                bins = [-5, 0, 5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55]
 
-            for d in primary_histdata:
-                hist, _ = np.histogram(primary_histdata[d]['data'], bins=bins, density=False)
-                h = []
+                for d in primary_histdata:
+                    hist, _ = np.histogram(primary_histdata[d]['data'], bins=bins, density=False)
+                    h = []
 
-                for i in range(len(hist)):
-                    h.append([bins[i], int(hist[i])])
+                    for i in range(len(hist)):
+                        h.append([bins[i], int(hist[i])])
 
-                primary_histdata[d]['hist'] = copy.deepcopy(h)
+                    primary_histdata[d]['hist'] = copy.deepcopy(h)
 
-            for d in secondary_histdata:
-                hist, _ = np.histogram(secondary_histdata[d]['data'], bins=bins, density=False)
-                h = []
+                for d in secondary_histdata:
+                    hist, _ = np.histogram(secondary_histdata[d]['data'], bins=bins, density=False)
+                    h = []
 
-                for i in range(len(hist)):
-                    h.append([bins[i], int(hist[i])])
+                    for i in range(len(hist)):
+                        h.append([bins[i], int(hist[i])])
 
-                secondary_histdata[d]['hist'] = copy.deepcopy(h)
+                    secondary_histdata[d]['hist'] = copy.deepcopy(h)
 
-            head = primary_histdata[next(iter(primary_histdata))]['hist']
+                head = primary_histdata[next(iter(primary_histdata))]['hist']
 
-            data.append(head)
+                if caml_params['charts']['histogram_primary']:
+                    data.append(head)
 
-            for d in primary_histdata:
-                d_hist = [d]
+                    for d in primary_histdata:
+                        d_hist = [d]
 
-                for bin in primary_histdata[d]['hist']:
-                    d_hist.append(bin)
+                        for bin in primary_histdata[d]['hist']:
+                            d_hist.append(bin)
 
-                data.append([d_hist])
+                        data.append([d_hist])
 
-            result[keyname(CHART, n_chart)] = {
-                "object": ["layer"],
-                "type": "histogram",
-                "title": "Frequency Distribution over Time",
-                "xAxis_label": f"{result[keyname(VAR, 0)]['name']} ({result[keyname(VAR, 0)]['units']})",
-                "yAxis_label": "frequency (count)",
-                "xySeries_data": copy.deepcopy(data),
-            }
+                    result[keyname(CHART, n_chart)] = {
+                        "object": ["layer"],
+                        "type": "histogram",
+                        "title": "Frequency Distribution over Time",
+                        "xAxis_label": f"{result[keyname(VAR, 0)]['name']} ({result[keyname(VAR, 0)]['units']})",
+                        "yAxis_label": "frequency (count)",
+                        "xySeries_data": copy.deepcopy(data),
+                    }
 
-            n_chart += 1
-            data.clear()
+                    n_chart += 1
+                    data.clear()
 
-            data.append(head)
+                if caml_params['charts']['histogram_secondary']:
+                    data.append(head)
 
-            for d in secondary_histdata:
-                d_hist = [d]
+                    for d in secondary_histdata:
+                        d_hist = [d]
 
-                for bin in secondary_histdata[d]['hist']:
-                    d_hist.append(bin)
+                        for bin in secondary_histdata[d]['hist']:
+                            d_hist.append(bin)
 
-                data.append([d_hist])
+                        data.append([d_hist])
 
-            result[keyname(CHART, n_chart)] = {
-                "object": ["feature"],
-                "type": "histogram",
-                "title": "Frequency Distribution over Time",
-                "xAxis_label": f"{result[keyname(VAR, 1)]['name']} ({result[keyname(VAR, 1)]['units']})",
-                "yAxis_label": "frequency (count)",
-                "xySeries_data": copy.deepcopy(data),
-            }
+                    result[keyname(CHART, n_chart)] = {
+                        "object": ["feature"],
+                        "type": "histogram",
+                        "title": "Frequency Distribution over Time",
+                        "xAxis_label": f"{result[keyname(VAR, 1)]['name']} ({result[keyname(VAR, 1)]['units']})",
+                        "yAxis_label": "frequency (count)",
+                        "xySeries_data": copy.deepcopy(data),
+                    }
 
-            data.clear()
-
-            for r in results:
-                secondary = None
-                secondary_match = None
-
-                for s in r['matches']:
-                    try:
-                        secondary_match = get_match_by_variable_name(s['secondary'], caml_params['feature'])
-                        secondary = s
-                        break
-                    except:
-                        pass
-
-                if secondary_match is None:
-                    continue
-
-                primary = get_match_by_variable_name(r['primary'], caml_params['layer'])
-
-                data.append([
-                    datetime_to_iso(secondary['time']),
-                    [secondary['lat'], secondary['lon']],
-                    primary['variable_value'] - secondary_match['variable_value']
-                ])
-
-            result['map'] = {
-                "object": ["feature"],
-                "type": "trajectory",
-                "title": "Along track colocation differences",
-                "colorbar_label": f"{result[keyname(VAR, 1)]['name']} ({result[keyname(VAR, 1)]['units']})",
-                "xySeries_data": copy.deepcopy(data),
-            }
+                    data.clear()
+
+            if caml_params['charts']['trajectory']:
+                for r in results:
+                    secondary = None
+                    secondary_match = None
+
+                    for s in r['matches']:
+                        try:
+                            secondary_match = get_match_by_variable_name(s['secondary'], caml_params['feature'])
+                            secondary = s
+                            break
+                        except:
+                            pass
+
+                    if secondary_match is None:
+                        continue
+
+                    primary = get_match_by_variable_name(r['primary'], caml_params['layer'])
+
+                    data.append([
+                        datetime_to_iso(secondary['time']),
+                        [secondary['lat'], secondary['lon']],
+                        primary['variable_value'] - secondary_match['variable_value']
+                    ])
+
+                result['map'] = {
+                    "object": ["feature"],
+                    "type": "trajectory",
+                    "title": "Along track colocation differences",
+                    "colorbar_label": f"{result[keyname(VAR, 1)]['name']} ({result[keyname(VAR, 1)]['units']})",
+                    "xySeries_data": copy.deepcopy(data),
+                }
 
         return json.dumps(
             {'query': query, 'result': result, 'test_params': params},

From 3a2879b52d82272b74be8094006922c02192f0ee Mon Sep 17 00:00:00 2001
From: rileykk <rileykk@jpl.nasa.gov>
Date: Wed, 14 Sep 2022 13:41:27 -0700
Subject: [PATCH 16/38] Fixed output lat/lon types in diff map

---
 analysis/webservice/algorithms/doms/BaseDomsHandler.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/analysis/webservice/algorithms/doms/BaseDomsHandler.py b/analysis/webservice/algorithms/doms/BaseDomsHandler.py
index 7445646a..c3fd0783 100644
--- a/analysis/webservice/algorithms/doms/BaseDomsHandler.py
+++ b/analysis/webservice/algorithms/doms/BaseDomsHandler.py
@@ -796,7 +796,7 @@ def round_down_day(dt):
 
                     data.append([
                         datetime_to_iso(secondary['time']),
-                        [secondary['lat'], secondary['lon']],
+                        [float(secondary['lat']), float(secondary['lon'])],
                         primary['variable_value'] - secondary_match['variable_value']
                     ])
 

From aa805c4e3cc8007bc320525e9a794567b51627f7 Mon Sep 17 00:00:00 2001
From: rileykk <rileykk@jpl.nasa.gov>
Date: Fri, 16 Sep 2022 12:53:01 -0700
Subject: [PATCH 17/38] Renamed layer & feature variables to better fit
 CDMS/SDAP terminology

---
 .../algorithms/doms/BaseDomsHandler.py        | 44 +++++++++----------
 .../webservice/algorithms_spark/Matchup.py    | 18 ++++----
 2 files changed, 31 insertions(+), 31 deletions(-)

diff --git a/analysis/webservice/algorithms/doms/BaseDomsHandler.py b/analysis/webservice/algorithms/doms/BaseDomsHandler.py
index c3fd0783..de63baee 100644
--- a/analysis/webservice/algorithms/doms/BaseDomsHandler.py
+++ b/analysis/webservice/algorithms/doms/BaseDomsHandler.py
@@ -566,8 +566,8 @@ def round_down_day(dt):
 
         query['apiRequest'] = f"{request.protocol}://{request.host}{request.uri}"
         query['analysisName'] = 'colocation_trajectory'
-        query['layerName'] = params['primary']
-        query['featureName'] = params['matchup']
+        query['primaryName'] = params['primary']
+        query['secondaryName'] = params['matchup']
 
         b = params['bbox'].split(',')
 
@@ -588,17 +588,17 @@ def round_down_day(dt):
 
         if len(results) > 0:
             result[keyname(VAR, n_variable)] = {
-                "object": "layer",
-                "name": caml_params['layer'],
-                "units": empty_if_none(get_match_by_variable_name(results[0]['primary'], caml_params['layer'])["variable_unit"])
+                "object": "primary",
+                "name": caml_params['primary'],
+                "units": empty_if_none(get_match_by_variable_name(results[0]['primary'], caml_params['primary'])["variable_unit"])
             }
 
             n_variable += 1
 
             result[keyname(VAR, n_variable)] = {
-                "object": "feature",
-                "name": caml_params['feature'],
-                "units": empty_if_none(get_match_by_variable_name(results[0]['matches'][0]['secondary'], caml_params['feature'])["variable_unit"])
+                "object": "secondary",
+                "name": caml_params['secondary'],
+                "units": empty_if_none(get_match_by_variable_name(results[0]['matches'][0]['secondary'], caml_params['secondary'])["variable_unit"])
             }
 
             n_variable += 1
@@ -614,7 +614,7 @@ def round_down_day(dt):
 
                     for s in r['matches']:
                         try:
-                            secondary_match = get_match_by_variable_name(s['secondary'], caml_params['feature'])
+                            secondary_match = get_match_by_variable_name(s['secondary'], caml_params['secondary'])
                             secondary = s
                             break
                         except:
@@ -623,17 +623,17 @@ def round_down_day(dt):
                     if secondary is None:
                         continue
 
-                    data[0].append([datetime_to_iso(r['time']), get_match_by_variable_name(r['primary'], caml_params['layer'])['variable_value']])
+                    data[0].append([datetime_to_iso(r['time']), get_match_by_variable_name(r['primary'], caml_params['primary'])['variable_value']])
                     data[1].append([datetime_to_iso(secondary['time']), secondary_match['variable_value']])
 
                 result[keyname(CHART, n_chart)] = {
-                    "object": ["layer", "feature"],
+                    "object": ["primary", "secondary"],
                     "type": "xy_line_point",
                     "title": "Time Series",
                     "xAxis_label": 'Time',
                     "yAxis_label": f"{result[keyname(VAR, 1)]['name']} ({result[keyname(VAR, 1)]['units']})",
                     "xySeries_data": copy.deepcopy(data),
-                    "xySeries_labels": [query["layerName"], query["featureName"]]
+                    "xySeries_labels": [query["primaryName"], query["secondaryName"]]
                 }
 
                 n_chart += 1
@@ -645,7 +645,7 @@ def round_down_day(dt):
 
                     for s in r['matches']:
                         try:
-                            secondary_match = get_match_by_variable_name(s['secondary'], caml_params['feature'])
+                            secondary_match = get_match_by_variable_name(s['secondary'], caml_params['secondary'])
                             break
                         except:
                             pass
@@ -655,13 +655,13 @@ def round_down_day(dt):
 
                     data.append(
                         [
-                            get_match_by_variable_name(r['primary'], caml_params['layer'])['variable_value'],
+                            get_match_by_variable_name(r['primary'], caml_params['primary'])['variable_value'],
                             secondary_match['variable_value']
                         ]
                     )
 
                 result[keyname(CHART, n_chart)] = {
-                    "object": ["layer", "feature"],
+                    "object": ["primary", "secondary"],
                     "type": "xy_scatter_point",
                     "title": "Scatter Plot",
                     "xAxis_label": f"{result[keyname(VAR, 0)]['name']} ({result[keyname(VAR, 0)]['units']})",
@@ -682,7 +682,7 @@ def round_down_day(dt):
 
                     for s in r['matches']:
                         try:
-                            secondary_match = get_match_by_variable_name(s['secondary'], caml_params['feature'])
+                            secondary_match = get_match_by_variable_name(s['secondary'], caml_params['secondary'])
                             secondary = s
                             break
                         except:
@@ -706,7 +706,7 @@ def round_down_day(dt):
                             'hist': None
                         }
 
-                    primary_histdata[pts]['data'].append(get_match_by_variable_name(r['primary'], caml_params['layer'])['variable_value'])
+                    primary_histdata[pts]['data'].append(get_match_by_variable_name(r['primary'], caml_params['primary'])['variable_value'])
                     secondary_histdata[sts]['data'].append(secondary_match['variable_value'])
 
                 bins = [-5, 0, 5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55]
@@ -743,7 +743,7 @@ def round_down_day(dt):
                         data.append([d_hist])
 
                     result[keyname(CHART, n_chart)] = {
-                        "object": ["layer"],
+                        "object": ["primary"],
                         "type": "histogram",
                         "title": "Frequency Distribution over Time",
                         "xAxis_label": f"{result[keyname(VAR, 0)]['name']} ({result[keyname(VAR, 0)]['units']})",
@@ -766,7 +766,7 @@ def round_down_day(dt):
                         data.append([d_hist])
 
                     result[keyname(CHART, n_chart)] = {
-                        "object": ["feature"],
+                        "object": ["secondary"],
                         "type": "histogram",
                         "title": "Frequency Distribution over Time",
                         "xAxis_label": f"{result[keyname(VAR, 1)]['name']} ({result[keyname(VAR, 1)]['units']})",
@@ -783,7 +783,7 @@ def round_down_day(dt):
 
                     for s in r['matches']:
                         try:
-                            secondary_match = get_match_by_variable_name(s['secondary'], caml_params['feature'])
+                            secondary_match = get_match_by_variable_name(s['secondary'], caml_params['secondary'])
                             secondary = s
                             break
                         except:
@@ -792,7 +792,7 @@ def round_down_day(dt):
                     if secondary_match is None:
                         continue
 
-                    primary = get_match_by_variable_name(r['primary'], caml_params['layer'])
+                    primary = get_match_by_variable_name(r['primary'], caml_params['primary'])
 
                     data.append([
                         datetime_to_iso(secondary['time']),
@@ -801,7 +801,7 @@ def round_down_day(dt):
                     ])
 
                 result['map'] = {
-                    "object": ["feature"],
+                    "object": ["secondary"],
                     "type": "trajectory",
                     "title": "Along track colocation differences",
                     "colorbar_label": f"{result[keyname(VAR, 1)]['name']} ({result[keyname(VAR, 1)]['units']})",
diff --git a/analysis/webservice/algorithms_spark/Matchup.py b/analysis/webservice/algorithms_spark/Matchup.py
index 71bdbafb..0c1c4f64 100644
--- a/analysis/webservice/algorithms_spark/Matchup.py
+++ b/analysis/webservice/algorithms_spark/Matchup.py
@@ -204,13 +204,13 @@ def parse_arguments(self, request):
         caml_params = {}
 
         if output_type == 'CAML':
-            layer = request.get_argument("camlLayer")
-            if layer is None:
-                raise NexusProcessingException(reason="Layer argument is required when outputting in CAML format", code=400)
+            primary = request.get_argument("camlPrimary")
+            if primary is None:
+                raise NexusProcessingException(reason="Primary dataset argument is required when outputting in CAML format", code=400)
 
-            feature = request.get_argument("camlFeature")
-            if feature is None:
-                raise NexusProcessingException(reason="Feature argument is required when outputting in CAML format", code=400)
+            secondary = request.get_argument("camlSecondary")
+            if secondary is None:
+                raise NexusProcessingException(reason="Secondary dataset argument is required when outputting in CAML format", code=400)
 
             raise_if_missing = request.get_boolean_arg("camlRaiseIfMissing")
 
@@ -226,7 +226,7 @@ def parse_arguments(self, request):
 
             if types_arg is None:
                 types = {
-                    'time_series': True,
+                    'time_series': False,
                     'scatter': True,
                     'histogram_primary': True,
                     'histogram_secondary': True,
@@ -252,8 +252,8 @@ def parse_arguments(self, request):
 
                     types[t] = True
 
-            caml_params['layer'] = layer
-            caml_params['feature'] = feature
+            caml_params['primary'] = primary
+            caml_params['secondary'] = secondary
             caml_params['raise_if_missing'] = raise_if_missing
             caml_params['charts'] = types
 

From deb440906292c42f7d9d428a5fda1b0b2af00cad Mon Sep 17 00:00:00 2001
From: rileykk <rileykk@jpl.nasa.gov>
Date: Fri, 16 Sep 2022 14:40:44 -0700
Subject: [PATCH 18/38] Sorted data in time series output so that it doesn't
 need to be sorted when plotting

---
 analysis/webservice/algorithms/doms/BaseDomsHandler.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/analysis/webservice/algorithms/doms/BaseDomsHandler.py b/analysis/webservice/algorithms/doms/BaseDomsHandler.py
index de63baee..40d8a6e9 100644
--- a/analysis/webservice/algorithms/doms/BaseDomsHandler.py
+++ b/analysis/webservice/algorithms/doms/BaseDomsHandler.py
@@ -626,6 +626,9 @@ def round_down_day(dt):
                     data[0].append([datetime_to_iso(r['time']), get_match_by_variable_name(r['primary'], caml_params['primary'])['variable_value']])
                     data[1].append([datetime_to_iso(secondary['time']), secondary_match['variable_value']])
 
+                data[0].sort(key=lambda e: e[0])
+                data[1].sort(key=lambda e: e[0])
+
                 result[keyname(CHART, n_chart)] = {
                     "object": ["primary", "secondary"],
                     "type": "xy_line_point",

From 596bce5d1f7ae2e0c1cf11343e8ff3398b299cd6 Mon Sep 17 00:00:00 2001
From: rileykk <rileykk@jpl.nasa.gov>
Date: Mon, 19 Sep 2022 11:26:17 -0700
Subject: [PATCH 19/38] Updated changelog

---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 84b7a160..a7d620f0 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -18,6 +18,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Matchup validates insitu parameter using insitu API schema endpoint
 - Added domsresults endpoint to openapi spec
 - Added markdown table to matchup `platform` param in openapi spec
+- SDAP-401: Added JSON output formatting for use with COVERAGE to `/match_spark`
 ### Changed
 - SDAP-390: Changed `/doms` to `/cdms` and `doms_reader.py` to `cdms_reader.py`
 - domslist endpoint points to AWS insitu instead of doms insitu

From f264c57452e481b214b8d210f696fe9585d7f4b5 Mon Sep 17 00:00:00 2001
From: rileykk <rileykk@jpl.nasa.gov>
Date: Mon, 19 Sep 2022 11:48:21 -0700
Subject: [PATCH 20/38] Added EID to CAML output

---
 analysis/webservice/algorithms/doms/BaseDomsHandler.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/analysis/webservice/algorithms/doms/BaseDomsHandler.py b/analysis/webservice/algorithms/doms/BaseDomsHandler.py
index 40d8a6e9..1d8cfed0 100644
--- a/analysis/webservice/algorithms/doms/BaseDomsHandler.py
+++ b/analysis/webservice/algorithms/doms/BaseDomsHandler.py
@@ -812,7 +812,7 @@ def round_down_day(dt):
                 }
 
         return json.dumps(
-            {'query': query, 'result': result, 'test_params': params},
+            {'executionId': executionId, 'query': query, 'result': result, 'test_params': params},
             indent=4,
             cls=DomsEncoder
         )

From 7f635db7e5ad62102cd4d7c4bf169a607b2a6f2c Mon Sep 17 00:00:00 2001
From: rileykk <rileykk@jpl.nasa.gov>
Date: Mon, 19 Sep 2022 11:49:36 -0700
Subject: [PATCH 21/38] CAMLSecondary parameter will override parameter
 parameter

It does not make sense for them to differ. Checks are maintained that it is a valid insitu parameter
---
 analysis/webservice/algorithms_spark/Matchup.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/analysis/webservice/algorithms_spark/Matchup.py b/analysis/webservice/algorithms_spark/Matchup.py
index 0c1c4f64..ee8acfe1 100644
--- a/analysis/webservice/algorithms_spark/Matchup.py
+++ b/analysis/webservice/algorithms_spark/Matchup.py
@@ -212,6 +212,12 @@ def parse_arguments(self, request):
             if secondary is None:
                 raise NexusProcessingException(reason="Secondary dataset argument is required when outputting in CAML format", code=400)
 
+            if secondary not in insitu_params:
+                raise NexusProcessingException(
+                    reason=f"Parameter {secondary} not supported. Must be one of {insitu_params}", code=400)
+
+            parameter_s = secondary # Override parameter as it makes no sense for it to differ
+
             raise_if_missing = request.get_boolean_arg("camlRaiseIfMissing")
 
             CHART_TYPES = [

From 83b57ec22386fda873fa863643c2fed0be0dab2a Mon Sep 17 00:00:00 2001
From: rileykk <rileykk@jpl.nasa.gov>
Date: Mon, 19 Sep 2022 12:12:41 -0700
Subject: [PATCH 22/38] Start of CAML formatting impl for domsresults endpoint

---
 .../algorithms/doms/ResultsRetrieval.py       | 74 +++++++++++++++++++
 1 file changed, 74 insertions(+)

diff --git a/analysis/webservice/algorithms/doms/ResultsRetrieval.py b/analysis/webservice/algorithms/doms/ResultsRetrieval.py
index 566c718e..7e149128 100644
--- a/analysis/webservice/algorithms/doms/ResultsRetrieval.py
+++ b/analysis/webservice/algorithms/doms/ResultsRetrieval.py
@@ -19,8 +19,11 @@
 from . import ResultsStorage
 from webservice.NexusHandler import nexus_handler
 from webservice.webmodel import NexusProcessingException
+from webservice.algorithms.doms.insitu import query_insitu_schema
 
 
+insitu_schema = query_insitu_schema()
+
 @nexus_handler
 class DomsResultsRetrievalHandler(BaseDomsHandler.BaseDomsQueryCalcHandler):
     name = "DOMS Resultset Retrieval"
@@ -34,8 +37,73 @@ def __init__(self, tile_service_factory, config=None):
         self.config = config
 
     def calc(self, computeOptions, **args):
+        from webservice.algorithms_spark.Matchup import get_insitu_params
+
         execution_id = computeOptions.get_argument("id", None)
 
+        caml_params = {}
+
+        output_type = computeOptions.get_argument("output", default='JSON')
+
+        if output_type == 'CAML':
+            primary = computeOptions.get_argument("camlPrimary")
+            if primary is None:
+                raise NexusProcessingException(
+                    reason="Primary dataset argument is required when outputting in CAML format", code=400)
+
+            secondary = computeOptions.get_argument("camlSecondary")
+            if secondary is None:
+                raise NexusProcessingException(
+                    reason="Secondary dataset argument is required when outputting in CAML format", code=400)
+
+            insitu_params = get_insitu_params(insitu_schema)
+
+            if secondary not in insitu_params:
+                raise NexusProcessingException(
+                    reason=f"Parameter {secondary} not supported. Must be one of {insitu_params}", code=400)
+
+            CHART_TYPES = [
+                'time_series',
+                'scatter',
+                'histogram_primary',
+                'histogram_secondary',
+                'trajectory'
+            ]
+
+            types_arg = computeOptions.get_argument("camlChartTypes")
+
+            if types_arg is None:
+                types = {
+                    'time_series': False,
+                    'scatter': True,
+                    'histogram_primary': True,
+                    'histogram_secondary': True,
+                    'trajectory': True
+                }
+            else:
+                types_arg = types_arg.split(',')
+
+                types = {
+                    'time_series': False,
+                    'scatter': False,
+                    'histogram_primary': False,
+                    'histogram_secondary': False,
+                    'trajectory': False
+                }
+
+                for t in types_arg:
+                    if t not in CHART_TYPES:
+                        raise NexusProcessingException(
+                            reason=f"Invalid chart type argument: {t}",
+                            code=500
+                        )
+
+                    types[t] = True
+
+            caml_params['primary'] = primary
+            caml_params['secondary'] = secondary
+            caml_params['charts'] = types
+
         try:
             execution_id = uuid.UUID(execution_id)
         except:
@@ -46,5 +114,11 @@ def calc(self, computeOptions, **args):
         with ResultsStorage.ResultsRetrieval(self.config) as storage:
             params, stats, data = storage.retrieveResults(execution_id, trim_data=simple_results)
 
+        if output_type == 'CAML':
+            params['caml_params'] = caml_params
+
+        if output_type == 'CAML':
+            raise NexusProcessingException(reason='CAML output for results retrieval is not yet implemented.', code=501)
+
         return BaseDomsHandler.DomsQueryResults(results=data, args=params, details=stats, bounds=None, count=None,
                                                 computeOptions=None, executionId=execution_id)

From 3fe5017c8f99d1b86a275570edaa6138d7222719 Mon Sep 17 00:00:00 2001
From: rileykk <rileykk@jpl.nasa.gov>
Date: Mon, 19 Sep 2022 12:14:50 -0700
Subject: [PATCH 23/38] Fix key error for results retrieval formatting

---
 .../algorithms/doms/BaseDomsHandler.py        | 35 +++++++++++++------
 1 file changed, 25 insertions(+), 10 deletions(-)

diff --git a/analysis/webservice/algorithms/doms/BaseDomsHandler.py b/analysis/webservice/algorithms/doms/BaseDomsHandler.py
index 1d8cfed0..8257d9af 100644
--- a/analysis/webservice/algorithms/doms/BaseDomsHandler.py
+++ b/analysis/webservice/algorithms/doms/BaseDomsHandler.py
@@ -587,19 +587,34 @@ def round_down_day(dt):
         CHART = "chart"
 
         if len(results) > 0:
-            result[keyname(VAR, n_variable)] = {
-                "object": "primary",
-                "name": caml_params['primary'],
-                "units": empty_if_none(get_match_by_variable_name(results[0]['primary'], caml_params['primary'])["variable_unit"])
-            }
+            try:
+                result[keyname(VAR, n_variable)] = {
+                    "object": "primary",
+                    "name": caml_params['primary'],
+                    "units": empty_if_none(get_match_by_variable_name(results[0]['primary'], caml_params['primary'])["variable_unit"])
+                }
+            except KeyError:
+                result[keyname(VAR, n_variable)] = {
+                    "object": "primary",
+                    "name": caml_params['primary'],
+                    "units": ""
+                }
 
             n_variable += 1
 
-            result[keyname(VAR, n_variable)] = {
-                "object": "secondary",
-                "name": caml_params['secondary'],
-                "units": empty_if_none(get_match_by_variable_name(results[0]['matches'][0]['secondary'], caml_params['secondary'])["variable_unit"])
-            }
+            try:
+                result[keyname(VAR, n_variable)] = {
+                    "object": "secondary",
+                    "name": caml_params['secondary'],
+                    "units": empty_if_none(get_match_by_variable_name(results[0]['matches'][0]['secondary'], caml_params['secondary'])["variable_unit"])
+                }
+            except KeyError:
+                result[keyname(VAR, n_variable)] = {
+                    "object": "secondary",
+                    "name": caml_params['secondary'],
+                    "units": ""
+                }
+
 
             n_variable += 1
 

From b81c0d5da4b4ffc23c2531b109feffbe33a7a2f5 Mon Sep 17 00:00:00 2001
From: rileykk <rileykk@jpl.nasa.gov>
Date: Mon, 19 Sep 2022 12:15:53 -0700
Subject: [PATCH 24/38] Removed unused param

---
 analysis/webservice/algorithms_spark/Matchup.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/analysis/webservice/algorithms_spark/Matchup.py b/analysis/webservice/algorithms_spark/Matchup.py
index ee8acfe1..da5d11d8 100644
--- a/analysis/webservice/algorithms_spark/Matchup.py
+++ b/analysis/webservice/algorithms_spark/Matchup.py
@@ -218,8 +218,6 @@ def parse_arguments(self, request):
 
             parameter_s = secondary # Override parameter as it makes no sense for it to differ
 
-            raise_if_missing = request.get_boolean_arg("camlRaiseIfMissing")
-
             CHART_TYPES = [
                 'time_series',
                 'scatter',
@@ -260,7 +258,6 @@ def parse_arguments(self, request):
 
             caml_params['primary'] = primary
             caml_params['secondary'] = secondary
-            caml_params['raise_if_missing'] = raise_if_missing
             caml_params['charts'] = types
 
         return bounding_polygon, primary_ds_name, secondary_ds_names, parameter_s, \

From 99228cc98190edb9b62c944ed63ea4ee3ab19b27 Mon Sep 17 00:00:00 2001
From: rileykk <rileykk@jpl.nasa.gov>
Date: Mon, 19 Sep 2022 12:29:35 -0700
Subject: [PATCH 25/38] Fixed error code

---
 analysis/webservice/algorithms_spark/Matchup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/analysis/webservice/algorithms_spark/Matchup.py b/analysis/webservice/algorithms_spark/Matchup.py
index da5d11d8..6b1e7482 100644
--- a/analysis/webservice/algorithms_spark/Matchup.py
+++ b/analysis/webservice/algorithms_spark/Matchup.py
@@ -251,7 +251,7 @@ def parse_arguments(self, request):
                     if t not in CHART_TYPES:
                         raise NexusProcessingException(
                             reason=f"Invalid chart type argument: {t}",
-                            code=500
+                            code=400
                         )
 
                     types[t] = True

From b589dc0a18554bfc1dc54a1b91bf7854a41d0bd3 Mon Sep 17 00:00:00 2001
From: rileykk <rileykk@jpl.nasa.gov>
Date: Mon, 19 Sep 2022 12:30:10 -0700
Subject: [PATCH 26/38] Fixed error code

---
 analysis/webservice/algorithms/doms/ResultsRetrieval.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/analysis/webservice/algorithms/doms/ResultsRetrieval.py b/analysis/webservice/algorithms/doms/ResultsRetrieval.py
index 7e149128..05542785 100644
--- a/analysis/webservice/algorithms/doms/ResultsRetrieval.py
+++ b/analysis/webservice/algorithms/doms/ResultsRetrieval.py
@@ -95,7 +95,7 @@ def calc(self, computeOptions, **args):
                     if t not in CHART_TYPES:
                         raise NexusProcessingException(
                             reason=f"Invalid chart type argument: {t}",
-                            code=500
+                            code=400
                         )
 
                     types[t] = True

From 9e497bd756d70c8f2c036c853f7b330587213be9 Mon Sep 17 00:00:00 2001
From: rileykk <rileykk@jpl.nasa.gov>
Date: Mon, 19 Sep 2022 16:20:07 -0700
Subject: [PATCH 27/38] Implementation of CAML formatter for domsresults
 endpoint

May still need a bit of testing...
---
 .../algorithms/doms/BaseDomsHandler.py        | 72 ++++++++++++++-----
 .../algorithms/doms/ResultsRetrieval.py       |  2 +
 .../webservice/algorithms_spark/Matchup.py    |  1 +
 3 files changed, 57 insertions(+), 18 deletions(-)

diff --git a/analysis/webservice/algorithms/doms/BaseDomsHandler.py b/analysis/webservice/algorithms/doms/BaseDomsHandler.py
index 8257d9af..1a5c22d7 100644
--- a/analysis/webservice/algorithms/doms/BaseDomsHandler.py
+++ b/analysis/webservice/algorithms/doms/BaseDomsHandler.py
@@ -602,6 +602,8 @@ def round_down_day(dt):
 
             n_variable += 1
 
+            results.sort(key=lambda e: e['time'])
+
             try:
                 result[keyname(VAR, n_variable)] = {
                     "object": "secondary",
@@ -629,20 +631,29 @@ def round_down_day(dt):
 
                     for s in r['matches']:
                         try:
-                            secondary_match = get_match_by_variable_name(s['secondary'], caml_params['secondary'])
-                            secondary = s
-                            break
+                            if caml_params['format'] == 'Matchup':
+                                secondary_match = get_match_by_variable_name(s['secondary'], caml_params['secondary'])
+                                secondary = s
+                                break
+                            else:
+                                if caml_params['secondary'] in s:
+                                    secondary = s
+                                    secondary_match = {'variable_value': s[caml_params['secondary']]}
+                                    break
                         except:
                             pass
 
                     if secondary is None:
                         continue
 
-                    data[0].append([datetime_to_iso(r['time']), get_match_by_variable_name(r['primary'], caml_params['primary'])['variable_value']])
+                    if caml_params['format'] == 'Matchup':
+                        data[0].append([datetime_to_iso(r['time']), get_match_by_variable_name(r['primary'], caml_params['primary'])['variable_value']])
+                    else:
+                        data[0].append([datetime_to_iso(r['time']), r[caml_params['primary']]])
                     data[1].append([datetime_to_iso(secondary['time']), secondary_match['variable_value']])
 
-                data[0].sort(key=lambda e: e[0])
-                data[1].sort(key=lambda e: e[0])
+                data[0].sort(key=lambda e: (e[0], e[1]))
+                data[1].sort(key=lambda e: (e[0], e[1]))
 
                 result[keyname(CHART, n_chart)] = {
                     "object": ["primary", "secondary"],
@@ -663,8 +674,13 @@ def round_down_day(dt):
 
                     for s in r['matches']:
                         try:
-                            secondary_match = get_match_by_variable_name(s['secondary'], caml_params['secondary'])
-                            break
+                            if caml_params['format'] == 'Matchup':
+                                secondary_match = get_match_by_variable_name(s['secondary'], caml_params['secondary'])
+                                break
+                            else:
+                                if caml_params['secondary'] in s:
+                                    secondary_match = {'variable_value': s[caml_params['secondary']]}
+                                    break
                         except:
                             pass
 
@@ -673,11 +689,14 @@ def round_down_day(dt):
 
                     data.append(
                         [
-                            get_match_by_variable_name(r['primary'], caml_params['primary'])['variable_value'],
+                            get_match_by_variable_name(r['primary'], caml_params['primary'])['variable_value']
+                            if caml_params['format'] == 'Matchup' else r[caml_params['primary']],
                             secondary_match['variable_value']
                         ]
                     )
 
+                data.sort(key=lambda e: (e[0], e[1]))
+
                 result[keyname(CHART, n_chart)] = {
                     "object": ["primary", "secondary"],
                     "type": "xy_scatter_point",
@@ -700,9 +719,15 @@ def round_down_day(dt):
 
                     for s in r['matches']:
                         try:
-                            secondary_match = get_match_by_variable_name(s['secondary'], caml_params['secondary'])
-                            secondary = s
-                            break
+                            if caml_params['format'] == 'Matchup':
+                                secondary_match = get_match_by_variable_name(s['secondary'], caml_params['secondary'])
+                                secondary = s
+                                break
+                            else:
+                                if caml_params['secondary'] in s:
+                                    secondary = s
+                                    secondary_match = {'variable_value': s[caml_params['secondary']]}
+                                    break
                         except:
                             pass
 
@@ -724,7 +749,10 @@ def round_down_day(dt):
                             'hist': None
                         }
 
-                    primary_histdata[pts]['data'].append(get_match_by_variable_name(r['primary'], caml_params['primary'])['variable_value'])
+                    if caml_params['format'] == 'Matchup':
+                        primary_histdata[pts]['data'].append(get_match_by_variable_name(r['primary'], caml_params['primary'])['variable_value'])
+                    else:
+                        primary_histdata[pts]['data'].append(r[caml_params['primary']])
                     secondary_histdata[sts]['data'].append(secondary_match['variable_value'])
 
                 bins = [-5, 0, 5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55]
@@ -801,16 +829,22 @@ def round_down_day(dt):
 
                     for s in r['matches']:
                         try:
-                            secondary_match = get_match_by_variable_name(s['secondary'], caml_params['secondary'])
-                            secondary = s
-                            break
+                            if caml_params['format'] == 'Matchup':
+                                secondary_match = get_match_by_variable_name(s['secondary'], caml_params['secondary'])
+                                secondary = s
+                                break
+                            else:
+                                if caml_params['secondary'] in s:
+                                    secondary = s
+                                    secondary_match = {'variable_value': s[caml_params['secondary']]}
+                                    break
                         except:
                             pass
 
                     if secondary_match is None:
                         continue
 
-                    primary = get_match_by_variable_name(r['primary'], caml_params['primary'])
+                    primary = get_match_by_variable_name(r['primary'], caml_params['primary']) if caml_params['format'] == 'Matchup' else {'variable_value': r[caml_params['primary']]}
 
                     data.append([
                         datetime_to_iso(secondary['time']),
@@ -818,6 +852,8 @@ def round_down_day(dt):
                         primary['variable_value'] - secondary_match['variable_value']
                     ])
 
+                data.sort(key=lambda e: (e[0], e[1][0], e[1][1], e[2]))
+
                 result['map'] = {
                     "object": ["secondary"],
                     "type": "trajectory",
@@ -827,7 +863,7 @@ def round_down_day(dt):
                 }
 
         return json.dumps(
-            {'executionId': executionId, 'query': query, 'result': result, 'test_params': params},
+            {'executionId': executionId, 'query': query, 'result': result, 'params': params},
             indent=4,
             cls=DomsEncoder
         )
diff --git a/analysis/webservice/algorithms/doms/ResultsRetrieval.py b/analysis/webservice/algorithms/doms/ResultsRetrieval.py
index 05542785..c203a2ee 100644
--- a/analysis/webservice/algorithms/doms/ResultsRetrieval.py
+++ b/analysis/webservice/algorithms/doms/ResultsRetrieval.py
@@ -103,6 +103,7 @@ def calc(self, computeOptions, **args):
             caml_params['primary'] = primary
             caml_params['secondary'] = secondary
             caml_params['charts'] = types
+            caml_params['format'] = 'Results'
 
         try:
             execution_id = uuid.UUID(execution_id)
@@ -116,6 +117,7 @@ def calc(self, computeOptions, **args):
 
         if output_type == 'CAML':
             params['caml_params'] = caml_params
+            params['matchup'] = params['matchup'][0]
 
         if output_type == 'CAML':
             raise NexusProcessingException(reason='CAML output for results retrieval is not yet implemented.', code=501)
diff --git a/analysis/webservice/algorithms_spark/Matchup.py b/analysis/webservice/algorithms_spark/Matchup.py
index 6b1e7482..7da8a240 100644
--- a/analysis/webservice/algorithms_spark/Matchup.py
+++ b/analysis/webservice/algorithms_spark/Matchup.py
@@ -259,6 +259,7 @@ def parse_arguments(self, request):
             caml_params['primary'] = primary
             caml_params['secondary'] = secondary
             caml_params['charts'] = types
+            caml_params['format'] = 'Matchup'
 
         return bounding_polygon, primary_ds_name, secondary_ds_names, parameter_s, \
                start_time, start_seconds_from_epoch, end_time, end_seconds_from_epoch, \

From 2de2903d1ddba4e59eb291669175e3e08f7d6829 Mon Sep 17 00:00:00 2001
From: Riley Kuttruff <72955101+RKuttruff@users.noreply.github.com>
Date: Mon, 19 Sep 2022 20:44:24 -0700
Subject: [PATCH 28/38] Removed error raised on trying to use CAML formatter in
 domsresults

---
 analysis/webservice/algorithms/doms/ResultsRetrieval.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/analysis/webservice/algorithms/doms/ResultsRetrieval.py b/analysis/webservice/algorithms/doms/ResultsRetrieval.py
index c203a2ee..b3781060 100644
--- a/analysis/webservice/algorithms/doms/ResultsRetrieval.py
+++ b/analysis/webservice/algorithms/doms/ResultsRetrieval.py
@@ -119,8 +119,5 @@ def calc(self, computeOptions, **args):
             params['caml_params'] = caml_params
             params['matchup'] = params['matchup'][0]
 
-        if output_type == 'CAML':
-            raise NexusProcessingException(reason='CAML output for results retrieval is not yet implemented.', code=501)
-
         return BaseDomsHandler.DomsQueryResults(results=data, args=params, details=stats, bounds=None, count=None,
                                                 computeOptions=None, executionId=execution_id)

From 3a15b14d2d76402b5ba2e6f551656d006f42ac05 Mon Sep 17 00:00:00 2001
From: rileykk <rileykk@jpl.nasa.gov>
Date: Wed, 21 Sep 2022 15:37:55 -0700
Subject: [PATCH 29/38] Added argument to choose bins for histogram plots

---
 .../algorithms/doms/BaseDomsHandler.py        |  5 +++-
 .../algorithms/doms/ResultsRetrieval.py       | 23 +++++++++++++++++++
 .../webservice/algorithms_spark/Matchup.py    | 23 +++++++++++++++++++
 3 files changed, 50 insertions(+), 1 deletion(-)

diff --git a/analysis/webservice/algorithms/doms/BaseDomsHandler.py b/analysis/webservice/algorithms/doms/BaseDomsHandler.py
index 1a5c22d7..f7ea9042 100644
--- a/analysis/webservice/algorithms/doms/BaseDomsHandler.py
+++ b/analysis/webservice/algorithms/doms/BaseDomsHandler.py
@@ -755,7 +755,10 @@ def round_down_day(dt):
                         primary_histdata[pts]['data'].append(r[caml_params['primary']])
                     secondary_histdata[sts]['data'].append(secondary_match['variable_value'])
 
-                bins = [-5, 0, 5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55]
+                if "histogram_bins" in caml_params:
+                    bins = caml_params['histogram_bins']
+                else:
+                    bins = [-5, 0, 5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55]
 
                 for d in primary_histdata:
                     hist, _ = np.histogram(primary_histdata[d]['data'], bins=bins, density=False)
diff --git a/analysis/webservice/algorithms/doms/ResultsRetrieval.py b/analysis/webservice/algorithms/doms/ResultsRetrieval.py
index b3781060..d5688172 100644
--- a/analysis/webservice/algorithms/doms/ResultsRetrieval.py
+++ b/analysis/webservice/algorithms/doms/ResultsRetrieval.py
@@ -105,6 +105,29 @@ def calc(self, computeOptions, **args):
             caml_params['charts'] = types
             caml_params['format'] = 'Results'
 
+            hist_bins = computeOptions.get_argument("camlHistBins")
+
+            if hist_bins and (types['histogram_primary'] or types['histogram_secondary']):
+                hist_bins = hist_bins.split(',')
+
+                bins = []
+
+                for b in hist_bins:
+                    try:
+                        v = int(b)
+                        if v in bins:
+                            raise NexusProcessingException(reason="duplicate bin in parameter", code=400)
+                        bins.append(v)
+                    except:
+                        raise NexusProcessingException("non numeric argument provided for bins", code=400)
+
+                if len(bins) == 0:
+                    raise NexusProcessingException(reason='No bins given in argument', code=400)
+
+                bins.sort()
+
+                caml_params['histogram_bins'] = bins
+
         try:
             execution_id = uuid.UUID(execution_id)
         except:
diff --git a/analysis/webservice/algorithms_spark/Matchup.py b/analysis/webservice/algorithms_spark/Matchup.py
index 7da8a240..e9547865 100644
--- a/analysis/webservice/algorithms_spark/Matchup.py
+++ b/analysis/webservice/algorithms_spark/Matchup.py
@@ -261,6 +261,29 @@ def parse_arguments(self, request):
             caml_params['charts'] = types
             caml_params['format'] = 'Matchup'
 
+            hist_bins = request.get_argument("camlHistBins")
+
+            if hist_bins and (types['histogram_primary'] or types['histogram_secondary']):
+                hist_bins = hist_bins.split(',')
+
+                bins = []
+
+                for b in hist_bins:
+                    try:
+                        v = int(b)
+                        if v in bins:
+                            raise NexusProcessingException(reason="duplicate bin in parameter", code=400)
+                        bins.append(v)
+                    except:
+                        raise NexusProcessingException("non numeric argument provided for bins", code=400)
+
+                if len(bins) == 0:
+                    raise NexusProcessingException(reason='No bins given in argument', code=400)
+
+                bins.sort()
+
+                caml_params['histogram_bins'] = bins
+
         return bounding_polygon, primary_ds_name, secondary_ds_names, parameter_s, \
                start_time, start_seconds_from_epoch, end_time, end_seconds_from_epoch, \
                depth_min, depth_max, time_tolerance, radius_tolerance, \

From 568c6ffb40800c72fec36e800f4ee1783b6e6d2f Mon Sep 17 00:00:00 2001
From: rileykk <rileykk@jpl.nasa.gov>
Date: Mon, 26 Sep 2022 11:23:05 -0700
Subject: [PATCH 30/38] Removed unused code

---
 analysis/webservice/algorithms/doms/BaseDomsHandler.py | 2 +-
 analysis/webservice/webmodel/NexusRequestObject.py     | 3 ---
 2 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/analysis/webservice/algorithms/doms/BaseDomsHandler.py b/analysis/webservice/algorithms/doms/BaseDomsHandler.py
index f7ea9042..52d6e3ea 100644
--- a/analysis/webservice/algorithms/doms/BaseDomsHandler.py
+++ b/analysis/webservice/algorithms/doms/BaseDomsHandler.py
@@ -30,7 +30,7 @@
 from . import config
 from . import geo
 from webservice.algorithms.NexusCalcHandler import NexusCalcHandler
-from webservice.webmodel import NexusResults, NexusProcessingException
+from webservice.webmodel import NexusResults
 
 EPOCH = timezone('UTC').localize(datetime(1970, 1, 1))
 ISO_8601 = '%Y-%m-%dT%H:%M:%S%z'
diff --git a/analysis/webservice/webmodel/NexusRequestObject.py b/analysis/webservice/webmodel/NexusRequestObject.py
index 7aa41a37..3779cf96 100644
--- a/analysis/webservice/webmodel/NexusRequestObject.py
+++ b/analysis/webservice/webmodel/NexusRequestObject.py
@@ -23,9 +23,6 @@ def __init__(self, reqHandler):
     def get_argument(self, name, default=None):
         return self.requestHandler.get_argument(name, default=default)
 
-    def get_arguments(self, name):
-        return self.requestHandler.get_arguments(name)
-
     def get_list_int_arg(self, name, default=None):
         arg = self.get_argument(name, default=default)
         return arg.split(',')

From 172595748db90befa021b211a425e50017072c0c Mon Sep 17 00:00:00 2001
From: rileykk <rileykk@jpl.nasa.gov>
Date: Wed, 19 Oct 2022 11:53:37 -0700
Subject: [PATCH 31/38] Replaced dynamic c+v keys with lists + separated time
 series chart into two charts

---
 .../algorithms/doms/BaseDomsHandler.py        | 91 ++++++++++---------
 1 file changed, 46 insertions(+), 45 deletions(-)

diff --git a/analysis/webservice/algorithms/doms/BaseDomsHandler.py b/analysis/webservice/algorithms/doms/BaseDomsHandler.py
index 52d6e3ea..bfbd8b1d 100644
--- a/analysis/webservice/algorithms/doms/BaseDomsHandler.py
+++ b/analysis/webservice/algorithms/doms/BaseDomsHandler.py
@@ -538,12 +538,6 @@ class DomsCAMLFormatter:
     def create(executionId, results, params, details, request):
         import copy
 
-        def keyname(name, number):
-            if number == 0:
-                return name
-            else:
-                return f"{name}_{number}"
-
         def empty_if_none(string):
             if string is not None:
                 return string
@@ -580,45 +574,39 @@ def round_down_day(dt):
             "time_end": datetime_to_iso(params['endTime'])
         }
 
-        n_variable = 0
-        n_chart = 0
-
-        VAR = "variable"
-        CHART = "chart"
+        variables = []
+        charts = []
 
         if len(results) > 0:
             try:
-                result[keyname(VAR, n_variable)] = {
+                variables.append({
                     "object": "primary",
                     "name": caml_params['primary'],
                     "units": empty_if_none(get_match_by_variable_name(results[0]['primary'], caml_params['primary'])["variable_unit"])
-                }
+                })
             except KeyError:
-                result[keyname(VAR, n_variable)] = {
+                variables.append({
                     "object": "primary",
                     "name": caml_params['primary'],
                     "units": ""
-                }
-
-            n_variable += 1
+                })
 
             results.sort(key=lambda e: e['time'])
 
             try:
-                result[keyname(VAR, n_variable)] = {
+                variables.append({
                     "object": "secondary",
                     "name": caml_params['secondary'],
                     "units": empty_if_none(get_match_by_variable_name(results[0]['matches'][0]['secondary'], caml_params['secondary'])["variable_unit"])
-                }
+                })
             except KeyError:
-                result[keyname(VAR, n_variable)] = {
+                variables.append({
                     "object": "secondary",
                     "name": caml_params['secondary'],
                     "units": ""
-                }
-
+                })
 
-            n_variable += 1
+            result['variable'] = variables
 
             data = []
 
@@ -655,17 +643,30 @@ def round_down_day(dt):
                 data[0].sort(key=lambda e: (e[0], e[1]))
                 data[1].sort(key=lambda e: (e[0], e[1]))
 
-                result[keyname(CHART, n_chart)] = {
-                    "object": ["primary", "secondary"],
+                charts.append({
+                    "object": ["primary"],
                     "type": "xy_line_point",
                     "title": "Time Series",
                     "xAxis_label": 'Time',
-                    "yAxis_label": f"{result[keyname(VAR, 1)]['name']} ({result[keyname(VAR, 1)]['units']})",
-                    "xySeries_data": copy.deepcopy(data),
-                    "xySeries_labels": [query["primaryName"], query["secondaryName"]]
-                }
+                    "yAxis_label": f"{variables[0]['name']} ({variables[0]['units']})",
+                    "xAxis_type": "datetime",
+                    "yAxis_type": "number",
+                    "xySeries_data": copy.deepcopy(data[0]),
+                    "xySeries_labels": [query["primaryName"]]
+                })
+
+                charts.append({
+                    "object": ["secondary"],
+                    "type": "xy_line_point",
+                    "title": "Time Series",
+                    "xAxis_label": 'Time',
+                    "yAxis_label": f"{variables[1]['name']} ({variables[1]['units']})",
+                    "xAxis_type": "datetime",
+                    "yAxis_type": "number",
+                    "xySeries_data": copy.deepcopy(data[1]),
+                    "xySeries_labels": [query["secondaryName"]]
+                })
 
-                n_chart += 1
                 data.clear()
 
             if caml_params['charts']['scatter']:
@@ -697,16 +698,15 @@ def round_down_day(dt):
 
                 data.sort(key=lambda e: (e[0], e[1]))
 
-                result[keyname(CHART, n_chart)] = {
+                charts.append({
                     "object": ["primary", "secondary"],
                     "type": "xy_scatter_point",
                     "title": "Scatter Plot",
-                    "xAxis_label": f"{result[keyname(VAR, 0)]['name']} ({result[keyname(VAR, 0)]['units']})",
-                    "yAxis_label": f"{result[keyname(VAR, 1)]['name']} ({result[keyname(VAR, 1)]['units']})",
+                    "xAxis_label": f"{variables[0]['name']} ({variables[0]['units']})",
+                    "yAxis_label": f"{variables[1]['name']} ({variables[1]['units']})",
                     "xySeries_data": copy.deepcopy(data),
-                }
+                })
 
-                n_chart += 1
                 data.clear()
 
             if caml_params['charts']['histogram_primary'] or caml_params['charts']['histogram_secondary']:
@@ -791,16 +791,15 @@ def round_down_day(dt):
 
                         data.append([d_hist])
 
-                    result[keyname(CHART, n_chart)] = {
+                    charts.append({
                         "object": ["primary"],
                         "type": "histogram",
                         "title": "Frequency Distribution over Time",
-                        "xAxis_label": f"{result[keyname(VAR, 0)]['name']} ({result[keyname(VAR, 0)]['units']})",
+                        "xAxis_label": f"{variables[0]['name']} ({variables[0]['units']})",
                         "yAxis_label": "frequency (count)",
                         "xySeries_data": copy.deepcopy(data),
-                    }
+                    })
 
-                    n_chart += 1
                     data.clear()
 
                 if caml_params['charts']['histogram_secondary']:
@@ -814,14 +813,14 @@ def round_down_day(dt):
 
                         data.append([d_hist])
 
-                    result[keyname(CHART, n_chart)] = {
+                    charts.append({
                         "object": ["secondary"],
                         "type": "histogram",
                         "title": "Frequency Distribution over Time",
-                        "xAxis_label": f"{result[keyname(VAR, 1)]['name']} ({result[keyname(VAR, 1)]['units']})",
+                        "xAxis_label": f"{variables[1]['name']} ({variables[1]['units']})",
                         "yAxis_label": "frequency (count)",
                         "xySeries_data": copy.deepcopy(data),
-                    }
+                    })
 
                     data.clear()
 
@@ -857,13 +856,15 @@ def round_down_day(dt):
 
                 data.sort(key=lambda e: (e[0], e[1][0], e[1][1], e[2]))
 
-                result['map'] = {
+                charts.append({
                     "object": ["secondary"],
                     "type": "trajectory",
                     "title": "Along track colocation differences",
-                    "colorbar_label": f"{result[keyname(VAR, 1)]['name']} ({result[keyname(VAR, 1)]['units']})",
+                    "colorbar_label": f"{variables[1]['name']} ({variables[1]['units']})",
                     "xySeries_data": copy.deepcopy(data),
-                }
+                })
+
+                result['chart'] = charts
 
         return json.dumps(
             {'executionId': executionId, 'query': query, 'result': result, 'params': params},

From 153672fab4a51c8772f65d83a7fadc6606538633 Mon Sep 17 00:00:00 2001
From: rileykk <rileykk@jpl.nasa.gov>
Date: Wed, 19 Oct 2022 12:14:53 -0700
Subject: [PATCH 32/38] Updated scatter plot chart

---
 analysis/webservice/algorithms/doms/BaseDomsHandler.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/analysis/webservice/algorithms/doms/BaseDomsHandler.py b/analysis/webservice/algorithms/doms/BaseDomsHandler.py
index bfbd8b1d..86763b93 100644
--- a/analysis/webservice/algorithms/doms/BaseDomsHandler.py
+++ b/analysis/webservice/algorithms/doms/BaseDomsHandler.py
@@ -702,8 +702,10 @@ def round_down_day(dt):
                     "object": ["primary", "secondary"],
                     "type": "xy_scatter_point",
                     "title": "Scatter Plot",
-                    "xAxis_label": f"{variables[0]['name']} ({variables[0]['units']})",
-                    "yAxis_label": f"{variables[1]['name']} ({variables[1]['units']})",
+                    "xAxis_label": f"{query['primaryName']} ({variables[0]['units']})",
+                    "yAxis_label": f"{query['secondaryName']} ({variables[1]['units']})",
+                    "xAxis_type": "number",
+                    "yAxis_type": "number",
                     "xySeries_data": copy.deepcopy(data),
                 })
 

From 9e996bbeed32979de6c49055a2258f7899ecdebd Mon Sep 17 00:00:00 2001
From: rileykk <rileykk@jpl.nasa.gov>
Date: Wed, 19 Oct 2022 12:15:32 -0700
Subject: [PATCH 33/38] Added histogram timeseries arguments

---
 analysis/webservice/algorithms/doms/ResultsRetrieval.py | 9 ++++++++-
 analysis/webservice/algorithms_spark/Matchup.py         | 9 ++++++++-
 2 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/analysis/webservice/algorithms/doms/ResultsRetrieval.py b/analysis/webservice/algorithms/doms/ResultsRetrieval.py
index d5688172..da3ea345 100644
--- a/analysis/webservice/algorithms/doms/ResultsRetrieval.py
+++ b/analysis/webservice/algorithms/doms/ResultsRetrieval.py
@@ -67,6 +67,8 @@ def calc(self, computeOptions, **args):
                 'scatter',
                 'histogram_primary',
                 'histogram_secondary',
+                'histogram_primary_timeseries',
+                'histogram_secondary_timeseries',
                 'trajectory'
             ]
 
@@ -78,6 +80,8 @@ def calc(self, computeOptions, **args):
                     'scatter': True,
                     'histogram_primary': True,
                     'histogram_secondary': True,
+                    'histogram_primary_timeseries': True,
+                    'histogram_secondary_timeseries': True,
                     'trajectory': True
                 }
             else:
@@ -88,6 +92,8 @@ def calc(self, computeOptions, **args):
                     'scatter': False,
                     'histogram_primary': False,
                     'histogram_secondary': False,
+                    'histogram_primary_timeseries': False,
+                    'histogram_secondary_timeseries': False,
                     'trajectory': False
                 }
 
@@ -107,7 +113,8 @@ def calc(self, computeOptions, **args):
 
             hist_bins = computeOptions.get_argument("camlHistBins")
 
-            if hist_bins and (types['histogram_primary'] or types['histogram_secondary']):
+            if hist_bins and (types['histogram_primary'] or types['histogram_secondary'] or
+                              types['histogram_primary_timeseries'] or types['histogram_secondary_timeseries']):
                 hist_bins = hist_bins.split(',')
 
                 bins = []
diff --git a/analysis/webservice/algorithms_spark/Matchup.py b/analysis/webservice/algorithms_spark/Matchup.py
index 66e4608e..dd9529a8 100644
--- a/analysis/webservice/algorithms_spark/Matchup.py
+++ b/analysis/webservice/algorithms_spark/Matchup.py
@@ -237,6 +237,8 @@ def parse_arguments(self, request):
                 'scatter',
                 'histogram_primary',
                 'histogram_secondary',
+                'histogram_primary_timeseries',
+                'histogram_secondary_timeseries',
                 'trajectory'
             ]
 
@@ -248,6 +250,8 @@ def parse_arguments(self, request):
                     'scatter': True,
                     'histogram_primary': True,
                     'histogram_secondary': True,
+                    'histogram_primary_timeseries': True,
+                    'histogram_secondary_timeseries': True,
                     'trajectory': True
                 }
             else:
@@ -258,6 +262,8 @@ def parse_arguments(self, request):
                     'scatter': False,
                     'histogram_primary': False,
                     'histogram_secondary': False,
+                    'histogram_primary_timeseries': False,
+                    'histogram_secondary_timeseries': False,
                     'trajectory': False
                 }
 
@@ -277,7 +283,8 @@ def parse_arguments(self, request):
 
             hist_bins = request.get_argument("camlHistBins")
 
-            if hist_bins and (types['histogram_primary'] or types['histogram_secondary']):
+            if hist_bins and (types['histogram_primary'] or types['histogram_secondary'] or
+                              types['histogram_primary_timeseries'] or types['histogram_secondary_timeseries']):
                 hist_bins = hist_bins.split(',')
 
                 bins = []

From 35cf2ad144ac2ccb22258dcf95582a8b6d7d5b1b Mon Sep 17 00:00:00 2001
From: rileykk <rileykk@jpl.nasa.gov>
Date: Wed, 19 Oct 2022 12:36:14 -0700
Subject: [PATCH 34/38] Added histogram timeseries charts

---
 .../algorithms/doms/BaseDomsHandler.py        | 112 ++++++++++++++++++
 1 file changed, 112 insertions(+)

diff --git a/analysis/webservice/algorithms/doms/BaseDomsHandler.py b/analysis/webservice/algorithms/doms/BaseDomsHandler.py
index 86763b93..9061066d 100644
--- a/analysis/webservice/algorithms/doms/BaseDomsHandler.py
+++ b/analysis/webservice/algorithms/doms/BaseDomsHandler.py
@@ -826,6 +826,118 @@ def round_down_day(dt):
 
                     data.clear()
 
+            if caml_params['charts']['histogram_primary_timeseries'] or \
+                    caml_params['charts']['histogram_secondary_timeseries']:
+                primary_histdata = {}
+                secondary_histdata = {}
+
+                for r in results:
+                    secondary = None
+                    secondary_match = None
+
+                    for s in r['matches']:
+                        try:
+                            if caml_params['format'] == 'Matchup':
+                                secondary_match = get_match_by_variable_name(s['secondary'], caml_params['secondary'])
+                                secondary = s
+                                break
+                            else:
+                                if caml_params['secondary'] in s:
+                                    secondary = s
+                                    secondary_match = {'variable_value': s[caml_params['secondary']]}
+                                    break
+                        except:
+                            pass
+
+                    if secondary is None:
+                        continue
+
+                    pts = datetime_to_iso(round_down_day(r['time']))
+                    sts = datetime_to_iso(round_down_day(secondary['time']))
+
+                    if pts not in primary_histdata:
+                        primary_histdata[pts] = {
+                            'data': [],
+                            'hist': None
+                        }
+
+                    if sts not in secondary_histdata:
+                        secondary_histdata[sts] = {
+                            'data': [],
+                            'hist': None
+                        }
+
+                    if caml_params['format'] == 'Matchup':
+                        primary_histdata[pts]['data'].append(get_match_by_variable_name(r['primary'], caml_params['primary'])['variable_value'])
+                    else:
+                        primary_histdata[pts]['data'].append(r[caml_params['primary']])
+                    secondary_histdata[sts]['data'].append(secondary_match['variable_value'])
+
+                if "histogram_bins" in caml_params:
+                    bins = caml_params['histogram_bins']
+                else:
+                    bins = [-5, 0, 5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55]
+
+                for d in primary_histdata:
+                    hist, _ = np.histogram(primary_histdata[d]['data'], bins=bins, density=False)
+                    h = []
+
+                    for i in range(len(hist)):
+                        h.append([bins[i], int(hist[i])])
+
+                    primary_histdata[d]['hist'] = copy.deepcopy(h)
+
+                for d in secondary_histdata:
+                    hist, _ = np.histogram(secondary_histdata[d]['data'], bins=bins, density=False)
+                    h = []
+
+                    for i in range(len(hist)):
+                        h.append([bins[i], int(hist[i])])
+
+                    secondary_histdata[d]['hist'] = copy.deepcopy(h)
+
+                if caml_params['charts']['histogram_primary_timeseries']:
+                    for d in primary_histdata:
+                        d_hist = [d]
+
+                        for bin in primary_histdata[d]['hist']:
+                            d_hist.append(bin)
+
+                        data.append([d_hist])
+
+                    charts.append({
+                        "object": ["primary"],
+                        "type": "histogram_timeseries",
+                        "title": "Frequency Distribution over Time",
+                        "xAxis_label": f"{variables[0]['name']} ({variables[0]['units']})",
+                        "yAxis_label": "frequency (count)",
+                        "xySeries_data": copy.deepcopy(data),
+                        "xySeries_labels": [query['primaryName']]
+                    })
+
+                    data.clear()
+
+                if caml_params['charts']['histogram_secondary_timeseries']:
+                    for d in secondary_histdata:
+                        d_hist = [d]
+
+                        for bin in secondary_histdata[d]['hist']:
+                            d_hist.append(bin)
+
+                        data.append([d_hist])
+
+                    charts.append({
+                        "object": ["secondary"],
+                        "type": "histogram_timeseries",
+                        "title": "Frequency Distribution over Time",
+                        "xAxis_label": f"{variables[1]['name']} ({variables[1]['units']})",
+                        "yAxis_label": "frequency (count)",
+                        "xySeries_data": copy.deepcopy(data),
+                        "xySeries_labels": [query['secondaryName']]
+                    })
+
+                    data.clear()
+
             if caml_params['charts']['trajectory']:
                 for r in results:
                     secondary = None

From f57fd95a186c2098c65cf763e3bea92f05cef17f Mon Sep 17 00:00:00 2001
From: rileykk <rileykk@jpl.nasa.gov>
Date: Wed, 19 Oct 2022 13:03:09 -0700
Subject: [PATCH 35/38] Misc

---
 .../algorithms/doms/BaseDomsHandler.py        | 35 +++++++++----------
 1 file changed, 17 insertions(+), 18 deletions(-)

diff --git a/analysis/webservice/algorithms/doms/BaseDomsHandler.py b/analysis/webservice/algorithms/doms/BaseDomsHandler.py
index 9061066d..75d7f680 100644
--- a/analysis/webservice/algorithms/doms/BaseDomsHandler.py
+++ b/analysis/webservice/algorithms/doms/BaseDomsHandler.py
@@ -707,6 +707,7 @@ def round_down_day(dt):
                     "xAxis_type": "number",
                     "yAxis_type": "number",
                     "xySeries_data": copy.deepcopy(data),
+                    "xySeries_labels": f"{variables[0]['name']} x {variables[1]['name']}"
                 })
 
                 data.clear()
@@ -736,8 +737,8 @@ def round_down_day(dt):
                     if secondary is None:
                         continue
 
-                    pts = datetime_to_iso(round_down_day(r['time']))
-                    sts = datetime_to_iso(round_down_day(secondary['time']))
+                    pts = 'fixed'
+                    sts = 'fixed'
 
                     if pts not in primary_histdata:
                         primary_histdata[pts] = {
@@ -783,45 +784,39 @@ def round_down_day(dt):
                 head = primary_histdata[next(iter(primary_histdata))]['hist']
 
                 if caml_params['charts']['histogram_primary']:
-                    data.append(head)
-
                     for d in primary_histdata:
-                        d_hist = [d]
-
                         for bin in primary_histdata[d]['hist']:
-                            d_hist.append(bin)
-
-                        data.append([d_hist])
+                            data.append(bin)
 
                     charts.append({
                         "object": ["primary"],
                         "type": "histogram",
-                        "title": "Frequency Distribution over Time",
+                        "title": "Frequency Distribution",
                         "xAxis_label": f"{variables[0]['name']} ({variables[0]['units']})",
                         "yAxis_label": "frequency (count)",
+                        "xAxis_type": "number",
+                        "yAxis_type": "number",
                         "xySeries_data": copy.deepcopy(data),
+                        "xySeries_labels": [query['primaryName']]
                     })
 
                     data.clear()
 
                 if caml_params['charts']['histogram_secondary']:
-                    data.append(head)
-
                     for d in secondary_histdata:
-                        d_hist = [d]
-
                         for bin in secondary_histdata[d]['hist']:
-                            d_hist.append(bin)
-
-                        data.append([d_hist])
+                            data.append(bin)
 
                     charts.append({
                         "object": ["secondary"],
                         "type": "histogram",
-                        "title": "Frequency Distribution over Time",
+                        "title": "Frequency Distribution",
                         "xAxis_label": f"{variables[1]['name']} ({variables[1]['units']})",
                         "yAxis_label": "frequency (count)",
+                        "xAxis_type": "number",
+                        "yAxis_type": "number",
                         "xySeries_data": copy.deepcopy(data),
+                        "xySeries_labels": [query['secondaryName']]
                     })
 
                     data.clear()
@@ -911,6 +906,8 @@ def round_down_day(dt):
                         "title": "Frequency Distribution over Time",
                         "xAxis_label": f"{variables[0]['name']} ({variables[0]['units']})",
                         "yAxis_label": "frequency (count)",
+                        "xAxis_type": "number",
+                        "yAxis_type": "number",
                         "xySeries_data": copy.deepcopy(data),
                         "xySeries_labels": [query['primaryName']]
                     })
@@ -932,6 +929,8 @@ def round_down_day(dt):
                         "title": "Frequency Distribution over Time",
                         "xAxis_label": f"{variables[1]['name']} ({variables[1]['units']})",
                         "yAxis_label": "frequency (count)",
+                        "xAxis_type": "number",
+                        "yAxis_type": "number",
                         "xySeries_data": copy.deepcopy(data),
                         "xySeries_labels": [query['secondaryName']]
                     })

From c0f8efc83a1d263feba5483c04dddb69799a8f20 Mon Sep 17 00:00:00 2001
From: rileykk <rileykk@jpl.nasa.gov>
Date: Wed, 19 Oct 2022 13:27:40 -0700
Subject: [PATCH 36/38] Removed unused variable

---
 analysis/webservice/algorithms/doms/BaseDomsHandler.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/analysis/webservice/algorithms/doms/BaseDomsHandler.py b/analysis/webservice/algorithms/doms/BaseDomsHandler.py
index 75d7f680..aa388ba5 100644
--- a/analysis/webservice/algorithms/doms/BaseDomsHandler.py
+++ b/analysis/webservice/algorithms/doms/BaseDomsHandler.py
@@ -781,8 +781,6 @@ def round_down_day(dt):
 
                     secondary_histdata[d]['hist'] = copy.deepcopy(h)
 
-                head = primary_histdata[next(iter(primary_histdata))]['hist']
-
                 if caml_params['charts']['histogram_primary']:
                     for d in primary_histdata:
                         for bin in primary_histdata[d]['hist']:

From 374c7e9314b47ec3c0740c06a5b4ae8bd6764702 Mon Sep 17 00:00:00 2001
From: rileykk <rileykk@jpl.nasa.gov>
Date: Wed, 19 Oct 2022 13:30:24 -0700
Subject: [PATCH 37/38] Lazy load schema for domsresults to prevent issues

---
 .../algorithms/doms/ResultsRetrieval.py          | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/analysis/webservice/algorithms/doms/ResultsRetrieval.py b/analysis/webservice/algorithms/doms/ResultsRetrieval.py
index da3ea345..ad7fdf6b 100644
--- a/analysis/webservice/algorithms/doms/ResultsRetrieval.py
+++ b/analysis/webservice/algorithms/doms/ResultsRetrieval.py
@@ -22,7 +22,19 @@
 from webservice.algorithms.doms.insitu import query_insitu_schema
 
 
-insitu_schema = query_insitu_schema()
+class Schema:
+    def __init__(self):
+        self.schema = None
+
+    def get(self):
+        if self.schema is None:
+            self.schema = query_insitu_schema()
+
+        return self.schema
+
+
+insitu_schema = Schema()
+
 
 @nexus_handler
 class DomsResultsRetrievalHandler(BaseDomsHandler.BaseDomsQueryCalcHandler):
@@ -56,7 +68,7 @@ def calc(self, computeOptions, **args):
                 raise NexusProcessingException(
                     reason="Secondary dataset argument is required when outputting in CAML format", code=400)
 
-            insitu_params = get_insitu_params(insitu_schema)
+            insitu_params = get_insitu_params(insitu_schema.get())
 
             if secondary not in insitu_params:
                 raise NexusProcessingException(

From c075a384db64c085a0f28b29cfefd892f91563a0 Mon Sep 17 00:00:00 2001
From: rileykk <rileykk@jpl.nasa.gov>
Date: Wed, 14 Dec 2022 12:13:29 -0800
Subject: [PATCH 38/38] Moved changelog entry

---
 CHANGELOG.md | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index a5d153cb..e5ab69a2 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,6 +4,15 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [Unreleased]
+### Added
+- SDAP-401: Added JSON output formatting for use with COVERAGE to `/match_spark`
+### Changed
+### Deprecated
+### Removed
+### Fixed
+### Security
+
 ## [1.0.0] - 2022-11-22
 ### Added
 - SDAP-388: Enable SDAP to proxy/redirect to alternate SDAP
@@ -25,7 +34,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - SDAP-407: Added depth to `/domsresults` endpoint
 - Added documentation for building SDAP docker images
   - Prepared documentation for v1.0.0 release.
-- SDAP-401: Added JSON output formatting for use with COVERAGE to `/match_spark`
 ### Changed
 - SDAP-390: Changed `/doms` to `/cdms` and `doms_reader.py` to `cdms_reader.py`
 - domslist endpoint points to AWS insitu instead of doms insitu