From 14663a75b8ed7d3b09952447d0c4dff9fa7ebe04 Mon Sep 17 00:00:00 2001 From: Antognetti Date: Mon, 20 May 2024 11:28:45 -0500 Subject: [PATCH 01/12] Draft for beginning to implement xarray as containers for data from MDSPlus --- src/wipplpy/modules/generic_get_data.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/wipplpy/modules/generic_get_data.py b/src/wipplpy/modules/generic_get_data.py index 1eb1f49..eec0eda 100644 --- a/src/wipplpy/modules/generic_get_data.py +++ b/src/wipplpy/modules/generic_get_data.py @@ -521,6 +521,8 @@ def get( # noqa: PLR0912, PLR0915 ) try_loading = True self.tree = get_remote_shot_tree(self.shot_number, reconnect=True) + + #Going to be changing to put into xarray arrays if self.ignore_errors: try: @@ -535,7 +537,7 @@ def get( # noqa: PLR0912, PLR0915 data = data.astype(np_data_type) logging.debug(f"Changed data to type '{np_data_type}'.") - # Only save calls if the dictionary exists. The dictionary doesn't exist during some stages of initialization so that we don't save incorrect data. + # Only save calls if the dictionary exists. Th dictionary doesn't exist during some stages of initialization so that we don't save incorrect data. if hasattr(self, "saved_calls"): # logging.debug("Adding data from get call with `save_name='{}'` into `saved_calls`.".format(save_name)) if save_name in self.saved_calls: @@ -543,7 +545,7 @@ def get( # noqa: PLR0912, PLR0915 "Save name ({}) is the same as a save name already in the data to save dictionary. Overwriting old data." ) self.saved_calls[save_name] = data - + # return data def to_raw_index(self, time_index): From 859259d08f6a5953c8b3a11febe8b05649c80f7e Mon Sep 17 00:00:00 2001 From: Antognetti Date: Mon, 20 May 2024 12:01:03 -0500 Subject: [PATCH 02/12] Added xarray to environment dependencies --- mamba_environment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/mamba_environment.yml b/mamba_environment.yml index 62df5eb..b596316 100644 --- a/mamba_environment.yml +++ b/mamba_environment.yml @@ -14,3 +14,4 @@ dependencies: - configobj - pyyaml - sphinx + - xarray \ No newline at end of file From b1f7b5bf6b5381727517d19ba5999a0392e8449a Mon Sep 17 00:00:00 2001 From: Antognetti Date: Thu, 23 May 2024 13:05:37 -0500 Subject: [PATCH 03/12] Introduced gather_DataArray and package_DataSet function --- src/wipplpy/modules/generic_get_data.py | 34 ++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/src/wipplpy/modules/generic_get_data.py b/src/wipplpy/modules/generic_get_data.py index eec0eda..140e4f7 100644 --- a/src/wipplpy/modules/generic_get_data.py +++ b/src/wipplpy/modules/generic_get_data.py @@ -2,6 +2,7 @@ import re import numpy as np +import xarray as xr from MDSplus.connection import Connection, MdsIpException from MDSplus.mdsExceptions import MDSplusException, SsSUCCESS from wipplpy.modules.shot_loader import get_remote_shot_tree @@ -429,7 +430,7 @@ def _get_many(self, tree, variable_booleans, get_calls): return variable_vals def get( # noqa: PLR0912, PLR0915 - self, get_call, np_data_type=np.float64, change_data=True, load_from_saved=True + self, get_call, np_data_type=np.float64, change_data=True, load_from_saved=True, ): """ Get data from the mdsplus tree and change it to the correct type using a get call. @@ -521,8 +522,6 @@ def get( # noqa: PLR0912, PLR0915 ) try_loading = True self.tree = get_remote_shot_tree(self.shot_number, reconnect=True) - - #Going to be changing to put into xarray arrays if self.ignore_errors: try: @@ -545,8 +544,35 @@ def get( # noqa: PLR0912, PLR0915 "Save name ({}) is the same as a save name already in the data to save dictionary. Overwriting old data." ) self.saved_calls[save_name] = data + + self.to_xarray(data, xarray_args=xarray_args, xarray_kwargs=xarray_kwargs) + + def gather_to_xarray(self, path): + """ + Package data along with dimensions and attributes into an xarray DataArray. Pulls data from MDSPlus or local files. + """ + #get data first as numpy array + #get dimensions of array + # spatial () + # time (add sampling rate as attribute, might be unnecessary) + #get coordinates for dimensions that need it + #units on dimensions (is this stored as an attribute of the array or of the coord) + #fill in attributes (e.g. machine, maybe port info) + # + #data = get(path) # - return data + #data_array = xr.DataArray(data) + #any other additions to the DataArray + #self.data = data_array + pass + + def package_as_DataSet(self, path_list): + ''' + Gather and combine multiple xarray DataArrays into a single DataSet. May + have multiple calls to gather_to_xarray, or maybe can also pass in DataArrays + explicitly which have already been put together. + ''' + pass def to_raw_index(self, time_index): """ From 3719363508946d966cd8cabc02064d4a52ca5976 Mon Sep 17 00:00:00 2001 From: Antognetti Date: Thu, 13 Jun 2024 15:30:53 -0500 Subject: [PATCH 04/12] Building coordinate dictionary --- src/wipplpy/modules/generic_get_data.py | 39 +++++++++++++++++-------- 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/src/wipplpy/modules/generic_get_data.py b/src/wipplpy/modules/generic_get_data.py index 140e4f7..3845663 100644 --- a/src/wipplpy/modules/generic_get_data.py +++ b/src/wipplpy/modules/generic_get_data.py @@ -545,31 +545,46 @@ def get( # noqa: PLR0912, PLR0915 ) self.saved_calls[save_name] = data - self.to_xarray(data, xarray_args=xarray_args, xarray_kwargs=xarray_kwargs) + return data - def gather_to_xarray(self, path): + def gather_DataArray(self, node, dim_names): """ Package data along with dimensions and attributes into an xarray DataArray. Pulls data from MDSPlus or local files. + + Parameters + ---------- + node : str + MDSPlus node for the wanted data. + + Returns + ------- + array : xr.DataArray + DataArray with stored data and metadata + """ #get data first as numpy array - #get dimensions of array - # spatial () - # time (add sampling rate as attribute, might be unnecessary) + data = node.data() + ndim = data.ndim + if ndim != len(dim_names): raise Exception("Number of dimension names and dimenion of data do not match") + + coords = {} + #dim_units = {} + for dim in range(ndim): + coords[dim_names[dim]] = data.dim_of(dim).data() + #get coordinates for dimensions that need it #units on dimensions (is this stored as an attribute of the array or of the coord) - #fill in attributes (e.g. machine, maybe port info) - # - #data = get(path) - # + #fill in attributes + #maybe summary info like average Ip, ne, Bt, q (what would be some equivalent things for BRB? ) but what would be the resource cost for this? + #maybe main contact person + their email for the diagnostic #data_array = xr.DataArray(data) - #any other additions to the DataArray #self.data = data_array pass - def package_as_DataSet(self, path_list): + def package_DataSet(self, path_list): ''' Gather and combine multiple xarray DataArrays into a single DataSet. May - have multiple calls to gather_to_xarray, or maybe can also pass in DataArrays + have multiple calls to gather_to_DataArray. Could also be passed DataArrays explicitly which have already been put together. ''' pass From 6f9f40c1d59e55de9e1df9d683919e95f67f4929 Mon Sep 17 00:00:00 2001 From: Antognetti Date: Thu, 13 Jun 2024 16:06:28 -0500 Subject: [PATCH 05/12] First xarray DataArray --- src/wipplpy/modules/generic_get_data.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/wipplpy/modules/generic_get_data.py b/src/wipplpy/modules/generic_get_data.py index 3845663..bff1ed2 100644 --- a/src/wipplpy/modules/generic_get_data.py +++ b/src/wipplpy/modules/generic_get_data.py @@ -577,8 +577,9 @@ def gather_DataArray(self, node, dim_names): #fill in attributes #maybe summary info like average Ip, ne, Bt, q (what would be some equivalent things for BRB? ) but what would be the resource cost for this? #maybe main contact person + their email for the diagnostic - #data_array = xr.DataArray(data) - #self.data = data_array + + data_array = xr.DataArray(data, dims = dim_names, coords = coords) + self.data = data_array pass def package_DataSet(self, path_list): From 9b4fad8c43dafba1f9b24e82e31b5915a8218b59 Mon Sep 17 00:00:00 2001 From: Antognetti Date: Fri, 6 Sep 2024 09:48:34 -0500 Subject: [PATCH 06/12] renamed gather_DataArray to gather_data_array --- docs/source/contributing.rst | 4 +- docs/source/installation.rst | 63 +++++++++++------------ src/wipplpy/modules/generic_get_data.py | 67 +++++++++++++------------ 3 files changed, 70 insertions(+), 64 deletions(-) diff --git a/docs/source/contributing.rst b/docs/source/contributing.rst index d945d53..94d999a 100644 --- a/docs/source/contributing.rst +++ b/docs/source/contributing.rst @@ -4,6 +4,6 @@ Contributing ************ -We follow the same design as PlasmaPy for contributing. Please see the `PlasmaPy -contributing workflow guide `_ +We follow the same design as PlasmaPy for contributing. Please see the `PlasmaPy +contributing workflow guide `_ for more information. diff --git a/docs/source/installation.rst b/docs/source/installation.rst index 4867e7d..9ca2b50 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -7,7 +7,7 @@ Installation Installing WiPPLPy as a Contributor =================================== -This installation is very similar to how PlasmaPy installation is done for +This installation is very similar to how PlasmaPy installation is done for contributors: https://docs.plasmapy.org/en/stable/contributing/getting_ready.html. We've done some copy-pasting as to which steps should be followed. @@ -19,8 +19,8 @@ Pre-requisites Opening a terminal ^^^^^^^^^^^^^^^^^^ -The commands in this page are intended to be run in a ``Unix`` terminal. If you -are new to Unix, check out this `Unix tutorial`_ and these `frequently used +The commands in this page are intended to be run in a ``Unix`` terminal. If you +are new to Unix, check out this `Unix tutorial`_ and these `frequently used Unix commands`_. Windows @@ -28,19 +28,19 @@ Windows There are several options for terminals on Windows. -* Powershell_ comes pre-installed with Windows. These instructions cover - `opening Powershell`_. We recommend Powershell for a quick start, if Windows +* Powershell_ comes pre-installed with Windows. These instructions cover + `opening Powershell`_. We recommend Powershell for a quick start, if Windows is the only operating system you use, or if you have not used Unix before. -* We recommend `Windows Subsystem for Linux`_ (WSL) if you are familiar with - Unix, you use macOS or Linux too, or you expect to contribute to WiPPLPy - extensively. These instructions cover `installing WSL`_. If you choose WSL, +* We recommend `Windows Subsystem for Linux`_ (WSL) if you are familiar with + Unix, you use macOS or Linux too, or you expect to contribute to WiPPLPy + extensively. These instructions cover `installing WSL`_. If you choose WSL, follow the tabs for :guilabel:`Linux/WSL` below. macOS """"" -In the :guilabel:`Finder`, go to :guilabel:`Applications`. Enter the +In the :guilabel:`Finder`, go to :guilabel:`Applications`. Enter the :guilabel:`Utilities` folder and double click on :guilabel:`Terminal`. Linux/WSL @@ -53,12 +53,12 @@ Open a terminal by using :kbd:`Ctrl + Alt + T`. Installing Python ^^^^^^^^^^^^^^^^^ -We suggest using Mamba_ to install Python_. Mamba_ is a versatile package and -environment management system which is widely used in the data science and -scientific Python communities that is similar to Conda_. It is a command line -interface for installing packages. +We suggest using Mamba_ to install Python_. Mamba_ is a versatile package and +environment management system which is widely used in the data science and +scientific Python communities that is similar to Conda_. It is a command line +interface for installing packages. -* If you already have Conda_ installed, then follow the `installing Mamba from +* If you already have Conda_ installed, then follow the `installing Mamba from an existing Conda install`_. * Otherwise, follow the `installing Mamba`_ instructions. @@ -109,7 +109,7 @@ contributing code to WiPPLPy, please take the following steps: Installing Your Clone of WiPPLPy -------------------------------- -Once Mamba_ is installed and you have cloned the git repository then we need to +Once Mamba_ is installed and you have cloned the git repository then we need to create an environment for running the code in. 1. :ref:`Open a terminal `. @@ -126,8 +126,8 @@ create an environment for running the code in. In Windows, the directory path will be :file:`C:\\Users\\\\repos\\WiPPLPy`. -3. Create a Mamba_ environment using the packages from - :file:`mamba_environment.yml`. If you are on a computer that does not use +3. Create a Mamba_ environment using the packages from + :file:`mamba_environment.yml`. If you are on a computer that does not use the Mac silicon processor you can do so by running .. code-block:: bash @@ -149,30 +149,32 @@ create an environment for running the code in. mamba activate WiPPLPy python -5. We need to add the :file:`WiPPLPy/source/` directory to the Python path. We - can find the path to :file:`WiPPLPy/source/` and the :file:`site-packages` +5. We need to add the :file:`WiPPLPy/source/` directory to the Python path. We + can find the path to :file:`WiPPLPy/source/` and the :file:`site-packages` directory by running the following Python code: .. code-block:: python from distutils.sysconfig import get_python_lib - print('site-packaged directory:', get_python_lib()) + + print("site-packaged directory:", get_python_lib()) import os - print('WiPPLPy source code directory:', os.getcwd()) -6. Exit out of python and navigate to the :file:`site-packages` directory that + print("WiPPLPy source code directory:", os.getcwd()) + +6. Exit out of python and navigate to the :file:`site-packages` directory that was printed. It should end in :file:`site-packages`. -7. Create a file called :file:`wipplpy.pth`. This file should contain the path - to the :file:`WiPPLPy/src/` directory. For example, if the path to the - :file:`WiPPLPy/src/` directory is :file:`/Users/username/repos/WiPPLPy/src/` +7. Create a file called :file:`wipplpy.pth`. This file should contain the path + to the :file:`WiPPLPy/src/` directory. For example, if the path to the + :file:`WiPPLPy/src/` directory is :file:`/Users/username/repos/WiPPLPy/src/` then the :file:`wipplpy.pth` file should contain the following: .. code-block:: bash /Users/username/repos/WiPPLPy/src/ -8. Test that the installation was successful by running the following Python +8. Test that the installation was successful by running the following Python code: .. code-block:: python @@ -185,16 +187,16 @@ create an environment for running the code in. Creating Documentation ====================== -To create the documentation, you will need to have `Sphinx`_ installed and +To create the documentation, you will need to have `Sphinx`_ installed and `make`_. This should have already been installed when you created your Mamba environment. -Then, you can create the documentation by entering the ``docs/`` directory and +Then, you can create the documentation by entering the ``docs/`` directory and running the following command:: make html -This will create the documentation in the :file:`docs/build/html/` directory. -Open the documentation by double clicking on the :file:`index.html` file in the +This will create the documentation in the :file:`docs/build/html/` directory. +Open the documentation by double clicking on the :file:`index.html` file in the :file:`docs/build/html/` directory. .. _Sphinx: https://www.sphinx-doc.org/en/master/usage/installation.html @@ -233,4 +235,3 @@ Open the documentation by double clicking on the :file:`index.html` file in the .. _WSL: https://learn.microsoft.com/en-us/windows/wsl .. _git: https://git-scm.com .. _GitHub: https://github.com - diff --git a/src/wipplpy/modules/generic_get_data.py b/src/wipplpy/modules/generic_get_data.py index bff1ed2..126f9a7 100644 --- a/src/wipplpy/modules/generic_get_data.py +++ b/src/wipplpy/modules/generic_get_data.py @@ -5,9 +5,10 @@ import xarray as xr from MDSplus.connection import Connection, MdsIpException from MDSplus.mdsExceptions import MDSplusException, SsSUCCESS -from wipplpy.modules.shot_loader import get_remote_shot_tree from scipy.io import loadmat, savemat +from wipplpy.modules.shot_loader import get_remote_shot_tree + # This lazy get property is taken from https://towardsdatascience.com/what-is-lazy-evaluation-in-python-9efb1d3bfed0 def lazy_get(function): @@ -430,7 +431,11 @@ def _get_many(self, tree, variable_booleans, get_calls): return variable_vals def get( # noqa: PLR0912, PLR0915 - self, get_call, np_data_type=np.float64, change_data=True, load_from_saved=True, + self, + get_call, + np_data_type=np.float64, + change_data=True, + load_from_saved=True, ): """ Get data from the mdsplus tree and change it to the correct type using a get call. @@ -546,49 +551,49 @@ def get( # noqa: PLR0912, PLR0915 self.saved_calls[save_name] = data return data - - def gather_DataArray(self, node, dim_names): + + def gather_data_array(self, node, dimension_names, dimension_units): """ Package data along with dimensions and attributes into an xarray DataArray. Pulls data from MDSPlus or local files. - + Parameters ---------- node : str - MDSPlus node for the wanted data. + MDSPlus node for the chosen data. + dimension_names : list[str] + list of human readable dimension names. Should be consistent with WiPPLPy conventions --insert a link here to some info about common WiPPLPy dimensions and conventions + dimension_units : list[str] + List of dimension units. Should be in a format consistent with astropy.units https://docs.astropy.org/en/stable/units/ Returns ------- array : xr.DataArray DataArray with stored data and metadata - + """ - #get data first as numpy array + # get data first as numpy array data = node.data() - ndim = data.ndim - if ndim != len(dim_names): raise Exception("Number of dimension names and dimenion of data do not match") + num_dimensions = data.ndim + if num_dimensions != len(dimension_names): + raise Exception( + "Number of dimension names and number of data dimensions do not match" + ) coords = {} - #dim_units = {} - for dim in range(ndim): - coords[dim_names[dim]] = data.dim_of(dim).data() - - #get coordinates for dimensions that need it - #units on dimensions (is this stored as an attribute of the array or of the coord) - #fill in attributes - #maybe summary info like average Ip, ne, Bt, q (what would be some equivalent things for BRB? ) but what would be the resource cost for this? - #maybe main contact person + their email for the diagnostic - - data_array = xr.DataArray(data, dims = dim_names, coords = coords) - self.data = data_array - pass - - def package_DataSet(self, path_list): - ''' - Gather and combine multiple xarray DataArrays into a single DataSet. May - have multiple calls to gather_to_DataArray. Could also be passed DataArrays - explicitly which have already been put together. - ''' - pass + dimension_units = {} + + for dim_num, dim_name in dimension_names: + coords[dim_name] = data.dim_of(dim_num).data() + dimension_units[dim_name] = dimension_units + + # get coordinates arrays for dimensions that need it + + # get the rest of the attributes together into + {"units": dimension_units} + + data_array = xr.DataArray(data, dims=dimension_names, coords=coords) + + return data_array def to_raw_index(self, time_index): """ From 25df186b54efca935c56a4242db3cd3fb75cafab Mon Sep 17 00:00:00 2001 From: Antognetti Date: Thu, 7 Nov 2024 18:48:05 -0600 Subject: [PATCH 07/12] Changed get to call gather_data_array directly --- src/wipplpy/modules/generic_get_data.py | 43 +++++++++++++++---------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/src/wipplpy/modules/generic_get_data.py b/src/wipplpy/modules/generic_get_data.py index bdb665b..152ddda 100644 --- a/src/wipplpy/modules/generic_get_data.py +++ b/src/wipplpy/modules/generic_get_data.py @@ -406,11 +406,7 @@ def _get_many(self, tree, variable_booleans, get_calls): return variable_vals def get( # noqa: PLR0912, PLR0915 - self, - get_call, - np_data_type=np.float64, - change_data=True, - load_from_saved=True, + self, get_call, np_data_type=np.float64, change_data=True, load_from_saved=True ): """ Get data from the mdsplus tree and change it to the correct type using a get call. @@ -425,6 +421,8 @@ def get( # noqa: PLR0912, PLR0915 Whether to change the data type of what MDSplus returns. load_from_saved : bool, default=True Whether to try to load the data from the saved calls. + return_node : bool, default = False + Whether to return the data node instead of returning the data itself. Used for gathering xarray. Returns ------- @@ -493,20 +491,21 @@ def get( # noqa: PLR0912, PLR0915 try_loading = True self.tree = get_remote_shot_tree(self.shot_number, reconnect=True) + # Might be good to put the rest of these checks into a separate function, so it can be called here and in the gather_data_array if self.ignore_errors: try: - data = node.data() + data = self.gather_data_array(node) except Exception: return np.array() else: - data = node.data() + data = self.gather_data_array(node) logging.debug("Got data from tree.") if change_data: data = data.astype(np_data_type) logging.debug(f"Changed data to type '{np_data_type}'.") - # Only save calls if the dictionary exists. Th dictionary doesn't exist during some stages of initialization so that we don't save incorrect data. + # Only save calls if the dictionary exists. The dictionary doesn't exist during some stages of initialization so that we don't save incorrect data. if hasattr(self, "saved_calls"): # logging.debug("Adding data from get call with `save_name='{}'` into `saved_calls`.".format(save_name)) if save_name in self.saved_calls: @@ -517,14 +516,14 @@ def get( # noqa: PLR0912, PLR0915 return data - def gather_data_array(self, node, dimension_names, dimension_units): + def gather_data_array(self, mds_node, dimension_names=None, dimension_units=None): """ Package data along with dimensions and attributes into an xarray DataArray. Pulls data from MDSPlus or local files. Parameters ---------- - node : str - MDSPlus node for the chosen data. + mds_node : MDSPlus.node + mdsplus node to which desired data is attached dimension_names : list[str] list of human readable dimension names. Should be consistent with WiPPLPy conventions --insert a link here to some info about common WiPPLPy dimensions and conventions dimension_units : list[str] @@ -537,11 +536,18 @@ def gather_data_array(self, node, dimension_names, dimension_units): """ # get data first as numpy array - data = node.data() + + data = mds_node.data() num_dimensions = data.ndim + + if len(num_dimensions) != len(dimension_units): + raise Exception( + "*** Number of dimension names and number of dimension units does not match" + ) + if num_dimensions != len(dimension_names): raise Exception( - "Number of dimension names and number of data dimensions do not match" + "*** Number of dimension names and number of data dimensions does not match" ) coords = {} @@ -551,14 +557,17 @@ def gather_data_array(self, node, dimension_names, dimension_units): coords[dim_name] = data.dim_of(dim_num).data() dimension_units[dim_name] = dimension_units - # get coordinates arrays for dimensions that need it + coords["shot_num"] = mds_node.shot # get the rest of the attributes together into - {"units": dimension_units} + attributes = {"units": dimension_units, "call_string": "thing", "version": 1.0} - data_array = xr.DataArray(data, dims=dimension_names, coords=coords) + data_xarray = xr.DataArray( + data, dims=dimension_names, coords=coords, attrs=attributes + ) - return data_array + self.xarray = data_xarray # hmm which one + return data_xarray def to_raw_index(self, time_index): """ From 5793022ed314cacb36600afc5042acef442bbc2a Mon Sep 17 00:00:00 2001 From: Antognetti Date: Tue, 18 Feb 2025 16:26:13 -0600 Subject: [PATCH 08/12] Removed bug with calling for length of int (silly silly me) --- src/wipplpy/modules/generic_get_data.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/wipplpy/modules/generic_get_data.py b/src/wipplpy/modules/generic_get_data.py index 152ddda..14c16b4 100644 --- a/src/wipplpy/modules/generic_get_data.py +++ b/src/wipplpy/modules/generic_get_data.py @@ -311,6 +311,7 @@ def _to_time_index(self, time): index : int Data index from start of digitizer recording. """ + try: times = self.time except AttributeError: @@ -540,7 +541,7 @@ def gather_data_array(self, mds_node, dimension_names=None, dimension_units=None data = mds_node.data() num_dimensions = data.ndim - if len(num_dimensions) != len(dimension_units): + if num_dimensions != len(dimension_units): raise Exception( "*** Number of dimension names and number of dimension units does not match" ) From 4dd94d24a48fe8d3feaf7b889ab5ef472fc7da21 Mon Sep 17 00:00:00 2001 From: Ben Antognetti Date: Tue, 25 Mar 2025 14:45:39 -0500 Subject: [PATCH 09/12] Intermediate commit for moving repo around on MST server --- mamba_environment_linux.yml | 15 --------------- src/wipplpy/modules/generic_get_data.py | 4 +++- src/wipplpy/modules/shot_loading_config.json | 4 ++-- 3 files changed, 5 insertions(+), 18 deletions(-) delete mode 100644 mamba_environment_linux.yml diff --git a/mamba_environment_linux.yml b/mamba_environment_linux.yml deleted file mode 100644 index 90ff2a2..0000000 --- a/mamba_environment_linux.yml +++ /dev/null @@ -1,15 +0,0 @@ -name: WiPPLPy -channels: - - defaults - - anaconda - - conda-forge -dependencies: - - python=3.9 - - jupyter - - numpy - - scipy - - matplotlib - - tqdm - - configobj - - pyyaml - - sphinx diff --git a/src/wipplpy/modules/generic_get_data.py b/src/wipplpy/modules/generic_get_data.py index 152ddda..3bea780 100644 --- a/src/wipplpy/modules/generic_get_data.py +++ b/src/wipplpy/modules/generic_get_data.py @@ -206,7 +206,9 @@ def __init__( # noqa: PLR0913 try: self.shot_number = int(tree) except ValueError: - self.shot_number = tree.shot_number + self.shot_number = tree.shot + except TypeError: + self.shot_number = tree.shot self.ignore_errors = ignore_errors self.silence_error_logging = silence_error_logging diff --git a/src/wipplpy/modules/shot_loading_config.json b/src/wipplpy/modules/shot_loading_config.json index 0afc389..50a53f0 100644 --- a/src/wipplpy/modules/shot_loading_config.json +++ b/src/wipplpy/modules/shot_loading_config.json @@ -1,4 +1,4 @@ { - "tree_name": "wipal", - "server_name": "skywalker.physics.wisc.edu" + "tree_name": "mst", + "server_name": "toni.physics.wisc.edu" } From 2a41f4b4a128ef7f8b37681501ecc011766e87c5 Mon Sep 17 00:00:00 2001 From: Ben Antognetti Date: Tue, 27 May 2025 12:50:17 -0500 Subject: [PATCH 10/12] Returned get to old way --- src/wipplpy/modules/generic_get_data.py | 72 ++++++++++++++++--------- 1 file changed, 46 insertions(+), 26 deletions(-) diff --git a/src/wipplpy/modules/generic_get_data.py b/src/wipplpy/modules/generic_get_data.py index 456564a..37eccb0 100644 --- a/src/wipplpy/modules/generic_get_data.py +++ b/src/wipplpy/modules/generic_get_data.py @@ -68,11 +68,13 @@ def __init__(self, call_string, name=None, signal=True): self.name = name self.signal = signal + def __str__(self) -> str: if self.signal: return f"Get({self.call_string}, is_signal)" else: return f"Get({self.call_string})" + def full_str(self, index_range=None, sample_period=1): """ @@ -331,6 +333,20 @@ def _to_time_index(self, time): return index + def set_array_dimensions(self): + """ + Function to be overwrriten when Data class is implemented for a specific data set + + Parameters + ---------- + dimension_names : list[str] + list of human readable dimension names. Should be consistent with WiPPLPy conventions --insert a link here to some info about common WiPPLPy dimensions and conventions-- + dimension_units : list[str] + List of dimension units. Should be in a format consistent with astropy.units https://docs.astropy.org/en/stable/units/ + """ + self.dimension_units=[] + self.dimension_names=[] + def _get_individuals(self, variable_booleans, get_calls): """ Get all data that has a True boolean associated with it. @@ -497,11 +513,11 @@ def get( # noqa: PLR0912, PLR0915 # Might be good to put the rest of these checks into a separate function, so it can be called here and in the gather_data_array if self.ignore_errors: try: - data = self.gather_data_array(node) + data = node.data() except Exception: return np.array() else: - data = self.gather_data_array(node) + data = node.data() logging.debug("Got data from tree.") if change_data: @@ -519,39 +535,43 @@ def get( # noqa: PLR0912, PLR0915 return data - def gather_data_array(self, mds_node, dimension_names=None, dimension_units=None): + def get_xarray(self, get_call, np_data_type=np.float64, change_data=True, load_from_saved=True): """ - Package data along with dimensions and attributes into an xarray DataArray. Pulls data from MDSPlus or local files. + Get data and dimensions from MDSplus tree, then store in an xarray along with metadata and proper dimensions as defined in the class Parameters ---------- - mds_node : MDSPlus.node - mdsplus node to which desired data is attached - dimension_names : list[str] - list of human readable dimension names. Should be consistent with WiPPLPy conventions --insert a link here to some info about common WiPPLPy dimensions and conventions - dimension_units : list[str] - List of dimension units. Should be in a format consistent with astropy.units https://docs.astropy.org/en/stable/units/ + get_call : str + A string that defines the call to use on the tree. + np_data_type : data-type, default=np.float64 + The numpy data type to change the data to. + change_data : bool, default=True + Whether to change the data type of what MDSplus returns. + load_from_saved : bool, default=True + Whether to try to load the data from the saved calls. Returns ------- - array : xr.DataArray - DataArray with stored data and metadata - + data : xarray DataArray + The data from the tree packaged into an xarray """ - # get data first as numpy array - data = mds_node.data() - num_dimensions = data.ndim + self.set_array_dimensions() - if num_dimensions != len(dimension_units): - raise Exception( - "*** Number of dimension names and number of dimension units does not match" - ) + dimension_units = self.dimension_units + dimension_names = self.dimension_names + print(type(connection)) + num_dimensions = connection + if isinstance(num_dimensions, list) and isinstance(dimension_names, list): + if num_dimensions != len(dimension_units): + raise Exception( + "*** Number of dimension names and number of dimension units does not match" + ) - if num_dimensions != len(dimension_names): - raise Exception( - "*** Number of dimension names and number of data dimensions does not match" - ) + if num_dimensions != len(dimension_names): + raise Exception( + "*** Number of dimension names and number of data dimensions does not match" + ) coords = {} dimension_units = {} @@ -559,11 +579,11 @@ def gather_data_array(self, mds_node, dimension_names=None, dimension_units=None for dim_num, dim_name in dimension_names: coords[dim_name] = data.dim_of(dim_num).data() dimension_units[dim_name] = dimension_units - + print(type(mds_node)) #delete me coords["shot_num"] = mds_node.shot # get the rest of the attributes together into - attributes = {"units": dimension_units, "call_string": "thing", "version": 1.0} + attributes = {"units": dimension_units, "call_string": self.call_string, "version": 1.0} data_xarray = xr.DataArray( data, dims=dimension_names, coords=coords, attrs=attributes From ec3897a6d804b892bc6f4ae5dff7e8484cd78f4a Mon Sep 17 00:00:00 2001 From: Ben Antognetti Date: Thu, 26 Jun 2025 14:28:48 -0500 Subject: [PATCH 11/12] First functioning Data class with xarray capability --- src/wipplpy/modules/generic_get_data.py | 67 ++++++++++++------------- 1 file changed, 32 insertions(+), 35 deletions(-) diff --git a/src/wipplpy/modules/generic_get_data.py b/src/wipplpy/modules/generic_get_data.py index 37eccb0..d67f6c9 100644 --- a/src/wipplpy/modules/generic_get_data.py +++ b/src/wipplpy/modules/generic_get_data.py @@ -217,6 +217,10 @@ def __init__( # noqa: PLR0913 self._tree = None + #Make attributes for dimension_names and dimension_units if specified + self.units=[] + self.names=[] + # Initialize the time index range as empty and then try to get something for it. We do this because some code in _to_time_index_range requires it. self.time_index_range = None self.sample_period = sample_period @@ -233,6 +237,8 @@ def __init__( # noqa: PLR0913 elif len(get_calls) == 0: logging.info("Did not do any get calls for preloading data into object.") + self.get_calls = get_calls + # Hold all the calls and call data gotten from MDSplus. # TODO: Change how calls are saved so that we can change loaded attributes and save the update. self.saved_calls = {} @@ -333,20 +339,6 @@ def _to_time_index(self, time): return index - def set_array_dimensions(self): - """ - Function to be overwrriten when Data class is implemented for a specific data set - - Parameters - ---------- - dimension_names : list[str] - list of human readable dimension names. Should be consistent with WiPPLPy conventions --insert a link here to some info about common WiPPLPy dimensions and conventions-- - dimension_units : list[str] - List of dimension units. Should be in a format consistent with astropy.units https://docs.astropy.org/en/stable/units/ - """ - self.dimension_units=[] - self.dimension_names=[] - def _get_individuals(self, variable_booleans, get_calls): """ Get all data that has a True boolean associated with it. @@ -535,7 +527,7 @@ def get( # noqa: PLR0912, PLR0915 return data - def get_xarray(self, get_call, np_data_type=np.float64, change_data=True, load_from_saved=True): + def get_xarray(self, np_data_type=np.float64, change_data=True, load_from_saved=True): """ Get data and dimensions from MDSplus tree, then store in an xarray along with metadata and proper dimensions as defined in the class @@ -555,42 +547,47 @@ def get_xarray(self, get_call, np_data_type=np.float64, change_data=True, load_f data : xarray DataArray The data from the tree packaged into an xarray """ + + units = self.units + names = self.names + get_call = self.get_calls[0] + print(get_call) - self.set_array_dimensions() - - dimension_units = self.dimension_units - dimension_names = self.dimension_names - print(type(connection)) - num_dimensions = connection - if isinstance(num_dimensions, list) and isinstance(dimension_names, list): - if num_dimensions != len(dimension_units): + #Needs to get expanded for multiple get_calls passed!!!!!!!!!!!!!&&&&&&&&&&&&&&&&&&&)))))))()()() + + data = self.get(get_call, np_data_type=np_data_type, change_data=change_data, load_from_saved=load_from_saved) + + num_dimensions = data.ndim + if isinstance(units, list) and isinstance(names, list): + if num_dimensions + 1 != len(units): raise Exception( "*** Number of dimension names and number of dimension units does not match" ) - if num_dimensions != len(dimension_names): + if num_dimensions + 1 != len(names): raise Exception( "*** Number of dimension names and number of data dimensions does not match" ) - coords = {} - dimension_units = {} + coords_dict = {} + units_dict = {} + + for dim_num, dim_name in enumerate(names[1:]): + coords_dict[dim_name] = self.get('DIM_OF('+ get_call + ', {:d})'.format(dim_num), np_data_type=np_data_type, change_data=change_data, load_from_saved=load_from_saved) + units_dict[dim_name] = units[dim_num+1] - for dim_num, dim_name in dimension_names: - coords[dim_name] = data.dim_of(dim_num).data() - dimension_units[dim_name] = dimension_units - print(type(mds_node)) #delete me - coords["shot_num"] = mds_node.shot + units_dict[names[0]] = units[0] + + coords_dict["shot_num"] = int(self.get('$SHOT')) # get the rest of the attributes together into - attributes = {"units": dimension_units, "call_string": self.call_string, "version": 1.0} + attributes = {"units": units_dict, "call_string": get_call, "version": 1.0, 'tree': self.get('$EXPT', change_data = False)} data_xarray = xr.DataArray( - data, dims=dimension_names, coords=coords, attrs=attributes + data, dims=names[1:], coords=coords_dict, attrs=attributes ) - self.xarray = data_xarray # hmm which one - return data_xarray + self.xarray = data_xarray def to_raw_index(self, time_index): """ From 80f111e6c6e054c0f5109a0b8f26780a5a3c4db9 Mon Sep 17 00:00:00 2001 From: Ben Antognetti Date: Fri, 27 Jun 2025 15:07:12 -0500 Subject: [PATCH 12/12] Added first example of diagnostic definition --- src/wipplpy/__init__.py | 1 + src/wipplpy/modules/generic_get_data.py | 2 +- src/wipplpy/mst/diagnostics/summary.py | 24 ++++++++++++++++++++++++ 3 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 src/wipplpy/mst/diagnostics/summary.py diff --git a/src/wipplpy/__init__.py b/src/wipplpy/__init__.py index 37579ee..66a65b3 100644 --- a/src/wipplpy/__init__.py +++ b/src/wipplpy/__init__.py @@ -8,3 +8,4 @@ ] from wipplpy import modules +from .mst.diagnostics.summary import PlasmaCurrent \ No newline at end of file diff --git a/src/wipplpy/modules/generic_get_data.py b/src/wipplpy/modules/generic_get_data.py index d67f6c9..f8a8783 100644 --- a/src/wipplpy/modules/generic_get_data.py +++ b/src/wipplpy/modules/generic_get_data.py @@ -167,7 +167,7 @@ def __init__( # noqa: PLR0913 ): """ Generic class for dealing with data from a remote MDSplus database. - + Parameters ---------- tree : Connection or int diff --git a/src/wipplpy/mst/diagnostics/summary.py b/src/wipplpy/mst/diagnostics/summary.py new file mode 100644 index 0000000..8fd9aa7 --- /dev/null +++ b/src/wipplpy/mst/diagnostics/summary.py @@ -0,0 +1,24 @@ +import matplotlib.pyplot as plt +from wipplpy.modules.generic_get_data import Data as Data +import numpy as np +import xarray as xr +import MDSplus as mds + +class PlasmaCurrent(Data): + def __init__(self, tree): + #Ordering does matter. Define calls, super init, then dimension specifications. Figuring out how to make it not matter, or streamline whats needed would be cool + calls = ['\MST_OPS::IP_F'] #want to add ability to have multiple in this list + super().__init__(tree, [True], calls) # could augment second argument as a keyword with Default None in which case it does all of calls + self.units = ['kA', 's'] + self.names = ['plasma_current', 'shot_time'] + + def plot(self): + if not hasattr(self, "xarray"): + self.get_xarray() + print("Put together xarray") + + fig, ax = plt.subplots(figsize=(8,4)) + plt.plot(self.xarray.shot_time * 1000, self.xarray.data) + ax.set_xlabel('Time (ms)') + ax.set_ylabel('Plasma Current (kA)') + plt.show() \ No newline at end of file