From a75ced3c08fa36300821cefd206ffd5cc4e0ad8a Mon Sep 17 00:00:00 2001 From: Peter Scheidler Date: Wed, 4 Jun 2025 15:16:33 -0400 Subject: [PATCH 01/16] Updating for python 3.13. (#228) * Updating for python 3.13. Includes updates for plotly 6 to remove heatmapgl and update mapbox to map, also fixed GPS plotting issue * replacing pkg_resource, updating github action versions, adding debug for xmlrpc error * Working on xmlrpc issue, disabling unit tests for a moment * Updated the doc tests to run on Python3.13, fixed a bunch of typos. Tested locally, which doesn't mean much * Removing the last of the debug stuff, setting project to Stable, adding .bak to gitignore * Sphinx fixes, minor doc edits --------- Co-authored-by: stokesMIDE --- .github/workflows/docs-tests.yml | 12 ++-- .github/workflows/publish-to-pypi.yml | 2 +- .github/workflows/unit-tests.yml | 2 +- .gitignore | 4 ++ .readthedocs.yaml | 4 +- docs/conf.py | 30 +++++++-- docs/endaq/ide_usage.rst | 62 +++++++++++++++++++ docs/requirements.txt | 19 +++--- docs/spelling_wordlist.txt | 19 ++++++ ...tro_Python_Acceleration_CSV_Analysis.ipynb | 6 +- ...ebinar_Introduction_NumPy_and_Pandas.ipynb | 6 +- .../Webinar_Introduction_Plotly.ipynb | 4 +- .../Webinar_enDAQ_Custom_Analysis.ipynb | 12 ++-- endaq/calc/rotation.py | 10 +-- endaq/plot/plots.py | 19 ++---- endaq/plot/utilities.py | 11 ++-- setup.py | 4 +- tests/plot/test_plots.py | 2 +- 18 files changed, 164 insertions(+), 64 deletions(-) diff --git a/.github/workflows/docs-tests.yml b/.github/workflows/docs-tests.yml index c0d6bc03..67d1372f 100644 --- a/.github/workflows/docs-tests.yml +++ b/.github/workflows/docs-tests.yml @@ -22,16 +22,16 @@ jobs: env: OS: ubuntu-latest - PYTHON-VERSION: "3.9" + PYTHON-VERSION: "3.13" runs-on: ubuntu-latest steps: - name: Setup Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: - python-version: "3.9" + python-version: "3.13" - name: Install Pandoc run: sudo apt-get install pandoc @@ -40,7 +40,7 @@ jobs: run: python -m pip install --upgrade pip - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Get pip cache location id: pip-cache @@ -57,10 +57,10 @@ jobs: print(f'text={now.year}/{now.month}-part{1 + now.day // 8}')" >> $GITHUB_OUTPUT - name: Load pip cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ${{ steps.pip-cache.outputs.dir }} - key: doctests-${{ runner.os }}-Python3.9-${{ steps.date.outputs.text }}-pip-${{ hashFiles('**/setup.py', './.github/workflows/unit-tests.yml') }} + key: doctests-${{ runner.os }}-Python3.13-${{ steps.date.outputs.text }}-pip-${{ hashFiles('**/setup.py', './.github/workflows/unit-tests.yml') }} - name: Install spellcheck library run: sudo apt-get install libenchant-2-2 diff --git a/.github/workflows/publish-to-pypi.yml b/.github/workflows/publish-to-pypi.yml index 1cffbc13..c8618c03 100644 --- a/.github/workflows/publish-to-pypi.yml +++ b/.github/workflows/publish-to-pypi.yml @@ -14,7 +14,7 @@ jobs: - uses: actions/checkout@master - name: Set up Python 3.9 - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: 3.9 diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 2fbb73e7..7f56fa44 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -21,7 +21,7 @@ jobs: fail-fast: false matrix: os: [windows-latest, ubuntu-latest, macos-latest] - python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] + python-version: ['3.9', '3.10', '3.11', '3.12', '3.13'] env: OS: ${{ matrix.os }} diff --git a/.gitignore b/.gitignore index 12245807..128b34b1 100644 --- a/.gitignore +++ b/.gitignore @@ -179,6 +179,7 @@ cython_debug/ .idea/**/usage.statistics.xml .idea/**/dictionaries .idea/**/shelf +.vscode/* # Generated files .idea/**/contentModel.xml @@ -244,3 +245,6 @@ fabric.properties # Test files *_scratch.ipynb + +# Backup files +*.bak \ No newline at end of file diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 6a7ad9ce..03701c14 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -7,9 +7,9 @@ version: 2 # Set the version of Python and other tools you might need build: - os: ubuntu-20.04 + os: ubuntu-24.04 tools: - python: "3.8" + python: "3.13" # Build documentation in the docs/ directory with Sphinx sphinx: diff --git a/docs/conf.py b/docs/conf.py index 4b2daa09..529c8c7d 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -13,18 +13,35 @@ # documentation root, use os.path.abspath to make it absolute, like shown here. # -import pkg_resources +import codecs +import os.path +import sys + +# go up a dir and include that guy = +p = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) +sys.path.insert(0, p) -import endaq # -- Project information ----------------------------------------------------- +def get_version(rel_path): + """ Read the version number directly from the source. """ + with codecs.open(rel_path, 'r') as fp: + for line in fp: + if line.startswith('__version__'): + delim = '"' if '"' in line else "'" + return line.split(delim)[1] + else: + raise RuntimeError("Unable to find version string.") + + project = 'enDAQ' copyright = '2021, Mide Technology Corp.' author = '' # The full version, including alpha/beta/rc tags -release = pkg_resources.get_distribution("endaq").version +release = get_version(os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'endaq', '__init__.py'))) + # The short X.Y version version = '.'.join(release.split(".")[:2]) @@ -44,6 +61,7 @@ 'sphinx.ext.ifconfig', 'sphinx.ext.githubpages', 'sphinx_plotly_directive', + 'sphinxcontrib.spelling', 'nbsphinx', ] @@ -64,7 +82,7 @@ # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language = 'en' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. @@ -100,7 +118,9 @@ "github_url": "https://github.com/MideTechnology/endaq-python", "twitter_url": "https://twitter.com/enDAQ_sensors", "collapse_navigation": True, - "google_analytics_id": "G-E9QXH4H5LP", + "analytics": { + "google_analytics_id": "G-E9QXH4H5LP", + } } # Add any paths that contain custom static files (such as style sheets) here, diff --git a/docs/endaq/ide_usage.rst b/docs/endaq/ide_usage.rst index 324392ed..7099c835 100644 --- a/docs/endaq/ide_usage.rst +++ b/docs/endaq/ide_usage.rst @@ -37,6 +37,68 @@ parameters: doc3 = get_doc("tests/test.ide", start="5s", end="10s") +Accessing measurement data in a Dataset/IDE file +------------------------------------------------ +An enDAQ device consists of many different sensors, and enDAQ devices record their measurement data into separate +Channels that correspond to the sensor taking the measurement. This is done because each Channel samples at a different +rate, so while Channel 59 (the Control Pad Pressure/Temperature/Humidity sensor) samples at 10 Hz, Channel 8 (the main +analog accelerometer channel) may sample at 20000 Hz. Channels themselves consist of different subchannels, which may be +different axes (X, Y, Z) or completely different measurements like temperature and pressure. All subchannels in a +channel are sampled at approximately the same time. + +The Channel data are stored in the ``channels`` property of a Dataset, which is returned from the :py:func:`~endaq.ide.get_doc() function. The easiest way to access this is to convert it to a +Pandas DataFrame using :py:func:`~endaq.ide.to_pandas(doc)`. Visit `our internal documentation `_ +for some quick tips on Pandas, or go `straight to the source `_. + +.. code:: python3 + + import endaq.ide + # Read in a doc + doc2 = endaq.ide.get_doc("https://drive.google.com/file/d/1t3JqbZGhuZbIK9agH24YZIdVE26-NOF5/view?usp=sharing") + # List the available Channels + print(f"{doc2.channels=}") + # Convert the Control Pad Pressure/Temperature/Humidity Channel (Channel 59) to a Pandas DataFrame + control_pad_data = endaq.ide.to_pandas(doc2.channels[59]) + # Print the subchannel names + print(f"{control_pad_data.columns=}") + # Print the max and min temperatures seen + print(f"Max Temp={control_pad_data['Control Pad Temperature'].max()}, Min Temp={control_pad_data['Control Pad Temperature'].min()}") + +The output of the above code is: + +.. code-block:: + + doc2.channels={32: , 80: , 36: , 70: , 59: , 76: } + control_pad_data.columns=Index(['Control Pad Pressure', 'Control Pad Temperature'], dtype='object') + Max Temp=24.899999618530273, Min Temp=24.260000228881836 + +Note that by default, :py:func:`~endaq.ide.to_pandas(doc)` uses ``datetime`` for the index format, meaning the +measurements are accessed based on the absolute time they were recorded. Users often prefer to access the data using +``timedelta``, the amount of time since the recording started. Using this, to get the duration of the Control Pad data +and the average of the first 5 seconds, we could use: + +.. code:: python3 + + import endaq.ide + import pandas as pd + # Read in a doc + doc2 = endaq.ide.get_doc("https://drive.google.com/file/d/1t3JqbZGhuZbIK9agH24YZIdVE26-NOF5/view?usp=sharing") + # Convert the Control Pad Pressure/Temperature/Humidity Channel (Channel 59) to a Pandas DataFrame + control_pad_data = endaq.ide.to_pandas(doc2.channels[59], time_mode='timedelta') + # Print the time duration + print(f"Duration={control_pad_data.index[-1]-control_pad_data.index[0]}") + # Print the mean of the first 5 seconds + print(f"{control_pad_data[pd.Timedelta(seconds=0):pd.Timedelta(seconds=5)].mean()}") + +The output of the above code is: + +.. code-block:: + + Duration=0 days 00:00:17.931518 + Control Pad Pressure 101728.414991 + Control Pad Temperature 24.607073 + dtype: float64 + Summarizing IDE files: :py:func:`endaq.ide.get_channel_table()` --------------------------------------------------------------- diff --git a/docs/requirements.txt b/docs/requirements.txt index baa9f312..4fa17c41 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,12 +1,13 @@ -Sphinx>=5.0.2 -sphinxcontrib-applehelp==1.0.2 -sphinxcontrib-devhelp==1.0.2 -sphinxcontrib-htmlhelp==2.0.0 +Sphinx>=8.1.3 +sphinxcontrib-applehelp==2.0.0 +sphinxcontrib-devhelp==2.0.0 +sphinxcontrib-htmlhelp==2.1.0 sphinxcontrib-jsmath==1.0.1 -sphinxcontrib-qthelp==1.0.3 -sphinxcontrib-serializinghtml==1.1.5 +sphinxcontrib-qthelp==2.0.0 +sphinxcontrib-serializinghtml==2.0.0 +sphinxcontrib-spelling==8.0.1 -pydata-sphinx-theme==0.7.1 +pydata-sphinx-theme==0.16.1 sphinx-plotly-directive==0.1.3 -nbsphinx==0.8.8 -ipython==8.10 +nbsphinx==0.9.7 +ipython==9.2.0 diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index 69285ba1..af58671c 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -5,14 +5,29 @@ sinusoids centric enDAQ +Mide NumPy SciPy Pandas Plotly +Matplotlib +Colab +Jupyter +Seaborn Tukey Chebyshev Butterworth Fourier +Heatmaps +ide +calc +endaq +accel +str +psd +pkl +kth +GetDataBuilder basename pathname @@ -24,6 +39,7 @@ subchannel subchannels DataFrame dataframe +dataframes ndarray dropdown iterable @@ -36,6 +52,7 @@ periodogram periodograms spectrogram spectrograms +spectrums resample resampled resampling @@ -43,3 +60,5 @@ SNE integrations quaternion quaternions +Welch +kurtosis \ No newline at end of file diff --git a/docs/webinars/Webinar_Intro_Python_Acceleration_CSV_Analysis.ipynb b/docs/webinars/Webinar_Intro_Python_Acceleration_CSV_Analysis.ipynb index 03a172c1..a073667f 100644 --- a/docs/webinars/Webinar_Intro_Python_Acceleration_CSV_Analysis.ipynb +++ b/docs/webinars/Webinar_Intro_Python_Acceleration_CSV_Analysis.ipynb @@ -14,7 +14,7 @@ "## Introduction\n", "This notebook serves as an introduction to Python for a mechanical engineer looking to plot and analyze some acceleration data in a CSV file. Being a Colab, this tool can freely be used without installing anything.\n", "\n", - "For more information on making the swith to Python see [enDAQ's blog, Why and How to Get Started in Python for a MATLAB User](https://blog.endaq.com/why-and-how-to-get-started-in-python-for-a-matlab-user).\n", + "For more information on making the switch to Python see [enDAQ's blog, Why and How to Get Started in Python for a MATLAB User](https://blog.endaq.com/why-and-how-to-get-started-in-python-for-a-matlab-user).\n", "\n", "This is part of our webinar series on Python for Mechanical Engineers:\n", "\n", @@ -1493,7 +1493,7 @@ }, "source": [ "### FFT from PSD\n", - "Here we can use the output of a PSD and convet it to a typical DFT. This has the benefit of allowing you to explicitely define the frequency bin width." + "Here we can use the output of a PSD and convert it to a typical DFT. This has the benefit of allowing you to explicitly define the frequency bin width." ] }, { @@ -1887,4 +1887,4 @@ }, "nbformat": 4, "nbformat_minor": 4 -} \ No newline at end of file +} diff --git a/docs/webinars/Webinar_Introduction_NumPy_and_Pandas.ipynb b/docs/webinars/Webinar_Introduction_NumPy_and_Pandas.ipynb index 54597578..01ee2025 100644 --- a/docs/webinars/Webinar_Introduction_NumPy_and_Pandas.ipynb +++ b/docs/webinars/Webinar_Introduction_NumPy_and_Pandas.ipynb @@ -14,7 +14,7 @@ "\n", "1. [Get Started with Python](https://colab.research.google.com/drive/1_pcGtgJleapV9tz5WfuRuqfWryjqhPHy#scrollTo=ikUJITDDIp19)\n", " * Blog: [Get Started with Python: Why and How Mechanical Engineers Should Make the Switch](https://blog.endaq.com/get-started-with-python-why-how-mechanical-engineers-should-make-the-switch)\n", - "2. **Introduction to Numpy & Pandas**\n", + "2. **Introduction to NumPy & Pandas**\n", " * [Watch Recording of This](https://info.endaq.com/pandas-and-numpy-for-data-analysis-webinar)\n", "3. [Introduction to Plotly](https://colab.research.google.com/drive/1pag2pKQQW5amWgRykAH8uMAPqHA2yUfU?usp=sharing)\n", "4. [Introduction of the enDAQ Library](https://colab.research.google.com/drive/1WAtQ8JJC_ny0fki7eUABACMA-isZzKB6)\n", @@ -505,7 +505,7 @@ "id": "cv0nw1SnLb5x" }, "source": [ - "Logspace is the equivalent of rasing a base by a linspaced array." + "Logspace is the equivalent of raising a base by a linspaced array." ] }, { @@ -7821,7 +7821,7 @@ "source": [ "### Installation\n", "\n", - "The code is live on [GitHub](https://github.com/MideTechnology/endaq-python), [PyPI](https://pypi.org/project/endaq/), and cleaner documentation is in process that will eventually live on a subdomain of endaq.com.\n", + "The code is live on [GitHub](https://github.com/MideTechnology/endaq-python), [PyPI](https://pypi.org/project/endaq/), and cleaner documentation lives at [docs.endaq.com](https://docs.endaq.com/en/latest/).\n", "\n", "It can easily be installed with pip." ] diff --git a/docs/webinars/Webinar_Introduction_Plotly.ipynb b/docs/webinars/Webinar_Introduction_Plotly.ipynb index 0ca23569..735c3a61 100644 --- a/docs/webinars/Webinar_Introduction_Plotly.ipynb +++ b/docs/webinars/Webinar_Introduction_Plotly.ipynb @@ -345,7 +345,7 @@ }, "source": [ "### [ggplot](https://plotnine.readthedocs.io/en/stable/)\n", - "Introduces a \"grammer of graphics\" logic to plotting data which allows explicit mapping of data to the visual representation. This is something plotly express excels at." + "Introduces a \"grammar of graphics\" logic to plotting data which allows explicit mapping of data to the visual representation. This is something plotly express excels at." ] }, { @@ -1304,7 +1304,7 @@ "id": "u_pZeZ5An8PG" }, "source": [ - "Now let's get crazy and customize all the \"common\" settings. But note that there are a LOT of different parameters that can be explicitely defined. Remember, Plotly has very thorough documentation, so check it out!\n", + "Now let's get crazy and customize all the \"common\" settings. But note that there are a LOT of different parameters that can be explicitly defined. Remember, Plotly has very thorough documentation, so check it out!\n", "* [Figure Layout](https://plotly.com/python/reference/layout/)\n", "* [X Axis](https://plotly.com/python/reference/layout/xaxis/)\n", "* [Y Axis](https://plotly.com/python/reference/layout/yaxis/)\n", diff --git a/docs/webinars/Webinar_enDAQ_Custom_Analysis.ipynb b/docs/webinars/Webinar_enDAQ_Custom_Analysis.ipynb index 1e583e47..2ce76142 100644 --- a/docs/webinars/Webinar_enDAQ_Custom_Analysis.ipynb +++ b/docs/webinars/Webinar_enDAQ_Custom_Analysis.ipynb @@ -17,7 +17,7 @@ "\n", "1. [Get Started with Python](https://colab.research.google.com/drive/1_pcGtgJleapV9tz5WfuRuqfWryjqhPHy#scrollTo=ikUJITDDIp19)\n", " * Blog: [Get Started with Python: Why and How Mechanical Engineers Should Make the Switch](https://blog.endaq.com/get-started-with-python-why-how-mechanical-engineers-should-make-the-switch)\n", - "2. [Introduction to Numpy & Pandas for Data Analysis](https://colab.research.google.com/drive/1O-VwAdRoSlcrineAk0Jkd_fcw7mFGHa4#scrollTo=ce97q1ZcBiwj)\n", + "2. [Introduction to NumPy & Pandas for Data Analysis](https://colab.research.google.com/drive/1O-VwAdRoSlcrineAk0Jkd_fcw7mFGHa4#scrollTo=ce97q1ZcBiwj)\n", "3. [Introduction to Plotly for Plotting Data](https://colab.research.google.com/drive/1pag2pKQQW5amWgRykAH8uMAPqHA2yUfU)\n", "4. [Introduction of the enDAQ Library](https://colab.research.google.com/drive/1WAtQ8JJC_ny0fki7eUABACMA-isZzKB6)\n", " - There are lots of examples in this!\n", @@ -1082,7 +1082,7 @@ "id": "env80d2NrsQ8" }, "source": [ - "Now we will use the [FFTW algorithm](http://www.fftw.org/) which is available in the [pyFFTW library](https://hgomersall.github.io/pyFFTW/index.html) under a GPL license (which makes it potentially difficult for us to use because we use the more premissive MIT license).\n", + "Now we will use the [FFTW algorithm](http://www.fftw.org/) which is available in the [pyFFTW library](https://hgomersall.github.io/pyFFTW/index.html) under a GPL license (which makes it potentially difficult for us to use because we use the more permissive MIT license).\n", "\n", "First let's download it." ] @@ -1117,7 +1117,7 @@ "id": "M70BmBjKsRvV" }, "source": [ - "Now let's use it in a function which allows for a drop-in replacement to the Numpy code. This algorithm is generally regarded as the fatest for computing discrete Fourier transforms - so we'll put it to the test!" + "Now let's use it in a function which allows for a drop-in replacement to the Numpy code. This algorithm is generally regarded as the fastest for computing discrete Fourier transforms - so we'll put it to the test!" ] }, { @@ -1585,7 +1585,7 @@ "id": "5bXza1s0Ep-Q" }, "source": [ - "So what does this mean!? FFTW is the fastest as expected, but only if we first structure the data in a more efficient way. But typically you will not have the data structured in this \"optimal\" way for FFTW which means the time it takes to restucture it is real.\n", + "So what does this mean!? FFTW is the fastest as expected, but only if we first structure the data in a more efficient way. But typically you will not have the data structured in this \"optimal\" way for FFTW which means the time it takes to restructure it is real.\n", "\n", "Long story short for *this audience*, using Welch's method is fastest!" ] @@ -4466,7 +4466,7 @@ "\n", "This is currently available, [see docs](https://docs.endaq.com/en/latest/endaq/batch.html), but we are working on a few bug fixes and improved functionality. This module allows you to batch process many *IDE* (only works for our sensors for now).\n", "\n", - "In a seperate document I first executed the following code to gather all the .IDE files I wanted to analyze (I hide the actual folder name).\n", + "In a separate document I first executed the following code to gather all the .IDE files I wanted to analyze (I hide the actual folder name).\n", "~~~python\n", "import glob\n", "directory = r\"C:\\Users\\shanly\\enDAQ-Notebooks\\...\"+\"\\\\\"\n", @@ -5450,7 +5450,7 @@ "id": "iFv3nr5DPBhj" }, "source": [ - "That's pretty cool! But we'll notice that the animation moves outside the inital bounds pretty quickly. So let's first find an easy way to calculate these metrics of max/min/median per frequency bin." + "That's pretty cool! But we'll notice that the animation moves outside the initial bounds pretty quickly. So let's first find an easy way to calculate these metrics of max/min/median per frequency bin." ] }, { diff --git a/endaq/calc/rotation.py b/endaq/calc/rotation.py index e3117ca9..abb19e8b 100644 --- a/endaq/calc/rotation.py +++ b/endaq/calc/rotation.py @@ -31,7 +31,7 @@ def _validate_euler_mode(mode: str) -> Tuple[str, List[str]]: if not set(_mode).issubset({'x', 'y', 'z'}): raise ValueError(f'Modes other than xyz (such as xyz, xyx, zxz) must ' f'separated with one of " ", "-" or "_". Mode ' - f'{mode} is not a valid euler angle mode.') + f'{mode} is not a valid Euler angle mode.') mode_list = list(_mode) if not ( @@ -63,18 +63,18 @@ def _validate_euler_mode(mode: str) -> Tuple[str, List[str]]: def quaternion_to_euler(df: pd.DataFrame, mode: str = 'x-y-z') -> pd.DataFrame: """ - Convert quaternion data in the dataframe ``df`` to euler angles. This can + Convert quaternion data in the dataframe ``df`` to Euler angles. This can be done with either intrinsic or extrinsic rotations, determined automatically based on ``mode``. - :param df: The input quaternions to convert. Must have columns labelled + :param df: The input quaternions to convert. Must have columns labeled 'X', 'Y', 'Z', and 'W'. :param mode: The order of the axes to rotate. The default is intrinsic rotation about x-y-z. - :return: A dataframe with the euler-angles of the quaternion data. + :return: A dataframe with the Euler-angles of the quaternion data. .. seealso:: - - `SciPy's documentation on converting into euler angles `_ + - `SciPy's documentation on converting into Euler angles `_ - `Wikipedia's article on Euler angles `_ """ diff --git a/endaq/plot/plots.py b/endaq/plot/plots.py index 69afcadc..7996570e 100644 --- a/endaq/plot/plots.py +++ b/endaq/plot/plots.py @@ -259,11 +259,9 @@ def gen_map(df_map: pd.DataFrame, (defaults to ground speed). :param df_map: The pandas dataframe containing the recording data. - :param mapbox_access_token: The access token (or API key) needed to be able to plot against a map using Mapbox, - `create a free account here `_ - - * If no access token is provided, a `"stamen-terrain"` tile will be used, - `see Plotly for more information `_ + :param mapbox_access_token: Deprecated, the access token is no longer needed, plots are now made through MapLibre, + `to learn more, see `_ + `see Plotly for more information `_ :param lat: The dataframe column title to use for latitude :param lon: The dataframe column title to use for longitude :param color_by_column: The dataframe column title to color the plotted points by. @@ -321,22 +319,17 @@ def gen_map(df_map: pd.DataFrame, zoom = determine_plotly_map_zoom(lats=df_map[lat], lons=df_map[lon]) center = get_center_of_coordinates(lats=df_map[lat], lons=df_map[lon]) - px.set_mapbox_access_token(mapbox_access_token) - - fig = px.scatter_mapbox( + fig = px.scatter_map( df_map, lat=lat, lon=lon, color=color_by_column, hover_data=hover_data, size_max=size_max, - zoom=zoom + zoom_offset, + zoom=int(zoom + zoom_offset), center=center, ) - if mapbox_access_token is None: - fig.update_layout(mapbox_style="stamen-terrain") - return fig.update_layout(margin={"r": 20, "t": 20, "l": 20, "b": 0}) @@ -658,7 +651,7 @@ def spectrum_over_time( * `Peak`: per timestamp the peak frequency is determined and plotted against time * `Lines`: the value in each frequency bin is plotted against time :param var_column: the column name in the dataframe that defines the different variables, default is `"variable"` - :param var_to_process: the variable value in the `var_column` to filter the input df down to, + :param var_to_process: the variable value in the `var_column` to filter the input `df` down to, if none is provided (the default) this function will filter to the first value :param time_column: the column name in the dataframe that defines the timestamps, default is `"timestamp"` :param freq_column: the column name in the dataframe that defines the frequency, default is `"frequency (Hz)"` diff --git a/endaq/plot/utilities.py b/endaq/plot/utilities.py index babf47f4..f1e62a3c 100644 --- a/endaq/plot/utilities.py +++ b/endaq/plot/utilities.py @@ -4,7 +4,8 @@ import plotly.graph_objects as go import numpy as np import typing -from typing import Union +from typing import Union, Optional +import copy def define_theme(template_name: str = "endaq_cloud", default_plotly_template: str = 'plotly_dark', @@ -44,7 +45,7 @@ def define_theme(template_name: str = "endaq_cloud", default_plotly_template: st pio.templates[template_name]['layout']['colorscale']['diverging'] = [[0.0, '#6914F0'], [0.5, '#f7f7f7'], [1.0, '#EE7F27']] - plot_types = ['contour', 'heatmap', 'heatmapgl', 'histogram2d', 'histogram2dcontour', 'surface'] + plot_types = ['contour', 'heatmap', 'histogram2d', 'histogram2dcontour', 'surface'] for p in plot_types: pio.templates[template_name]['data'][p][0].colorscale = colorbar @@ -151,8 +152,8 @@ def get_center_of_coordinates(lats: np.ndarray, lons: np.ndarray, as_list: bool on the formatting of this return value """ # Create Copy to Not Change Source Data - lats = np.copy(lats) - lons = np.copy(lons) + lats = copy.deepcopy(lats) + lons = copy.deepcopy(lons) # Convert coordinates to radians if given in degrees if as_degrees: @@ -197,7 +198,7 @@ def determine_plotly_map_zoom( margin: float = 1.2, ) -> float: """ - Finds optimal zoom for a plotly mapbox. Must be passed (``lons`` & ``lats``) or ``lonlats``. + Finds optimal zoom for a plotly map. Must be passed (``lons`` & ``lats``) or ``lonlats``. Originally based on the following post: https://stackoverflow.com/questions/63787612/plotly-automatic-zooming-for-mapbox-maps diff --git a/setup.py b/setup.py index a19b21ba..48bda63e 100644 --- a/setup.py +++ b/setup.py @@ -65,14 +65,14 @@ def get_version(rel_path): long_description_content_type='text/markdown', url='https://github.com/MideTechnology/endaq-python', license='MIT', - classifiers=['Development Status :: 4 - Beta', + classifiers=['Development Status :: 5 - Production/Stable', 'License :: OSI Approved :: MIT License', 'Natural Language :: English', - 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', 'Programming Language :: Python :: 3.12', + 'Programming Language :: Python :: 3.13', 'Topic :: Scientific/Engineering', ], keywords='ebml binary ide mide endaq', diff --git a/tests/plot/test_plots.py b/tests/plot/test_plots.py index 6cf08705..60f5d4b7 100644 --- a/tests/plot/test_plots.py +++ b/tests/plot/test_plots.py @@ -106,7 +106,7 @@ def test_map(): lon="Longitude", color_by_column="Ground Speed" ) - assert fig['data'][0]['subplot'] == 'mapbox' + assert fig['data'][0]['subplot'] == 'map' def test_table_plot(generate_dataframe): From 051f7fb91fe8ae5bb8e080b15403fdd39ec0a1bf Mon Sep 17 00:00:00 2001 From: Elin O'Neill Date: Fri, 27 Jun 2025 16:31:50 -0400 Subject: [PATCH 02/16] Implemented 'to_altitude()' and corresponding tests (nonstratosphere data) --- endaq/calc/utils.py | 101 ++++++++++++++++++++++- tests/calc/csv_to_df/default_sea_lvl.csv | 18 ++++ tests/calc/test_utils.py | 68 +++++++++++++++ 3 files changed, 184 insertions(+), 3 deletions(-) create mode 100644 tests/calc/csv_to_df/default_sea_lvl.csv diff --git a/endaq/calc/utils.py b/endaq/calc/utils.py index 2ba7eb98..17353800 100644 --- a/endaq/calc/utils.py +++ b/endaq/calc/utils.py @@ -3,7 +3,7 @@ from __future__ import annotations import typing -from typing import Optional, Union +from typing import Optional, Union, Literal import warnings import numpy as np @@ -161,9 +161,9 @@ def resample(df: pd.DataFrame, sample_rate: Optional[float] = None) -> pd.DataFr index=resampled_time.astype(df.index.dtype), columns=df.columns, ) - + resampled_df.index.name = df.index.name - + return resampled_df @@ -343,3 +343,98 @@ def convert_units( for i, c in enumerate(df.columns): converted_df[c] = vals[:, i] return converted_df + + +def to_altitude(df: pd.DataFrame, + base_altitude: float = 0, + base_press: Optional[float] = 101325, + base_temp: Optional[float] = 15, + units: Literal['m', 'ft'] = 'm') -> pd.DataFrame: + """ + Converts pressure (Pascals) to altitude (feet or meters). + + :param df: pandas DataFrame of one of the temperature/pressure channels. + The pressure column (named "Pressure (Pa)") should be present, if not + raise an error listing the expected column name that is absent + :param base_altitude: H_b; Height at the bottom of atmospheric layer in + meters [m] + :param base_press: P_b; static pressure (pressure at sea level) in Pascals + [Pa]. If set to None, use the first pressure measurement + :param base_temp: T_b standard temperature (temperature at sea level) in + Celsius [C]. If set to None and a Temperature column (named + "Temperature (C)") exists, use the first temperature value. If set to + None and no temperature column exists, error out + :param units: determines if altitude is represented in meters ('m') or feet + ('ft') + :returns: a pandas DataFrame with the same Index values as the + input, and a column of “Altitude (m)” or “Altitude (ft)” data. + """ + # Initial Check for Necessary Data + if "Pressure (Pa)" not in df.columns: + raise TypeError("'Pressure' column does not exist.") + + # Conversion Constants + C_K = 273.15 # Celsius to Kelvin Constant (C + 273.15 = K) + ft_m = 0.3048 # Feet to Meters Constant (ft * 0.3048 = m) & (m / 0.3048 = ft) + + # Constants + h_s = 11000 # Height at the Start of the Stratosphere (meters) + L_b = -0.0065 # Standard Temperature Lapse Rate [K/m] + g_0 = 9.80665 # Gravitational Acceleration Constant [m/s^2] + R = 8.31432 # Universal Gas Constant [N*m/mol*K] + M = 0.0289644 # Molar Mass of Earth's Air [kg/mol] + + # Constants from Parameters + press_col_index = df.columns.get_loc("Pressure (Pa)") + + if base_temp == None: + if "Temperature (C)" not in df.columns: + raise TypeError("Temp set to None with no existing temperature column.") + else: + temp_col_index = df.columns.get_loc("Temperature (C)") + T_b = df.iloc[0, temp_col_index] + C_K # Standard Temperature in Kelvin + else: + T_b = base_temp + C_K # Standard Temperature in Kelvin + + if base_press == None: + # Set to first pressure recording in the dataframe + P_b = df.iloc[0, press_col_index] # Static Pressure in Pascals + else: + P_b = base_press # Static Pressure in Pascals + + if units == "ft": + h_b = base_altitude * ft_m # Height at Bottom of Atmospheric Layer in Meters + else: + h_b = base_altitude # Height at Bottom of Atmospheric Layer in Meters + + # List of Altitudes to be added to Dataframe + altitude_column = [] + + # Pressure at base of stratosphere + stratosphere_pressure = (P_b * ((L_b * ((T_b / L_b) - h_b + h_s) / T_b) ** + ((-g_0 * M) / (R * L_b)))) + + # Calculate Altitude for the DataFrame + for index, row in df.iterrows(): + P = row["Pressure (Pa)"] + if P > stratosphere_pressure: + h = h_b + (T_b / L_b) * (((P / P_b) ** ((-R * L_b) / (g_0 * M))) - 1) + altitude_column.append(h) + else: + h = h_b + ((R * T_b * np.log(P / P_b)) / (-g_0 * M)) + altitude_column.append(h) + + # Convert Altitude back to feet if specified + if units == 'ft': + altitude_column = [x / ft_m for x in altitude_column] + # Add "Altitude (ft)" Column to Copy of Original Dataframe + alt_df = df.copy() + alt_df["Altitude (ft)"] = altitude_column + + # Add "Altitude (m)" Column to Copy of Original Dataframe + if units == 'm': + alt_df = df.copy() + alt_df["Altitude (m)"] = altitude_column + + # Return DataFrame with New Altitude Column + return alt_df diff --git a/tests/calc/csv_to_df/default_sea_lvl.csv b/tests/calc/csv_to_df/default_sea_lvl.csv new file mode 100644 index 00000000..75cb9874 --- /dev/null +++ b/tests/calc/csv_to_df/default_sea_lvl.csv @@ -0,0 +1,18 @@ +Pressure (Pa),Temperature (C) +101325,15 +100000, +95000, +90000, +85000, +80000, +75000, +70000, +65000, +60000, +55000, +50000, +45000, +40000, +35000, +30000, +25000, diff --git a/tests/calc/test_utils.py b/tests/calc/test_utils.py index b012c89e..effb44cc 100644 --- a/tests/calc/test_utils.py +++ b/tests/calc/test_utils.py @@ -5,6 +5,10 @@ import numpy as np import pandas as pd +import sys +import os +sys.path.insert(0, os.path.realpath(os.path.join(__file__, '..', '..', '..'))) + from endaq.calc import utils @@ -107,3 +111,67 @@ def test_convert_units(): df = pd.DataFrame({'Val': [-40, 0, 10]}) np.testing.assert_allclose(utils.convert_units('degC', 'degF', df).Val[0], -40) np.testing.assert_allclose(utils.convert_units('degC', 'degF', df).Val[1], 32) + + +def test_to_altitude(): + """ + Tests the accuracy of the to_altitude function, which converts air pressure + to altitude. These tests only cover measurements BELOW the stratosphere. + """ + # Pressure Data CSV File --> DataFrame + df = pd.read_csv("./csv_to_df/default_sea_lvl.csv") + + # Meters + # DataFrame 1; Default settings: + def_key = [0.00, 110.88, 540.34, 988.50, 1457.30, 1948.99, 2466.23, + 3012.18, 3590.69, 4206.43, 4865.22, 5574.44, 6343.62, 7185.44, + 8117.27, 9163.96, 10362.95] + default_df = utils.to_altitude(df=df) + altitude_list_default = default_df['Altitude (m)'].tolist() + altitude_list_default = [round(num, 2) for num in altitude_list_default] + assert (altitude_list_default == def_key + ), "Equation is not accurate with default settings." + + + # DataFrame 2; Different base temperature: + temp_key = [0.00, 116.66, 568.47, 1039.96, 1533.16, 2050.45, 2594.61, + 3168.99, 3777.61, 4425.40, 5118.48, 5864.62, 6673.85, 7559.48, + 8539.82, 9641.00, 10902.40] + diff_temp_df = utils.to_altitude(df=df, base_temp=30) + altitude_list_diff_temp = diff_temp_df['Altitude (m)'].tolist() + altitude_list_diff_temp = [round(num, 2) for num in altitude_list_diff_temp] + assert (altitude_list_diff_temp == temp_key + ), "Equation is not accurate with non-default base temperature." + + # DataFrame 3: Different base pressure: + press_key = [-111.16, 0, 430.53, 879.82, 1349.79, 1842.71, 2361.25, 2908.57, + 3488.53, 4105.81, 4766.26, 5477.25, 6248.37, 7092.29, 8026.46, + 9075.77, 10277.77] + diff_press_df = utils.to_altitude(df=df, base_press=100000) + altitude_list_diff_press = diff_press_df['Altitude (m)'].tolist() + altitude_list_diff_press = [round(num, 2) for num in altitude_list_diff_press] + assert (altitude_list_diff_press == press_key + ), "Equation is not accurate with non-default base pressure." + + # DataFrame 4: Different base temperature and pressure: + temp_press_key = [-116.95, 0.00, 452.94, 925.62, 1420.06, 1938.64, 2484.17, + 3059.98, 3670.13, 4319.54, 5014.37, 5762.38, 6573.63, + 7461.49, 8444.29, 9548.22, 10812.79] + diff_temp_and_press_df = utils.to_altitude(df=df, base_temp=30, + base_press=100000) + altitude_list_diff_temp_press = diff_temp_and_press_df['Altitude (m)'].tolist() + altitude_list_diff_temp_press = [round(num, 2) for num in + altitude_list_diff_temp_press] + assert (altitude_list_diff_temp_press == temp_press_key + ), "Equation is not accurate with non-default base temperature and pressure." + + # Feet --> Meters --> Feet + # DataFrame 1; Default settings; Units = Feet: + def_key = [0.00, 363.79, 1772.76, 3243.11, 4781.17, 6394.32, 8091.29, + 9882.49, 11780.47, 13800.61, 15962.0, 18288.84, 20812.4, + 23574.27, 26631.46, 30065.48, 33999.16] + default_df = utils.to_altitude(df=df, units='ft') + altitude_list_default = default_df['Altitude (ft)'].tolist() + altitude_list_default = [round(num, 2) for num in altitude_list_default] + assert (altitude_list_default == def_key + ), "Equation is not accurate for units='ft'." From dc1777f6587179dddc8ff9cadaabe82b6063b950 Mon Sep 17 00:00:00 2001 From: Elin O'Neill Date: Mon, 30 Jun 2025 08:39:20 -0400 Subject: [PATCH 03/16] Minor file path fix. --- tests/calc/test_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/calc/test_utils.py b/tests/calc/test_utils.py index effb44cc..a71e716f 100644 --- a/tests/calc/test_utils.py +++ b/tests/calc/test_utils.py @@ -119,7 +119,7 @@ def test_to_altitude(): to altitude. These tests only cover measurements BELOW the stratosphere. """ # Pressure Data CSV File --> DataFrame - df = pd.read_csv("./csv_to_df/default_sea_lvl.csv") + df = pd.read_csv("tests/calc/csv_to_df/default_sea_lvl.csv") # Meters # DataFrame 1; Default settings: From dc3c3c9b0a4ad9d171f60db4f89a6dacf305bfc9 Mon Sep 17 00:00:00 2001 From: Elin O'Neill Date: Mon, 30 Jun 2025 09:09:13 -0400 Subject: [PATCH 04/16] Equation fix. --- endaq/calc/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/endaq/calc/utils.py b/endaq/calc/utils.py index 17353800..51efb332 100644 --- a/endaq/calc/utils.py +++ b/endaq/calc/utils.py @@ -411,8 +411,8 @@ def to_altitude(df: pd.DataFrame, altitude_column = [] # Pressure at base of stratosphere - stratosphere_pressure = (P_b * ((L_b * ((T_b / L_b) - h_b + h_s) / T_b) ** - ((-g_0 * M) / (R * L_b)))) + stratosphere_pressure = (P_b * (1 + (L_b / T_b) * (h_s - h_b)) ** + ((-g_0 * M) / (R * L_b))) # Calculate Altitude for the DataFrame for index, row in df.iterrows(): From 0c75e7bf8668768c2ebdbd9f568739c9782a3a3b Mon Sep 17 00:00:00 2001 From: Elin O'Neill Date: Mon, 30 Jun 2025 09:43:43 -0400 Subject: [PATCH 05/16] Added edge-case tests --- tests/calc/csv_to_df/press_error.csv | 2 ++ tests/calc/csv_to_df/temp_error.csv | 18 ++++++++++++++++++ tests/calc/test_utils.py | 21 +++++++++++++++++++++ 3 files changed, 41 insertions(+) create mode 100644 tests/calc/csv_to_df/press_error.csv create mode 100644 tests/calc/csv_to_df/temp_error.csv diff --git a/tests/calc/csv_to_df/press_error.csv b/tests/calc/csv_to_df/press_error.csv new file mode 100644 index 00000000..6e4fcb58 --- /dev/null +++ b/tests/calc/csv_to_df/press_error.csv @@ -0,0 +1,2 @@ +Temperature (C) +15 \ No newline at end of file diff --git a/tests/calc/csv_to_df/temp_error.csv b/tests/calc/csv_to_df/temp_error.csv new file mode 100644 index 00000000..1e3223af --- /dev/null +++ b/tests/calc/csv_to_df/temp_error.csv @@ -0,0 +1,18 @@ +Pressure (Pa), +101325, +100000, +95000, +90000, +85000, +80000, +75000, +70000, +65000, +60000, +55000, +50000, +45000, +40000, +35000, +30000, +25000, diff --git a/tests/calc/test_utils.py b/tests/calc/test_utils.py index a71e716f..da86c551 100644 --- a/tests/calc/test_utils.py +++ b/tests/calc/test_utils.py @@ -175,3 +175,24 @@ def test_to_altitude(): altitude_list_default = [round(num, 2) for num in altitude_list_default] assert (altitude_list_default == def_key ), "Equation is not accurate for units='ft'." + + # Base Values = None + base_none_df = utils.to_altitude(df=df, base_press=None, base_temp=None) + altitude_list_none = base_none_df['Altitude (m)'].tolist() + altitude_list_none = [round(num, 2) for num in altitude_list_default] + assert (altitude_list_none == def_key + ), "Equation is not accurate with base settings = None." + + # Errors + temp_df = pd.read_csv("tests/calc/csv_to_df/temp_error.csv") + press_df = df = pd.read_csv("tests/calc/csv_to_df/press_error.csv") + + # No temperature column + with pytest.raises(TypeError) as exc_info: + test_to_altitude(df=temp_df, base_temp=None) + assert (exc_info.type == TypeError), "Error not raised for missing temperature column." + + # No pressure column + with pytest.raises(TypeError) as exc_info: + test_to_altitude(df=press_df, base_press=None) + assert (exc_info.type == TypeError), "Error not raised for missing pressure column." From a53f2a4a4465ca5f5e999e77438b1b3c22d3fb99 Mon Sep 17 00:00:00 2001 From: Elin O'Neill Date: Mon, 30 Jun 2025 10:56:46 -0400 Subject: [PATCH 06/16] Limited NumPy version requirement. --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c6f9a4a8..0d929e7c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ backports.cached-property; python_version<'3.8' ebmlite>=3.2.0 idelib>=3.2.8 jinja2 -numpy>=1.19.5 +numpy>=1.19.5, <2.3.0 pandas>=1.3 plotly>=5.3.1 pynmeagps From b2817e388959d42220449e9ce741b717f458c89c Mon Sep 17 00:00:00 2001 From: Elin O'Neill Date: Mon, 30 Jun 2025 11:09:54 -0400 Subject: [PATCH 07/16] Limiting NumPy version again. --- requirements.txt | 2 +- setup.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 0d929e7c..ac124b10 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ backports.cached-property; python_version<'3.8' ebmlite>=3.2.0 idelib>=3.2.8 jinja2 -numpy>=1.19.5, <2.3.0 +numpy>=1.19.5,<2.3.0 pandas>=1.3 plotly>=5.3.1 pynmeagps diff --git a/setup.py b/setup.py index 48bda63e..3e61e268 100644 --- a/setup.py +++ b/setup.py @@ -42,6 +42,7 @@ def get_version(rel_path): "pytest-cov", "pytest-xdist[psutil]", "sympy", + "numpy<=2.3.0" ] DOCS_REQUIRES = [ From 6ce2a49afdeac6ec24e5ec672f36e540ba297111 Mon Sep 17 00:00:00 2001 From: Elin O'Neill Date: Mon, 30 Jun 2025 11:25:35 -0400 Subject: [PATCH 08/16] Revision to NumPy limit. --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 3e61e268..e5965cbf 100644 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ def get_version(rel_path): "ebmlite>=3.2.0", "idelib>=3.2.8", "jinja2", - "numpy>=1.19.5", + "numpy>1.19.5,<=2.3.0", "pandas>=1.3", "plotly>=5.3.1", "pynmeagps", @@ -42,7 +42,6 @@ def get_version(rel_path): "pytest-cov", "pytest-xdist[psutil]", "sympy", - "numpy<=2.3.0" ] DOCS_REQUIRES = [ From 97ed61d2f6f934dc49ab7097cd12e4f1901c7711 Mon Sep 17 00:00:00 2001 From: Elin O'Neill Date: Mon, 30 Jun 2025 11:31:52 -0400 Subject: [PATCH 09/16] Revision to NumPy limit 2. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e5965cbf..812feced 100644 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ def get_version(rel_path): "ebmlite>=3.2.0", "idelib>=3.2.8", "jinja2", - "numpy>1.19.5,<=2.3.0", + "numpy>1.19.5,<2.3.0", "pandas>=1.3", "plotly>=5.3.1", "pynmeagps", From 153091f27762c1462e6be308961490cb380f9b5f Mon Sep 17 00:00:00 2001 From: Elin O'Neill Date: Mon, 30 Jun 2025 11:40:21 -0400 Subject: [PATCH 10/16] Revision to SciPy version limit. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 812feced..f5d686e8 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ def get_version(rel_path): "pynmeagps", "python-dotenv>=0.18.0", "requests>=2.25.1", - "scipy>=1.7.1", + "scipy>=1.7.1,<1.16.0", "pint>=0.18" ] From a9ba3291c65069fe70ec67a2c30055cd19f075c5 Mon Sep 17 00:00:00 2001 From: Elin O'Neill Date: Mon, 30 Jun 2025 12:09:26 -0400 Subject: [PATCH 11/16] Reverse NumPy changes. --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index ac124b10..c6f9a4a8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ backports.cached-property; python_version<'3.8' ebmlite>=3.2.0 idelib>=3.2.8 jinja2 -numpy>=1.19.5,<2.3.0 +numpy>=1.19.5 pandas>=1.3 plotly>=5.3.1 pynmeagps diff --git a/setup.py b/setup.py index f5d686e8..79243c55 100644 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ def get_version(rel_path): "ebmlite>=3.2.0", "idelib>=3.2.8", "jinja2", - "numpy>1.19.5,<2.3.0", + "numpy>1.19.5", "pandas>=1.3", "plotly>=5.3.1", "pynmeagps", From c71ebb051d6f119808f508eb10a63487ecb5c5d0 Mon Sep 17 00:00:00 2001 From: Elin O'Neill Date: Mon, 30 Jun 2025 15:05:51 -0400 Subject: [PATCH 12/16] Stratosphere equation and initial tests implemented. --- endaq/calc/utils.py | 33 ++++++++++++-------- tests/calc/csv_to_df/beyond_stratosphere.csv | 2 ++ tests/calc/csv_to_df/stratosphere.csv | 12 +++++++ tests/calc/test_utils.py | 28 +++++++++++++---- 4 files changed, 56 insertions(+), 19 deletions(-) create mode 100644 tests/calc/csv_to_df/beyond_stratosphere.csv create mode 100644 tests/calc/csv_to_df/stratosphere.csv diff --git a/endaq/calc/utils.py b/endaq/calc/utils.py index 51efb332..04a4e680 100644 --- a/endaq/calc/utils.py +++ b/endaq/calc/utils.py @@ -351,17 +351,17 @@ def to_altitude(df: pd.DataFrame, base_temp: Optional[float] = 15, units: Literal['m', 'ft'] = 'm') -> pd.DataFrame: """ - Converts pressure (Pascals) to altitude (feet or meters). + Converts pressure (Pascals) to altitude (feet or meters) up to 50km. :param df: pandas DataFrame of one of the temperature/pressure channels. The pressure column (named "Pressure (Pa)") should be present, if not raise an error listing the expected column name that is absent - :param base_altitude: H_b; Height at the bottom of atmospheric layer in - meters [m] + :param base_altitude: h_b; Height at the bottom of atmospheric layer in + meters (m) :param base_press: P_b; static pressure (pressure at sea level) in Pascals - [Pa]. If set to None, use the first pressure measurement + (Pa). If set to None, use the first pressure measurement :param base_temp: T_b standard temperature (temperature at sea level) in - Celsius [C]. If set to None and a Temperature column (named + Celsius (C). If set to None and a Temperature column (named "Temperature (C)") exists, use the first temperature value. If set to None and no temperature column exists, error out :param units: determines if altitude is represented in meters ('m') or feet @@ -378,11 +378,13 @@ def to_altitude(df: pd.DataFrame, ft_m = 0.3048 # Feet to Meters Constant (ft * 0.3048 = m) & (m / 0.3048 = ft) # Constants - h_s = 11000 # Height at the Start of the Stratosphere (meters) + h_sb = 11000 # Height at the Base of the Stratosphere (meters) + h_st = 50000 # Height at the Top of the Stratosphere (meters) L_b = -0.0065 # Standard Temperature Lapse Rate [K/m] g_0 = 9.80665 # Gravitational Acceleration Constant [m/s^2] R = 8.31432 # Universal Gas Constant [N*m/mol*K] M = 0.0289644 # Molar Mass of Earth's Air [kg/mol] + top_stratosphere_pressure = 100 # Air Pressure at Stratopause (1mb)=[100 Pa] # Constants from Parameters press_col_index = df.columns.get_loc("Pressure (Pa)") @@ -393,8 +395,10 @@ def to_altitude(df: pd.DataFrame, else: temp_col_index = df.columns.get_loc("Temperature (C)") T_b = df.iloc[0, temp_col_index] + C_K # Standard Temperature in Kelvin + T_bS = T_b -71.5 # Temperature at start of Stratosphere else: T_b = base_temp + C_K # Standard Temperature in Kelvin + T_bS = T_b -71.5 # Temperature at start of Stratosphere if base_press == None: # Set to first pressure recording in the dataframe @@ -407,22 +411,25 @@ def to_altitude(df: pd.DataFrame, else: h_b = base_altitude # Height at Bottom of Atmospheric Layer in Meters + # Pressure at base of Stratosphere + base_stratosphere_pressure = (P_b * (1 + (L_b / T_b) * (h_sb - h_b)) ** + ((-g_0 * M) / (R * L_b))) + # List of Altitudes to be added to Dataframe altitude_column = [] - # Pressure at base of stratosphere - stratosphere_pressure = (P_b * (1 + (L_b / T_b) * (h_s - h_b)) ** - ((-g_0 * M) / (R * L_b))) - # Calculate Altitude for the DataFrame for index, row in df.iterrows(): P = row["Pressure (Pa)"] - if P > stratosphere_pressure: + if P > base_stratosphere_pressure: h = h_b + (T_b / L_b) * (((P / P_b) ** ((-R * L_b) / (g_0 * M))) - 1) altitude_column.append(h) - else: - h = h_b + ((R * T_b * np.log(P / P_b)) / (-g_0 * M)) + elif top_stratosphere_pressure < P < base_stratosphere_pressure: + h = h_sb + ((R * T_bS * np.log(P / base_stratosphere_pressure)) / + (-g_0 * M)) altitude_column.append(h) + else: + raise ValueError("Altitudes above stratosphere not supported.") # Convert Altitude back to feet if specified if units == 'ft': diff --git a/tests/calc/csv_to_df/beyond_stratosphere.csv b/tests/calc/csv_to_df/beyond_stratosphere.csv new file mode 100644 index 00000000..53731f09 --- /dev/null +++ b/tests/calc/csv_to_df/beyond_stratosphere.csv @@ -0,0 +1,2 @@ +Pressure (Pa),Temperature (C) +25,15 diff --git a/tests/calc/csv_to_df/stratosphere.csv b/tests/calc/csv_to_df/stratosphere.csv new file mode 100644 index 00000000..55600fb3 --- /dev/null +++ b/tests/calc/csv_to_df/stratosphere.csv @@ -0,0 +1,12 @@ +Pressure (Pa),Temperature (C) +20000,-55 +18000 +16000 +14000 +12000 +10000 +8000 +6000 +4000 +2000 +500 \ No newline at end of file diff --git a/tests/calc/test_utils.py b/tests/calc/test_utils.py index da86c551..6d801a42 100644 --- a/tests/calc/test_utils.py +++ b/tests/calc/test_utils.py @@ -167,32 +167,48 @@ def test_to_altitude(): # Feet --> Meters --> Feet # DataFrame 1; Default settings; Units = Feet: - def_key = [0.00, 363.79, 1772.76, 3243.11, 4781.17, 6394.32, 8091.29, + def_ft_key = [0.00, 363.79, 1772.76, 3243.11, 4781.17, 6394.32, 8091.29, 9882.49, 11780.47, 13800.61, 15962.0, 18288.84, 20812.4, 23574.27, 26631.46, 30065.48, 33999.16] default_df = utils.to_altitude(df=df, units='ft') altitude_list_default = default_df['Altitude (ft)'].tolist() altitude_list_default = [round(num, 2) for num in altitude_list_default] - assert (altitude_list_default == def_key + assert (altitude_list_default == def_ft_key ), "Equation is not accurate for units='ft'." # Base Values = None base_none_df = utils.to_altitude(df=df, base_press=None, base_temp=None) altitude_list_none = base_none_df['Altitude (m)'].tolist() - altitude_list_none = [round(num, 2) for num in altitude_list_default] + altitude_list_none = [round(num, 2) for num in altitude_list_none] assert (altitude_list_none == def_key ), "Equation is not accurate with base settings = None." - # Errors + # Error test dfs temp_df = pd.read_csv("tests/calc/csv_to_df/temp_error.csv") press_df = df = pd.read_csv("tests/calc/csv_to_df/press_error.csv") + beyond_strat_df = pd.read_csv("tests/calc/csv_to_df/beyond_stratosphere.csv") # No temperature column with pytest.raises(TypeError) as exc_info: - test_to_altitude(df=temp_df, base_temp=None) + utils.to_altitude(df=temp_df, base_temp=None) assert (exc_info.type == TypeError), "Error not raised for missing temperature column." # No pressure column with pytest.raises(TypeError) as exc_info: - test_to_altitude(df=press_df, base_press=None) + utils.to_altitude(df=press_df, base_press=None) assert (exc_info.type == TypeError), "Error not raised for missing pressure column." + + # Beyond stratosphere (50km) + with pytest.raises(ValueError) as exc_info: + utils.to_altitude(df=beyond_strat_df) + assert(exc_info.type == ValueError), "Error not raised when above stratosphere." + + # Stratosphere Tests + df_2 = pd.read_csv("tests/calc/csv_to_df/stratosphere.csv") + strat_key = [11784.05, 12452.21, 13199.14, 14045.95, 15023.51, 16179.72, + 17594.82, 19419.19, 21990.49, 26386.17, 35177.52] + strat_df = utils.to_altitude(df=df_2) + altitude_list_strat = strat_df['Altitude (m)'].tolist() + altitude_list_strat = [round(num, 2) for num in altitude_list_strat] + assert (altitude_list_strat == strat_key + ), "Equation is not accurate in the Stratosphere." From e09a3e386a9f539d5a5ddf37c26e109706519b87 Mon Sep 17 00:00:00 2001 From: Elin O'Neill Date: Mon, 7 Jul 2025 09:56:16 -0400 Subject: [PATCH 13/16] Resolve SciPy version issues --- setup.py | 2 +- tests/batch/test_core.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 79243c55..bdf00d51 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ def get_version(rel_path): "pynmeagps", "python-dotenv>=0.18.0", "requests>=2.25.1", - "scipy>=1.7.1,<1.16.0", + "scipy>=1.7.1", "pint>=0.18" ] diff --git a/tests/batch/test_core.py b/tests/batch/test_core.py index 7a1a1360..f7561c3b 100644 --- a/tests/batch/test_core.py +++ b/tests/batch/test_core.py @@ -292,7 +292,7 @@ def assert_output_is_valid(output: endaq.batch.core.OutputStruct): @pytest.mark.filterwarnings("ignore:no acceleration channel in:UserWarning") @pytest.mark.filterwarnings( "ignore" - ":nperseg .* is greater than input length .*, using nperseg .*" + ":.*nperseg.* is greater than (signal|input) length.*, using nperseg .*" ":UserWarning" ) def test_aggregate_data(getdata_builder): From 3f06b8b4f64592e4156ee9fd0f9a8913337a509b Mon Sep 17 00:00:00 2001 From: Elin O'Neill Date: Tue, 22 Jul 2025 11:19:30 -0400 Subject: [PATCH 14/16] Implemented suggestions for to_altitude --- endaq/calc/utils.py | 74 ++++++++++++++++++++++++---------------- tests/calc/test_utils.py | 10 +++--- 2 files changed, 49 insertions(+), 35 deletions(-) diff --git a/endaq/calc/utils.py b/endaq/calc/utils.py index 04a4e680..c3b16ce2 100644 --- a/endaq/calc/utils.py +++ b/endaq/calc/utils.py @@ -346,38 +346,42 @@ def convert_units( def to_altitude(df: pd.DataFrame, - base_altitude: float = 0, base_press: Optional[float] = 101325, base_temp: Optional[float] = 15, + temp_col_index: Optional[int] = None, + press_col_index: Optional[int] = None, units: Literal['m', 'ft'] = 'm') -> pd.DataFrame: """ Converts pressure (Pascals) to altitude (feet or meters) up to 50km. :param df: pandas DataFrame of one of the temperature/pressure channels. - The pressure column (named "Pressure (Pa)") should be present, if not - raise an error listing the expected column name that is absent - :param base_altitude: h_b; Height at the bottom of atmospheric layer in - meters (m) - :param base_press: P_b; static pressure (pressure at sea level) in Pascals - (Pa). If set to None, use the first pressure measurement - :param base_temp: T_b standard temperature (temperature at sea level) in - Celsius (C). If set to None and a Temperature column (named - "Temperature (C)") exists, use the first temperature value. If set to - None and no temperature column exists, error out + A pressure column should be present, if not, raise an error + :param base_press: P_b; reference pressure (pressure at sea level) in + Pascals (Pa). If set to None, use the first pressure measurement + included in the dataframe (df) + :param base_temp: T_b reference temperature (temperature at sea level) in + Celsius (C). If set to None and a Temperature column exists in the + dataframe (df), use the first listed temperature value. If set to None + and no temperature column exists, error out + :param temp_col_index: the index (starting at 0) in the dataframe (df) of + the temperature column to pull data from. + :param press_col_index: the index (starting at 0) in the dataframe (df) of + the pressure column to pull data from :param units: determines if altitude is represented in meters ('m') or feet - ('ft') + ('ft') in both the input and output dataframes :returns: a pandas DataFrame with the same Index values as the - input, and a column of “Altitude (m)” or “Altitude (ft)” data. + input, and an added column of “Altitude (m)” or “Altitude (ft)” data """ - # Initial Check for Necessary Data - if "Pressure (Pa)" not in df.columns: - raise TypeError("'Pressure' column does not exist.") + # Column Name Placeholders + temp_col = None + press_col = None # Conversion Constants C_K = 273.15 # Celsius to Kelvin Constant (C + 273.15 = K) ft_m = 0.3048 # Feet to Meters Constant (ft * 0.3048 = m) & (m / 0.3048 = ft) # Constants + h_b = 0 # Reference Height of Sea Level (m) h_sb = 11000 # Height at the Base of the Stratosphere (meters) h_st = 50000 # Height at the Top of the Stratosphere (meters) L_b = -0.0065 # Standard Temperature Lapse Rate [K/m] @@ -386,31 +390,41 @@ def to_altitude(df: pd.DataFrame, M = 0.0289644 # Molar Mass of Earth's Air [kg/mol] top_stratosphere_pressure = 100 # Air Pressure at Stratopause (1mb)=[100 Pa] - # Constants from Parameters - press_col_index = df.columns.get_loc("Pressure (Pa)") + # Finding pressure column + if press_col_index is None: + for col in df.columns: + if "press" in col.lower(): + press_col = col + press_col_index = df.columns.get_loc(press_col) + break + if press_col is None: + raise ValueError("Pressure column not found.") + else: + press_col = df.columns[press_col_index] + # Checking for base temp and converting to K if base_temp == None: - if "Temperature (C)" not in df.columns: - raise TypeError("Temp set to None with no existing temperature column.") - else: - temp_col_index = df.columns.get_loc("Temperature (C)") - T_b = df.iloc[0, temp_col_index] + C_K # Standard Temperature in Kelvin - T_bS = T_b -71.5 # Temperature at start of Stratosphere + if temp_col_index is None: + for col in df.columns: + if "temp" in col.lower(): + temp_col = col + temp_col_index = df.columns.get_loc(temp_col) + break + if temp_col is None: + raise ValueError('Temperature column not found.') + T_b = df.iloc[0, temp_col_index] + C_K # Standard Temperature in Kelvin + T_bS = T_b -71.5 # Temperature at start of Stratosphere else: T_b = base_temp + C_K # Standard Temperature in Kelvin T_bS = T_b -71.5 # Temperature at start of Stratosphere + # Checking for base pressure if base_press == None: # Set to first pressure recording in the dataframe P_b = df.iloc[0, press_col_index] # Static Pressure in Pascals else: P_b = base_press # Static Pressure in Pascals - if units == "ft": - h_b = base_altitude * ft_m # Height at Bottom of Atmospheric Layer in Meters - else: - h_b = base_altitude # Height at Bottom of Atmospheric Layer in Meters - # Pressure at base of Stratosphere base_stratosphere_pressure = (P_b * (1 + (L_b / T_b) * (h_sb - h_b)) ** ((-g_0 * M) / (R * L_b))) @@ -420,7 +434,7 @@ def to_altitude(df: pd.DataFrame, # Calculate Altitude for the DataFrame for index, row in df.iterrows(): - P = row["Pressure (Pa)"] + P = row[press_col] if P > base_stratosphere_pressure: h = h_b + (T_b / L_b) * (((P / P_b) ** ((-R * L_b) / (g_0 * M))) - 1) altitude_column.append(h) diff --git a/tests/calc/test_utils.py b/tests/calc/test_utils.py index 6d801a42..03dd1043 100644 --- a/tests/calc/test_utils.py +++ b/tests/calc/test_utils.py @@ -116,7 +116,7 @@ def test_convert_units(): def test_to_altitude(): """ Tests the accuracy of the to_altitude function, which converts air pressure - to altitude. These tests only cover measurements BELOW the stratosphere. + to altitude. """ # Pressure Data CSV File --> DataFrame df = pd.read_csv("tests/calc/csv_to_df/default_sea_lvl.csv") @@ -189,14 +189,14 @@ def test_to_altitude(): beyond_strat_df = pd.read_csv("tests/calc/csv_to_df/beyond_stratosphere.csv") # No temperature column - with pytest.raises(TypeError) as exc_info: + with pytest.raises(ValueError) as exc_info: utils.to_altitude(df=temp_df, base_temp=None) - assert (exc_info.type == TypeError), "Error not raised for missing temperature column." + assert (exc_info.type == ValueError), "Error not raised for missing temperature column." # No pressure column - with pytest.raises(TypeError) as exc_info: + with pytest.raises(ValueError) as exc_info: utils.to_altitude(df=press_df, base_press=None) - assert (exc_info.type == TypeError), "Error not raised for missing pressure column." + assert (exc_info.type == ValueError), "Error not raised for missing pressure column." # Beyond stratosphere (50km) with pytest.raises(ValueError) as exc_info: From 5e1bc0533fccce4a221ecf2f295dfe517851e986 Mon Sep 17 00:00:00 2001 From: Elin O'Neill Date: Wed, 23 Jul 2025 12:29:31 -0400 Subject: [PATCH 15/16] Parameterized and split up test_to_altitude. --- .../calc/csv_to_df/keys/base_values_none.csv | 18 +++ .../calc/csv_to_df/keys/default_settings.csv | 18 +++ .../csv_to_df/keys/diff_base_pressure.csv | 18 +++ tests/calc/csv_to_df/keys/diff_base_temp.csv | 18 +++ .../csv_to_df/keys/diff_temp_and_pressure.csv | 18 +++ tests/calc/csv_to_df/keys/stratosphere.csv | 12 ++ tests/calc/csv_to_df/keys/units_feet.csv | 18 +++ tests/calc/test_utils.py | 145 ++++++------------ 8 files changed, 171 insertions(+), 94 deletions(-) create mode 100644 tests/calc/csv_to_df/keys/base_values_none.csv create mode 100644 tests/calc/csv_to_df/keys/default_settings.csv create mode 100644 tests/calc/csv_to_df/keys/diff_base_pressure.csv create mode 100644 tests/calc/csv_to_df/keys/diff_base_temp.csv create mode 100644 tests/calc/csv_to_df/keys/diff_temp_and_pressure.csv create mode 100644 tests/calc/csv_to_df/keys/stratosphere.csv create mode 100644 tests/calc/csv_to_df/keys/units_feet.csv diff --git a/tests/calc/csv_to_df/keys/base_values_none.csv b/tests/calc/csv_to_df/keys/base_values_none.csv new file mode 100644 index 00000000..0bc8d1f9 --- /dev/null +++ b/tests/calc/csv_to_df/keys/base_values_none.csv @@ -0,0 +1,18 @@ +Key,base settings = None +0.00 +110.88 +540.34 +988.50 +1457.30 +1948.99 +2466.23 +3012.18 +3590.69 +4206.43 +4865.22 +5574.44 +6343.62 +7185.44 +8117.27 +9163.96 +10362.95 \ No newline at end of file diff --git a/tests/calc/csv_to_df/keys/default_settings.csv b/tests/calc/csv_to_df/keys/default_settings.csv new file mode 100644 index 00000000..74291a5c --- /dev/null +++ b/tests/calc/csv_to_df/keys/default_settings.csv @@ -0,0 +1,18 @@ +Key,settings are default +0.00 +110.88 +540.34 +988.50 +1457.30 +1948.99 +2466.23 +3012.18 +3590.69 +4206.43 +4865.22 +5574.44 +6343.62 +7185.44 +8117.27 +9163.96 +10362.95 \ No newline at end of file diff --git a/tests/calc/csv_to_df/keys/diff_base_pressure.csv b/tests/calc/csv_to_df/keys/diff_base_pressure.csv new file mode 100644 index 00000000..486f9f32 --- /dev/null +++ b/tests/calc/csv_to_df/keys/diff_base_pressure.csv @@ -0,0 +1,18 @@ +Key,the base pressure is not set to the default value +-111.16 +0 +430.53 +879.82 +1349.79 +1842.71 +2361.25 +2908.57 +3488.53 +4105.81 +4766.26 +5477.25 +6248.37 +7092.29 +8026.46, +9075.77 +10277.77 \ No newline at end of file diff --git a/tests/calc/csv_to_df/keys/diff_base_temp.csv b/tests/calc/csv_to_df/keys/diff_base_temp.csv new file mode 100644 index 00000000..7d580eed --- /dev/null +++ b/tests/calc/csv_to_df/keys/diff_base_temp.csv @@ -0,0 +1,18 @@ +Key,the base temperature is not set to the default value +0.00 +116.66 +568.47 +1039.96 +1533.16 +2050.45 +2594.61 +3168.99 +3777.61 +4425.40 +5118.48 +5864.62 +6673.85 +7559.48 +8539.82 +9641.00 +10902.40 \ No newline at end of file diff --git a/tests/calc/csv_to_df/keys/diff_temp_and_pressure.csv b/tests/calc/csv_to_df/keys/diff_temp_and_pressure.csv new file mode 100644 index 00000000..2e72d291 --- /dev/null +++ b/tests/calc/csv_to_df/keys/diff_temp_and_pressure.csv @@ -0,0 +1,18 @@ +Key,the base temperature and pressure are not set to default values +-116.95 +0.00 +452.94 +925.62 +1420.06 +1938.64 +2484.17 +3059.98 +3670.13 +4319.54 +5014.37 +5762.38 +6573.63 +7461.49 +8444.29 +9548.22 +10812.79 \ No newline at end of file diff --git a/tests/calc/csv_to_df/keys/stratosphere.csv b/tests/calc/csv_to_df/keys/stratosphere.csv new file mode 100644 index 00000000..41bbcdf4 --- /dev/null +++ b/tests/calc/csv_to_df/keys/stratosphere.csv @@ -0,0 +1,12 @@ +Key,altitude is in the stratosphere +11784.05 +12452.21 +13199.14 +14045.95 +15023.51 +16179.72 +17594.82 +19419.19 +21990.49 +26386.17 +35177.52 \ No newline at end of file diff --git a/tests/calc/csv_to_df/keys/units_feet.csv b/tests/calc/csv_to_df/keys/units_feet.csv new file mode 100644 index 00000000..a250c2ac --- /dev/null +++ b/tests/calc/csv_to_df/keys/units_feet.csv @@ -0,0 +1,18 @@ +Key,units are set to feet ('ft') +0.00 +363.79 +1772.76 +3243.11 +4781.17 +6394.32 +8091.29 +9882.49 +11780.47 +13800.61 +15962.0 +18288.84 +20812.4, +23574.27 +26631.46 +30065.48 +33999.16 \ No newline at end of file diff --git a/tests/calc/test_utils.py b/tests/calc/test_utils.py index 03dd1043..b8a5ea9f 100644 --- a/tests/calc/test_utils.py +++ b/tests/calc/test_utils.py @@ -113,102 +113,59 @@ def test_convert_units(): np.testing.assert_allclose(utils.convert_units('degC', 'degF', df).Val[1], 32) -def test_to_altitude(): +# Dataframes for the following altitude test +df_1 = pd.read_csv("tests/calc/csv_to_df/default_sea_lvl.csv") +df_strat = pd.read_csv("tests/calc/csv_to_df/stratosphere.csv") + +@pytest.mark.parametrize("kwargs, key", [ + ({"df": df_1}, "default_settings.csv"), + ({"df": df_1, "base_temp": 30}, "diff_base_temp.csv"), + ({"df": df_1, "base_press": 100000}, "diff_base_pressure.csv"), + ({"df": df_1, "base_temp": 30, "base_press": 100000}, "diff_temp_and_pressure.csv"), + ({"df": df_1, "units":'ft'}, "units_feet.csv"), + ({"df": df_1, "base_press": None, "base_temp": None}, "base_values_none.csv"), + ({"df": df_strat}, "stratosphere.csv"), + ]) +def test_to_altitude(kwargs, key): """ Tests the accuracy of the to_altitude function, which converts air pressure to altitude. """ - # Pressure Data CSV File --> DataFrame - df = pd.read_csv("tests/calc/csv_to_df/default_sea_lvl.csv") - - # Meters - # DataFrame 1; Default settings: - def_key = [0.00, 110.88, 540.34, 988.50, 1457.30, 1948.99, 2466.23, - 3012.18, 3590.69, 4206.43, 4865.22, 5574.44, 6343.62, 7185.44, - 8117.27, 9163.96, 10362.95] - default_df = utils.to_altitude(df=df) - altitude_list_default = default_df['Altitude (m)'].tolist() - altitude_list_default = [round(num, 2) for num in altitude_list_default] - assert (altitude_list_default == def_key - ), "Equation is not accurate with default settings." - - - # DataFrame 2; Different base temperature: - temp_key = [0.00, 116.66, 568.47, 1039.96, 1533.16, 2050.45, 2594.61, - 3168.99, 3777.61, 4425.40, 5118.48, 5864.62, 6673.85, 7559.48, - 8539.82, 9641.00, 10902.40] - diff_temp_df = utils.to_altitude(df=df, base_temp=30) - altitude_list_diff_temp = diff_temp_df['Altitude (m)'].tolist() - altitude_list_diff_temp = [round(num, 2) for num in altitude_list_diff_temp] - assert (altitude_list_diff_temp == temp_key - ), "Equation is not accurate with non-default base temperature." - - # DataFrame 3: Different base pressure: - press_key = [-111.16, 0, 430.53, 879.82, 1349.79, 1842.71, 2361.25, 2908.57, - 3488.53, 4105.81, 4766.26, 5477.25, 6248.37, 7092.29, 8026.46, - 9075.77, 10277.77] - diff_press_df = utils.to_altitude(df=df, base_press=100000) - altitude_list_diff_press = diff_press_df['Altitude (m)'].tolist() - altitude_list_diff_press = [round(num, 2) for num in altitude_list_diff_press] - assert (altitude_list_diff_press == press_key - ), "Equation is not accurate with non-default base pressure." - - # DataFrame 4: Different base temperature and pressure: - temp_press_key = [-116.95, 0.00, 452.94, 925.62, 1420.06, 1938.64, 2484.17, - 3059.98, 3670.13, 4319.54, 5014.37, 5762.38, 6573.63, - 7461.49, 8444.29, 9548.22, 10812.79] - diff_temp_and_press_df = utils.to_altitude(df=df, base_temp=30, - base_press=100000) - altitude_list_diff_temp_press = diff_temp_and_press_df['Altitude (m)'].tolist() - altitude_list_diff_temp_press = [round(num, 2) for num in - altitude_list_diff_temp_press] - assert (altitude_list_diff_temp_press == temp_press_key - ), "Equation is not accurate with non-default base temperature and pressure." - - # Feet --> Meters --> Feet - # DataFrame 1; Default settings; Units = Feet: - def_ft_key = [0.00, 363.79, 1772.76, 3243.11, 4781.17, 6394.32, 8091.29, - 9882.49, 11780.47, 13800.61, 15962.0, 18288.84, 20812.4, - 23574.27, 26631.46, 30065.48, 33999.16] - default_df = utils.to_altitude(df=df, units='ft') - altitude_list_default = default_df['Altitude (ft)'].tolist() - altitude_list_default = [round(num, 2) for num in altitude_list_default] - assert (altitude_list_default == def_ft_key - ), "Equation is not accurate for units='ft'." - - # Base Values = None - base_none_df = utils.to_altitude(df=df, base_press=None, base_temp=None) - altitude_list_none = base_none_df['Altitude (m)'].tolist() - altitude_list_none = [round(num, 2) for num in altitude_list_none] - assert (altitude_list_none == def_key - ), "Equation is not accurate with base settings = None." - - # Error test dfs - temp_df = pd.read_csv("tests/calc/csv_to_df/temp_error.csv") - press_df = df = pd.read_csv("tests/calc/csv_to_df/press_error.csv") - beyond_strat_df = pd.read_csv("tests/calc/csv_to_df/beyond_stratosphere.csv") - - # No temperature column - with pytest.raises(ValueError) as exc_info: - utils.to_altitude(df=temp_df, base_temp=None) - assert (exc_info.type == ValueError), "Error not raised for missing temperature column." - - # No pressure column - with pytest.raises(ValueError) as exc_info: - utils.to_altitude(df=press_df, base_press=None) - assert (exc_info.type == ValueError), "Error not raised for missing pressure column." - - # Beyond stratosphere (50km) + possible_altitude_cols = ['Altitude (m)', 'Altitude (ft)'] + altitude_col = None + + key_df = pd.read_csv("tests/calc/csv_to_df/keys/" + key) + key_list = key_df['Key'].tolist() + + to_alt_df = utils.to_altitude(**kwargs) + for col in possible_altitude_cols: + if col in to_alt_df.columns: + altitude_col = col + break + + to_alt_list = to_alt_df[altitude_col].tolist() + to_alt_list = [round(num, 2) for num in to_alt_list] + assert (to_alt_list == key_list + ), f"Equation is not accurate when {key_df.columns[1]}." + + +# Dataframes for the following altitude error test +df_err_temp = pd.read_csv("tests/calc/csv_to_df/temp_error.csv") +df_err_press = pd.read_csv("tests/calc/csv_to_df/press_error.csv") +df_err_strat = pd.read_csv("tests/calc/csv_to_df/beyond_stratosphere.csv") + +@pytest.mark.parametrize("kwargs, error_message", [ + ({"df": df_err_temp, "base_temp": None}, + "there's no temperature column and base_temp = None"), + ({"df": df_err_press, "base_press": None}, + "there's no pressure column and base_press = None"), + ({"df": df_err_strat}, + "altitude is above the stratosphere (50km)"), + ]) +def test_to_altitude_errors(kwargs, error_message): + """ + Tests that the to_altitude function raises errors when expected to. + """ with pytest.raises(ValueError) as exc_info: - utils.to_altitude(df=beyond_strat_df) - assert(exc_info.type == ValueError), "Error not raised when above stratosphere." - - # Stratosphere Tests - df_2 = pd.read_csv("tests/calc/csv_to_df/stratosphere.csv") - strat_key = [11784.05, 12452.21, 13199.14, 14045.95, 15023.51, 16179.72, - 17594.82, 19419.19, 21990.49, 26386.17, 35177.52] - strat_df = utils.to_altitude(df=df_2) - altitude_list_strat = strat_df['Altitude (m)'].tolist() - altitude_list_strat = [round(num, 2) for num in altitude_list_strat] - assert (altitude_list_strat == strat_key - ), "Equation is not accurate in the Stratosphere." + utils.to_altitude(**kwargs) + assert (exc_info.type == ValueError), f"Error not raised when {error_message}." From fd255fac26a2b6269ab57a5b8ab3147c3fbc8529 Mon Sep 17 00:00:00 2001 From: Jeff Smithwick Date: Thu, 8 Jan 2026 14:18:33 -0500 Subject: [PATCH 16/16] fixed resample's start and end times to be consistent with original --- endaq/calc/utils.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/endaq/calc/utils.py b/endaq/calc/utils.py index c3b16ce2..6bde9698 100644 --- a/endaq/calc/utils.py +++ b/endaq/calc/utils.py @@ -11,7 +11,6 @@ import scipy.signal import pint - def sample_spacing( data: Union[np.ndarray, pd.DataFrame], convert: typing.Literal[None, "to_seconds"] = "to_seconds", @@ -150,7 +149,11 @@ def resample(df: pd.DataFrame, sample_rate: Optional[float] = None) -> pd.DataFr df, num_samples_after_resampling, t=df.index.values.astype(np.float64), - ) + ) + resampled_time = pd.date_range( + df.iloc[0].name, df.iloc[-1].name, + periods=num_samples_after_resampling, + ) # Check for datetimes, if so localize if 'datetime' in str(df.index.dtype): @@ -158,9 +161,9 @@ def resample(df: pd.DataFrame, sample_rate: Optional[float] = None) -> pd.DataFr resampled_df = pd.DataFrame( resampled_data, - index=resampled_time.astype(df.index.dtype), + index=(resampled_time), columns=df.columns, - ) + ) resampled_df.index.name = df.index.name