diff --git a/environment.yml b/environment.yml deleted file mode 100644 index 9de09dc..0000000 --- a/environment.yml +++ /dev/null @@ -1,13 +0,0 @@ -name: noaaplotter -channels: - - conda-forge - - defaults -dependencies: - - matplotlib>=3.9 - - numpy>=2.2 - - pandas>=2.2 - - python>=3.10 - - requests - - joblib>=1.4 - - tqdm>=4.67 - - geemap>=0.35 diff --git a/examples/example_daily_series.py b/examples/example_daily_series.py index 2990dcc..4fe6395 100644 --- a/examples/example_daily_series.py +++ b/examples/example_daily_series.py @@ -6,7 +6,7 @@ author: Ingmar Nitze """ -from noaaplotter.noaaplotter import NOAAPlotter +from src.noaaplotter import NOAAPlotter import logging def main(): diff --git a/examples/example_daily_series_winter.py b/examples/example_daily_series_winter.py index cc7e510..27bb40a 100644 --- a/examples/example_daily_series_winter.py +++ b/examples/example_daily_series_winter.py @@ -6,7 +6,7 @@ author: Ingmar Nitze """ -from noaaplotter.noaaplotter import NOAAPlotter +from src.noaaplotter import NOAAPlotter import logging def main(): diff --git a/examples/example_monthly_series.py b/examples/example_monthly_series.py index e4eb25b..6e6048e 100644 --- a/examples/example_monthly_series.py +++ b/examples/example_monthly_series.py @@ -7,7 +7,7 @@ author: Ingmar Nitze """ -from noaaplotter.noaaplotter import NOAAPlotter +from src.noaaplotter import NOAAPlotter import logging def main(): diff --git a/noaaplotter/__init__.py b/noaaplotter/__init__.py index 3eb0b87..e69de29 100644 --- a/noaaplotter/__init__.py +++ b/noaaplotter/__init__.py @@ -1 +0,0 @@ -from noaaplotter import * \ No newline at end of file diff --git a/noaaplotter/download_utils.py b/noaaplotter/download_utils.py deleted file mode 100644 index b7c7389..0000000 --- a/noaaplotter/download_utils.py +++ /dev/null @@ -1,95 +0,0 @@ -import csv -import os -from datetime import datetime - -import numpy as np -import pandas as pd -import tqdm -from joblib import Parallel, delayed - -import geemap -import ee - -from noaaplotter.utils import dl_noaa_api, assign_numeric_datatypes - - -def download_from_noaa(output_file, start_date, end_date, datatypes, loc_name, station_id, noaa_api_token, n_jobs=4): - # remove file if exists - if os.path.exists(output_file): - os.remove(output_file) - # Make query string - dtypes_string = '&'.join([f'datatypeid={dt}' for dt in datatypes]) - # convert datestring to dt - dt_start = datetime.strptime(start_date, '%Y-%m-%d') - dt_end = datetime.strptime(end_date, '%Y-%m-%d') - # calculate number of days - n_days = (dt_end - dt_start).days - # calculate number of splits to fit into 1000 lines/rows - split_size = np.floor(1000 / len(datatypes)) - # calculate splits - split_range = np.arange(0, n_days, split_size) - # Data Loading - print('Downloading data through NOAA API') - datasets_list = Parallel(n_jobs=n_jobs)( - delayed(dl_noaa_api)(i, datatypes, station_id, noaa_api_token, start_date, end_date, split_size) - for i in tqdm.tqdm(split_range[:]) - ) - # drop empty/None from datasets_list - datasets_list = [i for i in datasets_list if i is not None] - - # Merge subsets and create DataFrame - df = pd.concat(datasets_list) - - df_pivot = assign_numeric_datatypes(df) - df_pivot['DATE'] = df_pivot.apply(lambda x: datetime.fromisoformat(x['DATE']).strftime('%Y-%m-%d'), axis=1) - - df_pivot = df_pivot.reset_index(drop=False) - dr = pd.DataFrame(pd.date_range(start=start_date, end=end_date), columns=['DATE']) - dr['DATE'] = dr['DATE'].astype(str) - df_merged = pd.concat([df_pivot.set_index('DATE'), dr.set_index('DATE')], join='outer', axis=1, - sort=True) - df_merged['DATE'] = df_merged.index - df_merged['NAME'] = loc_name - df_merged['TAVG'] = None - df_merged['SNWD'] = None - final_cols = ["STATION", "NAME", "DATE", "PRCP", "SNWD", "TAVG", "TMAX", "TMIN"] - df_final = df_merged[final_cols] - df_final = df_final.replace({np.nan: None}) - print(f'Saving data to {output_file}') - df_final.to_csv(output_file, index=False, quoting=csv.QUOTE_ALL) - return 0 - - -def download_era5_from_gee(latitude, longitude, end_date, start_date, output_file): - ee.Initialize() - EE_LAYER = 'ECMWF/ERA5/DAILY' - location = ee.Geometry.Point([longitude, latitude]) - # load ImageCollection - col = ee.ImageCollection(EE_LAYER).filterBounds(location).filterDate(start_date, end_date) - # Download data - print("Start downloading daily ERA5 data.") - print("Download may take a while.\n1yr: ~5 seconds\n10yrs: ~35 seconds\n50yrs: ~8 min") - result = geemap.extract_pixel_values(col, region=location) - out_dict = result.getInfo() - df_gee = pd.DataFrame(data=[out_dict.keys(), out_dict.values()]).T - # parse dates and values - df_gee['time'] = df_gee[0].apply(lambda x: f'{x[:4]}-{x[4:6]}-{x[6:8]}') - df_gee['feature'] = df_gee[0].apply(lambda x: x[9:]) - df_gee['value'] = df_gee[1] - df = df_gee.pivot_table(values='value', columns=['feature'], index='time') # .reset_index(drop=False) - # #### recalculate values - df_new = pd.DataFrame(index=df.index) - temperature_cols = ['mean_2m_air_temperature', 'minimum_2m_air_temperature', 'maximum_2m_air_temperature', - 'dewpoint_2m_temperature'] - precipitation_cols = ['total_precipitation'] - df_joined = df_new.join(df[temperature_cols] - 273.15).join(df[precipitation_cols] * 1e3).reset_index(drop=False) - # Create Output - rename_dict = {'time': 'DATE', 'total_precipitation': 'PRCP', 'mean_2m_air_temperature': 'TAVG', - 'maximum_2m_air_temperature': 'TMAX', 'minimum_2m_air_temperature': 'TMIN'} - df_renamed = df_joined.rename(columns=rename_dict) - df_renamed['NAME'] = '' - df_renamed['STATION'] = '' - df_renamed['SNWD'] = '' - output_cols = ["STATION", "NAME", "DATE", "PRCP", "SNWD", "TAVG", "TMAX", "TMIN"] - df_save = df_renamed[output_cols].astype(str) - df_save.to_csv(output_file, index=False) diff --git a/noaaplotter/noaaplotter.py b/noaaplotter/noaaplotter.py index 1cd726a..d9245d7 100644 --- a/noaaplotter/noaaplotter.py +++ b/noaaplotter/noaaplotter.py @@ -8,16 +8,16 @@ # version: 2021-09-06 import numpy as np +from matplotlib import dates ######################## -from matplotlib import pyplot as plt, dates -import matplotlib.dates as mdates - -from .dataset import NOAAPlotterDailyClimateDataset as DS_daily -from .dataset import NOAAPlotterDailySummariesDataset as Dataset -from .dataset import NOAAPlotterMonthlyClimateDataset as DS_monthly -from .plot_utils import * -from .utils import * +from matplotlib import pyplot as plt + +from noaaplotter.utils.dataset import NOAAPlotterDailyClimateDataset as DS_daily +from noaaplotter.utils.dataset import NOAAPlotterDailySummariesDataset as Dataset +from noaaplotter.utils.dataset import NOAAPlotterMonthlyClimateDataset as DS_monthly +from noaaplotter.utils.plot_utils import * +from noaaplotter.utils.utils import * pd.plotting.register_matplotlib_converters() numeric_only = True diff --git a/noaaplotter/scripts/__pycache__/download_data.cpython-310.pyc b/noaaplotter/scripts/__pycache__/download_data.cpython-310.pyc new file mode 100644 index 0000000..0173ecf Binary files /dev/null and b/noaaplotter/scripts/__pycache__/download_data.cpython-310.pyc differ diff --git a/noaaplotter/scripts/__pycache__/plot_daily.cpython-310.pyc b/noaaplotter/scripts/__pycache__/plot_daily.cpython-310.pyc new file mode 100644 index 0000000..27301b6 Binary files /dev/null and b/noaaplotter/scripts/__pycache__/plot_daily.cpython-310.pyc differ diff --git a/noaaplotter/scripts/download_data.py b/noaaplotter/scripts/download_data.py new file mode 100644 index 0000000..2c9b7ea --- /dev/null +++ b/noaaplotter/scripts/download_data.py @@ -0,0 +1,96 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# Imports +import argparse + +from noaaplotter.utils.download_utils import download_from_noaa + + +def main(): + """ + Main Function + :return: + """ + ##### Parse arguments ##### + parser = argparse.ArgumentParser(description="Parse arguments.") + + parser.add_argument( + "-o", + dest="output_file", + type=str, + required=True, + default="data/data.csv", + help="csv file to save results", + ) + + parser.add_argument( + "-t", dest="token", type=str, required=False, default="", help="NOAA API token" + ) + + parser.add_argument( + "-sid", + dest="station_id", + type=str, + required=False, + default="", + help='NOAA Station ID, e.g. "GHCND:USW00026616" for Kotzebue, only if loading through NOAA API', + ) + + parser.add_argument( + "-loc", + dest="loc_name", + type=str, + required=False, + default="", + help="Location name", + ) + + parser.add_argument( + "-dt", + dest="datatypes", + type=list, + required=False, + default=["TMIN", "TMAX", "PRCP", "SNOW"], + ) + + parser.add_argument( + "-start", + dest="start_date", + type=str, + required=True, + help='start date of plot ("yyyy-mm-dd")', + ) + + parser.add_argument( + "-end", + dest="end_date", + type=str, + required=True, + help='end date of plot ("yyyy-mm-dd")', + ) + + parser.add_argument( + "-n_jobs", + dest="n_jobs", + type=int, + required=False, + default=1, + help="number of parallel processes", + ) + + args = parser.parse_args() + + download_from_noaa( + output_file=args.output_file, + start_date=args.start_date, + end_date=args.end_date, + datatypes=args.datatypes, + noaa_api_token=args.token, + loc_name=args.loc_name, + station_id=args.station_id, + n_jobs=args.n_jobs, + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/download_data_ERA5.py b/noaaplotter/scripts/download_data_ERA5.py similarity index 96% rename from scripts/download_data_ERA5.py rename to noaaplotter/scripts/download_data_ERA5.py index ac96b46..f84400f 100644 --- a/scripts/download_data_ERA5.py +++ b/noaaplotter/scripts/download_data_ERA5.py @@ -4,7 +4,7 @@ import argparse import os -from noaaplotter.download_utils import download_era5_from_gee +from src.download_utils import download_era5_from_gee def main(): diff --git a/scripts/download_data_SST.py b/noaaplotter/scripts/download_data_SST.py similarity index 95% rename from scripts/download_data_SST.py rename to noaaplotter/scripts/download_data_SST.py index 06aa660..a51f8d0 100644 --- a/scripts/download_data_SST.py +++ b/noaaplotter/scripts/download_data_SST.py @@ -9,7 +9,7 @@ import pandas as pd import tqdm from joblib import delayed, Parallel -from noaaplotter.utils import dl_noaa_api +from noaaplotter.utils. import dl_noaa_api import ee import geemap diff --git a/scripts/plot_daily.py b/noaaplotter/scripts/plot_daily.py similarity index 100% rename from scripts/plot_daily.py rename to noaaplotter/scripts/plot_daily.py diff --git a/scripts/plot_monthly.py b/noaaplotter/scripts/plot_monthly.py similarity index 100% rename from scripts/plot_monthly.py rename to noaaplotter/scripts/plot_monthly.py diff --git a/noaaplotter/utils/__init__.py b/noaaplotter/utils/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/noaaplotter/utils/__pycache__/__init__.cpython-310.pyc b/noaaplotter/utils/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000..fa5ac97 Binary files /dev/null and b/noaaplotter/utils/__pycache__/__init__.cpython-310.pyc differ diff --git a/noaaplotter/utils/__pycache__/dataset.cpython-310.pyc b/noaaplotter/utils/__pycache__/dataset.cpython-310.pyc new file mode 100644 index 0000000..d331c29 Binary files /dev/null and b/noaaplotter/utils/__pycache__/dataset.cpython-310.pyc differ diff --git a/noaaplotter/utils/__pycache__/download_utils.cpython-310.pyc b/noaaplotter/utils/__pycache__/download_utils.cpython-310.pyc new file mode 100644 index 0000000..e14e75f Binary files /dev/null and b/noaaplotter/utils/__pycache__/download_utils.cpython-310.pyc differ diff --git a/noaaplotter/utils/__pycache__/plot_utils.cpython-310.pyc b/noaaplotter/utils/__pycache__/plot_utils.cpython-310.pyc new file mode 100644 index 0000000..3e4e88c Binary files /dev/null and b/noaaplotter/utils/__pycache__/plot_utils.cpython-310.pyc differ diff --git a/noaaplotter/utils/__pycache__/utils.cpython-310.pyc b/noaaplotter/utils/__pycache__/utils.cpython-310.pyc new file mode 100644 index 0000000..6bbb33e Binary files /dev/null and b/noaaplotter/utils/__pycache__/utils.cpython-310.pyc differ diff --git a/noaaplotter/dataset.py b/noaaplotter/utils/dataset.py old mode 100644 new mode 100755 similarity index 100% rename from noaaplotter/dataset.py rename to noaaplotter/utils/dataset.py diff --git a/noaaplotter/utils/download_utils.py b/noaaplotter/utils/download_utils.py new file mode 100755 index 0000000..39f8e98 --- /dev/null +++ b/noaaplotter/utils/download_utils.py @@ -0,0 +1,177 @@ +import csv +import datetime as dt +import json +import os +from datetime import datetime, timedelta + +import ee +import geemap +import numpy as np +import pandas as pd +import requests +import tqdm +from joblib import Parallel, delayed + +from noaaplotter.utils.utils import assign_numeric_datatypes + + +def download_from_noaa( + output_file, + start_date, + end_date, + datatypes, + loc_name, + station_id, + noaa_api_token, + n_jobs=4, +): + # remove file if exists + if os.path.exists(output_file): + os.remove(output_file) + # Make query string + dtypes_string = "&".join([f"datatypeid={dt}" for dt in datatypes]) + # convert datestring to dt + dt_start = datetime.strptime(start_date, "%Y-%m-%d") + dt_end = datetime.strptime(end_date, "%Y-%m-%d") + # calculate number of days + n_days = (dt_end - dt_start).days + # calculate number of splits to fit into 1000 lines/rows + split_size = np.floor(1000 / len(datatypes)) + # calculate splits + split_range = np.arange(0, n_days, split_size) + # Data Loading + print("Downloading data through NOAA API") + datasets_list = Parallel(n_jobs=n_jobs)( + delayed(dl_noaa_api)( + i, datatypes, station_id, noaa_api_token, start_date, end_date, split_size + ) + for i in tqdm.tqdm(split_range[:]) + ) + # drop empty/None from datasets_list + datasets_list = [i for i in datasets_list if i is not None] + + # Merge subsets and create DataFrame + df = pd.concat(datasets_list) + + df_pivot = assign_numeric_datatypes(df) + df_pivot["DATE"] = df_pivot.apply( + lambda x: datetime.fromisoformat(x["DATE"]).strftime("%Y-%m-%d"), axis=1 + ) + + df_pivot = df_pivot.reset_index(drop=False) + dr = pd.DataFrame(pd.date_range(start=start_date, end=end_date), columns=["DATE"]) + dr["DATE"] = dr["DATE"].astype(str) + df_merged = pd.concat( + [df_pivot.set_index("DATE"), dr.set_index("DATE")], + join="outer", + axis=1, + sort=True, + ) + df_merged["DATE"] = df_merged.index + df_merged["NAME"] = loc_name + df_merged["TAVG"] = None + df_merged["SNWD"] = None + final_cols = ["STATION", "NAME", "DATE", "PRCP", "SNWD", "TAVG", "TMAX", "TMIN"] + df_final = df_merged[final_cols] + df_final = df_final.replace({np.nan: None}) + print(f"Saving data to {output_file}") + df_final.to_csv(output_file, index=False, quoting=csv.QUOTE_ALL) + return 0 + + +def dl_noaa_api(i, dtypes, station_id, Token, date_start, date_end, split_size): + """ + function to download from NOAA API + """ + dt_start = dt.datetime.strptime(date_start, "%Y-%m-%d") + dt_end = dt.datetime.strptime(date_end, "%Y-%m-%d") + + split_start = dt_start + timedelta(days=i) + split_end = dt_start + timedelta(days=i + split_size - 1) + if split_end > dt_end: + split_end = dt_end + + date_start_split = split_start.strftime("%Y-%m-%d") + date_end_split = split_end.strftime("%Y-%m-%d") + + # make the api call + request_url = "https://www.ncei.noaa.gov/access/services/data/v1" + request_params = dict( + dataset="daily-summaries", + dataTypes=dtypes, # ['PRCP', 'TMIN', 'TMAX'], + stations=station_id, + limit=1000, + startDate=date_start_split, + endDate=date_end_split, + units="metric", + format="json", + ) + r = requests.get(request_url, params=request_params, headers={"token": Token}) + + # workaround to skip empty returns (no data within period) + try: + # load the api response as a json + d = json.loads(r.text) + result = pd.DataFrame(d) + except json.JSONDecodeError: + print( + f"Warning: No data available for period {date_start_split} to {date_end_split}. Skipping." + ) + result = None + return result + + +def download_era5_from_gee(latitude, longitude, end_date, start_date, output_file): + ee.Initialize() + EE_LAYER = "ECMWF/ERA5/DAILY" + location = ee.Geometry.Point([longitude, latitude]) + # load ImageCollection + col = ( + ee.ImageCollection(EE_LAYER) + .filterBounds(location) + .filterDate(start_date, end_date) + ) + # Download data + print("Start downloading daily ERA5 data.") + print( + "Download may take a while.\n1yr: ~5 seconds\n10yrs: ~35 seconds\n50yrs: ~8 min" + ) + result = geemap.extract_pixel_values(col, region=location) + out_dict = result.getInfo() + df_gee = pd.DataFrame(data=[out_dict.keys(), out_dict.values()]).T + # parse dates and values + df_gee["time"] = df_gee[0].apply(lambda x: f"{x[:4]}-{x[4:6]}-{x[6:8]}") + df_gee["feature"] = df_gee[0].apply(lambda x: x[9:]) + df_gee["value"] = df_gee[1] + df = df_gee.pivot_table( + values="value", columns=["feature"], index="time" + ) # .reset_index(drop=False) + # #### recalculate values + df_new = pd.DataFrame(index=df.index) + temperature_cols = [ + "mean_2m_air_temperature", + "minimum_2m_air_temperature", + "maximum_2m_air_temperature", + "dewpoint_2m_temperature", + ] + precipitation_cols = ["total_precipitation"] + df_joined = ( + df_new.join(df[temperature_cols] - 273.15) + .join(df[precipitation_cols] * 1e3) + .reset_index(drop=False) + ) + # Create Output + rename_dict = { + "time": "DATE", + "total_precipitation": "PRCP", + "mean_2m_air_temperature": "TAVG", + "maximum_2m_air_temperature": "TMAX", + "minimum_2m_air_temperature": "TMIN", + } + df_renamed = df_joined.rename(columns=rename_dict) + df_renamed["NAME"] = "" + df_renamed["STATION"] = "" + df_renamed["SNWD"] = "" + output_cols = ["STATION", "NAME", "DATE", "PRCP", "SNWD", "TAVG", "TMAX", "TMIN"] + df_save = df_renamed[output_cols].astype(str) + df_save.to_csv(output_file, index=False) diff --git a/noaaplotter/plot_utils.py b/noaaplotter/utils/plot_utils.py old mode 100644 new mode 100755 similarity index 97% rename from noaaplotter/plot_utils.py rename to noaaplotter/utils/plot_utils.py index 34ae3fc..08b591b --- a/noaaplotter/plot_utils.py +++ b/noaaplotter/utils/plot_utils.py @@ -1,49 +1,49 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- - -######################## -# Credits here -# author: Ingmar Nitze, Alfred Wegener Institute for Polar and Marine Research -# contact: ingmar.nitze@awi.de -# version: 2021-09-11 - -######################## - -# TODO: move to external file -def setup_monthly_plot_props(information, anomaly): - plot_kwargs = {} - if information == 'Temperature': - plot_kwargs['cmap'] = 'RdBu_r' - plot_kwargs['fc_low'] = '#4393c3' - plot_kwargs['fc_high'] = '#d6604d' - if anomaly: - plot_kwargs['value_column'] = 'tmean_diff' - plot_kwargs['y_label'] = 'Temperature departure [°C]' - plot_kwargs['title'] = 'Monthly departure from climatological mean (1981-2010)' - plot_kwargs['legend_label_above'] = 'Above average' - plot_kwargs['legend_label_below'] = 'Below average' - else: - plot_kwargs['value_column'] = 'tmean_doy_mean' - plot_kwargs['y_label'] = 'Temperature [°C]' - plot_kwargs['title'] = 'Monthly Mean Temperature' - plot_kwargs['legend_label_above'] = 'Above freezing' - plot_kwargs['legend_label_below'] = 'Below freezing' - - elif information == 'Precipitation': - plot_kwargs['fc_low'] = '#d6604d' - plot_kwargs['fc_high'] = '#4393c3' - if anomaly: - plot_kwargs['cmap'] = 'RdBu' - plot_kwargs['value_column'] = 'prcp_diff' - plot_kwargs['y_label'] = 'Precipitation departure [mm]' - plot_kwargs['title'] = 'Monthly departure from climatological mean (1981-2010)' - plot_kwargs['legend_label_above'] = 'Above average' - plot_kwargs['legend_label_below'] = 'Below average' - else: - plot_kwargs['cmap'] = 'Blues' - plot_kwargs['value_column'] = 'prcp_sum' - plot_kwargs['y_label'] = 'Precipitation [mm]' - plot_kwargs['title'] = 'Monthly Precipitation' - plot_kwargs['legend_label_below'] = '' - plot_kwargs['legend_label_above'] = 'Monthly Precipitation' +#!/usr/bin/python +# -*- coding: utf-8 -*- + +######################## +# Credits here +# author: Ingmar Nitze, Alfred Wegener Institute for Polar and Marine Research +# contact: ingmar.nitze@awi.de +# version: 2021-09-11 + +######################## + +# TODO: move to external file +def setup_monthly_plot_props(information, anomaly): + plot_kwargs = {} + if information == 'Temperature': + plot_kwargs['cmap'] = 'RdBu_r' + plot_kwargs['fc_low'] = '#4393c3' + plot_kwargs['fc_high'] = '#d6604d' + if anomaly: + plot_kwargs['value_column'] = 'tmean_diff' + plot_kwargs['y_label'] = 'Temperature departure [°C]' + plot_kwargs['title'] = 'Monthly departure from climatological mean (1981-2010)' + plot_kwargs['legend_label_above'] = 'Above average' + plot_kwargs['legend_label_below'] = 'Below average' + else: + plot_kwargs['value_column'] = 'tmean_doy_mean' + plot_kwargs['y_label'] = 'Temperature [°C]' + plot_kwargs['title'] = 'Monthly Mean Temperature' + plot_kwargs['legend_label_above'] = 'Above freezing' + plot_kwargs['legend_label_below'] = 'Below freezing' + + elif information == 'Precipitation': + plot_kwargs['fc_low'] = '#d6604d' + plot_kwargs['fc_high'] = '#4393c3' + if anomaly: + plot_kwargs['cmap'] = 'RdBu' + plot_kwargs['value_column'] = 'prcp_diff' + plot_kwargs['y_label'] = 'Precipitation departure [mm]' + plot_kwargs['title'] = 'Monthly departure from climatological mean (1981-2010)' + plot_kwargs['legend_label_above'] = 'Above average' + plot_kwargs['legend_label_below'] = 'Below average' + else: + plot_kwargs['cmap'] = 'Blues' + plot_kwargs['value_column'] = 'prcp_sum' + plot_kwargs['y_label'] = 'Precipitation [mm]' + plot_kwargs['title'] = 'Monthly Precipitation' + plot_kwargs['legend_label_below'] = '' + plot_kwargs['legend_label_above'] = 'Monthly Precipitation' return plot_kwargs \ No newline at end of file diff --git a/noaaplotter/utils.py b/noaaplotter/utils/utils.py old mode 100644 new mode 100755 similarity index 96% rename from noaaplotter/utils.py rename to noaaplotter/utils/utils.py index ee9367c..ad3b827 --- a/noaaplotter/utils.py +++ b/noaaplotter/utils/utils.py @@ -1,110 +1,110 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- - -######################## -# Credits here -# author: Ingmar Nitze, Alfred Wegener Institute for Polar and Marine Research -# contact: ingmar.nitze@awi.de -# version: 2020-12-09 - -######################## -import datetime as dt -from datetime import timedelta -import requests, json -import pandas as pd - - -#import datetime - - -def parse_dates(date): - """ - - :param date: - :return: - """ - if isinstance(date, str): - return dt.datetime.strptime(date, '%Y-%m-%d') - elif isinstance(date, dt.datetime) or isinstance(date, dt.date): - return date - else: - raise ('Wrong date format. Either use native datetime format or "YYYY-mm-dd"') - - -def calc_trailing_mean(df, length, feature, new_feature): - """ - :param df: - :param length: - :param feature: - :param new_feature: - :return: - - """ - df[new_feature] = df[feature].rolling(length).mean() - return df - - -def parse_dates_YM(date): - """ - :param date: - :return: - """ - if isinstance(date, str): - return dt.datetime.strptime(date, '%Y-%m') - elif isinstance(date, dt.datetime): - return date - else: - raise('Wrong date format. Either use native datetime format or "YYYY-mm-dd"') - - -def dl_noaa_api(i, dtypes, station_id, Token, date_start, date_end, split_size): - """ - function to download from NOAA API - """ - dt_start = dt.datetime.strptime(date_start, '%Y-%m-%d') - dt_end = dt.datetime.strptime(date_end, '%Y-%m-%d') - - split_start = dt_start + timedelta(days=i) - split_end = dt_start + timedelta(days=i + split_size - 1) - if split_end > dt_end: - split_end = dt_end - - date_start_split = split_start.strftime('%Y-%m-%d') - date_end_split = split_end.strftime('%Y-%m-%d') - - # make the api call - request_url = 'https://www.ncei.noaa.gov/access/services/data/v1' - request_params = dict( - dataset = 'daily-summaries', - dataTypes = dtypes,#['PRCP', 'TMIN', 'TMAX'], - stations = station_id, - limit = 1000, - startDate = date_start_split, - endDate= date_end_split, - units='metric', - format='json' - ) - r = requests.get( - request_url, - params=request_params, - headers={'token': Token}) - - # workaround to skip empty returns (no data within period) - try: - # load the api response as a json - d = json.loads(r.text) - result = pd.DataFrame(d) - except json.JSONDecodeError: - print(f"Warning: No data available for period {date_start_split} to {date_end_split}. Skipping.") - result = None - return result - - -def assign_numeric_datatypes(df): - for col in df.columns: - if df[col].dtype == 'object': - try: - df[col] = pd.to_numeric(df[col]) - except: - pass +#!/usr/bin/python +# -*- coding: utf-8 -*- + +######################## +# Credits here +# author: Ingmar Nitze, Alfred Wegener Institute for Polar and Marine Research +# contact: ingmar.nitze@awi.de +# version: 2020-12-09 + +######################## +import datetime as dt +from datetime import timedelta +import requests, json +import pandas as pd + + +#import datetime + + +def parse_dates(date): + """ + + :param date: + :return: + """ + if isinstance(date, str): + return dt.datetime.strptime(date, '%Y-%m-%d') + elif isinstance(date, dt.datetime) or isinstance(date, dt.date): + return date + else: + raise ('Wrong date format. Either use native datetime format or "YYYY-mm-dd"') + + +def calc_trailing_mean(df, length, feature, new_feature): + """ + :param df: + :param length: + :param feature: + :param new_feature: + :return: + + """ + df[new_feature] = df[feature].rolling(length).mean() + return df + + +def parse_dates_YM(date): + """ + :param date: + :return: + """ + if isinstance(date, str): + return dt.datetime.strptime(date, '%Y-%m') + elif isinstance(date, dt.datetime): + return date + else: + raise('Wrong date format. Either use native datetime format or "YYYY-mm-dd"') + + +def dl_noaa_api(i, dtypes, station_id, Token, date_start, date_end, split_size): + """ + function to download from NOAA API + """ + dt_start = dt.datetime.strptime(date_start, '%Y-%m-%d') + dt_end = dt.datetime.strptime(date_end, '%Y-%m-%d') + + split_start = dt_start + timedelta(days=i) + split_end = dt_start + timedelta(days=i + split_size - 1) + if split_end > dt_end: + split_end = dt_end + + date_start_split = split_start.strftime('%Y-%m-%d') + date_end_split = split_end.strftime('%Y-%m-%d') + + # make the api call + request_url = 'https://www.ncei.noaa.gov/access/services/data/v1' + request_params = dict( + dataset = 'daily-summaries', + dataTypes = dtypes,#['PRCP', 'TMIN', 'TMAX'], + stations = station_id, + limit = 1000, + startDate = date_start_split, + endDate= date_end_split, + units='metric', + format='json' + ) + r = requests.get( + request_url, + params=request_params, + headers={'token': Token}) + + # workaround to skip empty returns (no data within period) + try: + # load the api response as a json + d = json.loads(r.text) + result = pd.DataFrame(d) + except json.JSONDecodeError: + print(f"Warning: No data available for period {date_start_split} to {date_end_split}. Skipping.") + result = None + return result + + +def assign_numeric_datatypes(df): + for col in df.columns: + if df[col].dtype == 'object': + try: + df[col] = pd.to_numeric(df[col]) + except: + pass return df \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml new file mode 100755 index 0000000..b2d2c8a --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,34 @@ +[build-system] +requires = ["setuptools>=42", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "noaaplotter" +version = "0.5.3" +description = "Package to plot fancy climate/weather data of NOAA" +authors = [ + { name = "Ingmar Nitze", email = "ingmar.nitze@awi.de" } +] +license = { text = "" } +readme = "README.md" # Specify a README file if available +# homepage = "https://github.com/initze/noaaplotter" +keywords = ["climate", "weather", "NOAA", "plotting"] + +dependencies = [ + "pandas>=2.2", + "numpy>=2.2", + "matplotlib>=3.9", + "requests", + "joblib>=1.4", + "tqdm>=4.67", + "geemap>=0.35" +] + +[tool.setuptools.packages.find] +include = ["noaaplotter*"] + +[project.scripts] +plot_daily = "noaaplotter.scripts.plot_daily:main" # Adjust if necessary +plot_monthly = "noaaplotter.scripts.plot_monthly:main" # Adjust if necessary +download_data = "noaaplotter.scripts.download_data:main" # Adjust if necessary +download_data_ERA5 = "noaaplotter.scripts.download_data_ERA5:main" # Adjust if necessary diff --git a/scripts/download_data.py b/scripts/download_data.py deleted file mode 100644 index f459b21..0000000 --- a/scripts/download_data.py +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- -# Imports -import argparse -from noaaplotter.utils import dl_noaa_api -from noaaplotter.download_utils import download_from_noaa - - -def main(): - """ - Main Function - :return: - """ - ##### Parse arguments ##### - parser = argparse.ArgumentParser(description='Parse arguments.') - - parser.add_argument('-o', dest='output_file', type=str, required=True, - default='data/data.csv', - help='csv file to save results') - - parser.add_argument('-t', dest='token', type=str, required=False, - default='', - help='NOAA API token') - - parser.add_argument('-sid', dest='station_id', type=str, required=False, - default='', - help='NOAA Station ID, e.g. "GHCND:USW00026616" for Kotzebue, only if loading through NOAA API') - - parser.add_argument('-loc', dest='loc_name', type=str, required=False, - default='', - help='Location name') - - parser.add_argument('-dt', dest='datatypes', type=list, required=False, default=['TMIN', 'TMAX', 'PRCP', 'SNOW']) - - parser.add_argument('-start', dest='start_date', type=str, required=True, - help='start date of plot ("yyyy-mm-dd")') - - parser.add_argument('-end', dest='end_date', type=str, required=True, - help='end date of plot ("yyyy-mm-dd")') - - parser.add_argument('-n_jobs', dest='n_jobs', type=int, required=False, default=1, - help='number of parallel processes') - - args = parser.parse_args() - - download_from_noaa(output_file=args.output_file, start_date=args.start_date, end_date=args.end_date, \ - datatypes=args.datatypes, noaa_api_token=args.token, loc_name=args.loc_name, \ - station_id=args.station_id, n_jobs=args.n_jobs) - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/setup.py b/setup.py deleted file mode 100644 index 259b0b8..0000000 --- a/setup.py +++ /dev/null @@ -1,27 +0,0 @@ -from distutils.core import setup - -setup( - name='noaaplotter', - version='0.5.2', - packages=['noaaplotter'], - url='https://github.com/initze/noaaplotter', - license='', - author='Ingmar Nitze', - author_email='ingmar.nitze@awi.de', - description='Package to plot fancy climate/weather data of NOAA', - install_requires=[ - 'pandas>=2.2', - 'numpy>=2.2', - 'matplotlib>=3.9', - 'requests', - 'joblib>=1.4', - 'tqdm>=4.67', - 'geemap>=0.35' - ], - scripts=[ - 'scripts/plot_daily.py', - 'scripts/plot_monthly.py', - 'scripts/download_data.py', - 'scripts/download_data_ERA5.py' - ] -)