From 8ebc7f32782299e3abb2c9c35b6dfd4d93fdf732 Mon Sep 17 00:00:00 2001 From: Eric Novotny Date: Fri, 27 Feb 2026 10:18:42 -0800 Subject: [PATCH 1/3] fix multi timeseries get json command --- cwms/timeseries/timeseries.py | 45 ++++++++++++++++++++- pyproject.toml | 2 +- tests/cda/timeseries/timeseries_CDA_test.py | 28 +++++++++++++ 3 files changed, 73 insertions(+), 2 deletions(-) diff --git a/cwms/timeseries/timeseries.py b/cwms/timeseries/timeseries.py index 4e9b9757..3fe75ffb 100644 --- a/cwms/timeseries/timeseries.py +++ b/cwms/timeseries/timeseries.py @@ -1,6 +1,7 @@ import concurrent.futures import logging from datetime import datetime, timedelta, timezone +from modulefinder import test from typing import Any, Dict, List, Optional, Tuple import pandas as pd @@ -228,8 +229,18 @@ def combine_timeseries_results(results: List[Data]) -> Data: combined_json["end"] = combined_df["date-time"].max().isoformat() combined_json["total"] = len(combined_df) + # make sure that dataTime column is in iso8601 formate. + # combined_df["date-time"] = pd.to_datetime(combined_df["date-time"], utc=True).apply( + # pd.Timestamp.isoformat + # ) + + combined_df["date-time"] = combined_df["date-time"].apply( + lambda x: int(pd.Timestamp(x).timestamp() * 1000) + ) + combined_df["date-time"] = combined_df["date-time"].astype("Int64") + combined_df = combined_df.reindex(columns=["date-time", "value", "quality-code"]) # Update the "values" key in the JSON to include the combined data - combined_json["values"] = combined_df.to_dict(orient="records") + combined_json["values"] = combined_df.values.tolist() # Return a new cwms Data object with the combined DataFrame and updated metadata return Data(combined_json, selector="values") @@ -455,6 +466,32 @@ def store_multi_timeseries_df( office_id: str, max_workers: Optional[int] = 30, ) -> None: + """stored mulitple timeseries from a dataframe. The dataframe must be a metled dataframe with columns + for date-time, value, quality-code(optional), ts_id, units, and version_date(optional). The dataframe will + be grouped by ts_id and version_date and each group will be posted as a separate timeseries using the store_timeseries + function. If version_date column is not included then all data will be stored as unversioned data. If version_date + column is included then data will be grouped by ts_id and version_date and stored as versioned timeseries with the + version date specified in the version_date column. + + Parameters + ---------- + data: dataframe + Time Series data to be stored. Dataframe must be melted with columns for date-time, value, quality-code(optional), + ts_id, units, and version_date(optional). + date-time value quality-code ts_id units version_date + 0 2023-12-20T14:45:00.000-05:00 93.1 0 OMA.Stage.Inst.6Hours.0.Fcst-MRBWM-GRFT ft 2024-04-22 07:00:00-05:00 + 1 2023-12-20T15:00:00.000-05:00 99.8 0 OMA.Stage.Inst.6Hours.0.Fcst-MRBWM-GRFT ft 2024-04-22 07:00:00-05:00 + 2 2023-12-20T15:15:00.000-05:00 98.5 0 OMA.Stage.Inst.6Hours.0.Fcst-MRBWM-GRFT ft 2024-04-22 07:15:00-05:00 + office_id: string + The owning office of the time series(s). + max_workers: Int, Optional, default is None + It is a number of Threads aka size of pool in concurrent.futures.ThreadPoolExecutor. + + Returns + ------- + None + """ + def store_ts_ids( data: pd.DataFrame, ts_id: str, @@ -476,6 +513,12 @@ def store_ts_ids( print(f"Error processing {ts_id}: {e}") return None + required_columns = ["date-time", "value", "ts_id", "units"] + for col in required_columns: + if col not in data.columns: + raise TypeError( + f"{col} is a required column in data when posting multiple timeseries from a dataframe. Make sure you are using a melted dataframe with columns for date-time, value, quality-code(optional), ts_id, units, and version_date(optional)." + ) ts_data_all = data.copy() if "version_date" not in ts_data_all.columns: ts_data_all = ts_data_all.assign(version_date=pd.to_datetime(pd.Series([]))) diff --git a/pyproject.toml b/pyproject.toml index bf9fbdce..20c9200f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,7 @@ name = "cwms-python" repository = "https://github.com/HydrologicEngineeringCenter/cwms-python" -version = "1.0.1" +version = "1.0.2" packages = [ diff --git a/tests/cda/timeseries/timeseries_CDA_test.py b/tests/cda/timeseries/timeseries_CDA_test.py index 1afe21a2..b21b848c 100644 --- a/tests/cda/timeseries/timeseries_CDA_test.py +++ b/tests/cda/timeseries/timeseries_CDA_test.py @@ -16,6 +16,7 @@ TEST_TSID_MULTI2 = f"{TEST_LOCATION_ID}.Stage.Inst.15Minutes.0.Raw-Multi-2" TEST_TSID_STORE = f"{TEST_LOCATION_ID}.Stage.Inst.15Minutes.0.Raw-Store" TEST_TSID_CHUNK_MULTI = f"{TEST_LOCATION_ID}.Stage.Inst.15Minutes.0.Raw-Multi-Chunk" +TEST_TSID_COPY = f"{TEST_LOCATION_ID}.Stage.Inst.15Minutes.0.Raw-Copy" TEST_TSID_DELETE = f"{TEST_LOCATION_ID}.Stage.Inst.15Minutes.0.Raw-Delete" TS_ID_REV_TEST = TEST_TSID_MULTI.replace("Raw-Multi", "Raw-Rev-Test") # Generate 15-minute interval timestamps @@ -266,6 +267,33 @@ def test_store_timeseries_chunk_ts(): ), f"Data frames do not match: original = {DF_CHUNK_MULTI.describe()}, stored = {df.describe()}" +def test_copy_timeseries_chunk_json(): + data_json = ts.get_timeseries( + ts_id=TEST_TSID_CHUNK_MULTI, + office_id=TEST_OFFICE, + begin=START_DATE_CHUNK_MULTI, + end=END_DATE_CHUNK_MULTI, + max_days_per_chunk=14, + unit="SI", + ).json + data_json["name"] = TEST_TSID_COPY + ts.store_timeseries(data_json) + + data_multithread = ts.get_timeseries( + ts_id=TEST_TSID_COPY, + office_id=TEST_OFFICE, + begin=START_DATE_CHUNK_MULTI, + end=END_DATE_CHUNK_MULTI, + max_days_per_chunk=14, + unit="SI", + ) + df = data_multithread.df + # make sure the dataframe matches stored dataframe + pdt.assert_frame_equal( + df, DF_CHUNK_MULTI + ), f"Data frames do not match: original = {DF_CHUNK_MULTI.describe()}, stored = {df.describe()}" + + def test_read_timeseries_chunk_ts(): # Capture the log output data_multithread = ts.get_timeseries( From 948dd479e0cb99d9b8d068baa326c40d6e5bd95a Mon Sep 17 00:00:00 2001 From: Eric Novotny Date: Fri, 27 Feb 2026 11:30:40 -0800 Subject: [PATCH 2/3] update tests --- cwms/timeseries/timeseries.py | 6 --- tests/cda/locations/location_groups_test.py | 11 ++-- tests/cda/timeseries/timeseries_CDA_test.py | 58 +++++++++++---------- 3 files changed, 37 insertions(+), 38 deletions(-) diff --git a/cwms/timeseries/timeseries.py b/cwms/timeseries/timeseries.py index 3fe75ffb..6fdc4077 100644 --- a/cwms/timeseries/timeseries.py +++ b/cwms/timeseries/timeseries.py @@ -1,7 +1,6 @@ import concurrent.futures import logging from datetime import datetime, timedelta, timezone -from modulefinder import test from typing import Any, Dict, List, Optional, Tuple import pandas as pd @@ -229,11 +228,6 @@ def combine_timeseries_results(results: List[Data]) -> Data: combined_json["end"] = combined_df["date-time"].max().isoformat() combined_json["total"] = len(combined_df) - # make sure that dataTime column is in iso8601 formate. - # combined_df["date-time"] = pd.to_datetime(combined_df["date-time"], utc=True).apply( - # pd.Timestamp.isoformat - # ) - combined_df["date-time"] = combined_df["date-time"].apply( lambda x: int(pd.Timestamp(x).timestamp() * 1000) ) diff --git a/tests/cda/locations/location_groups_test.py b/tests/cda/locations/location_groups_test.py index a5aa17e9..08a284f1 100644 --- a/tests/cda/locations/location_groups_test.py +++ b/tests/cda/locations/location_groups_test.py @@ -42,7 +42,6 @@ # Setup and teardown fixture for test location @pytest.fixture(scope="module", autouse=True) def setup_data(): - TEST_LATITUDE = 45.1704758 TEST_LONGITUDE = -92.8411439 @@ -68,9 +67,12 @@ def setup_data(): yield # Delete location and TS after tests - cwms.delete_location( - location_id=TEST_LOCATION_ID, office_id=TEST_OFFICE, cascade_delete=True - ) + try: + cwms.delete_location( + location_id=TEST_LOCATION_ID, office_id=TEST_OFFICE, cascade_delete=True + ) + except Exception as e: + print(f"Failed to delete location {TEST_LOCATION_ID}: {e}") @pytest.fixture(autouse=True) @@ -79,7 +81,6 @@ def init_session(request): def test_store_location_group(): - lg.store_location_groups(data=LOC_GROUP_DATA) data = lg.get_location_group( loc_group_id=TEST_GROUP_ID, diff --git a/tests/cda/timeseries/timeseries_CDA_test.py b/tests/cda/timeseries/timeseries_CDA_test.py index b21b848c..2c31abac 100644 --- a/tests/cda/timeseries/timeseries_CDA_test.py +++ b/tests/cda/timeseries/timeseries_CDA_test.py @@ -9,7 +9,7 @@ import cwms.timeseries.timeseries as ts TEST_OFFICE = "MVP" -TEST_LOCATION_ID = "pytest_group" +TEST_LOCATION_ID = "pytest_ts" TEST_TSID = f"{TEST_LOCATION_ID}.Stage.Inst.15Minutes.0.Raw-Test" TEST_TSID_MULTI = f"{TEST_LOCATION_ID}.Stage.Inst.15Minutes.0.Raw-Multi" TEST_TSID_MULTI1 = f"{TEST_LOCATION_ID}.Stage.Inst.15Minutes.0.Raw-Multi-1" @@ -30,6 +30,7 @@ TEST_TSID_MULTI2, TEST_TSID_STORE, TEST_TSID_CHUNK_MULTI, + TEST_TSID_COPY, ] @@ -72,6 +73,11 @@ [DF_MULTI_TIMESERIES1, DF_MULTI_TIMESERIES2] ).reset_index(drop=True) +DT = datetime(2023, 1, 1, 0, 0, tzinfo=timezone.utc) +EPOCH_MS = int(DT.timestamp() * 1000) +BEGIN = DT - timedelta(minutes=5) +END = DT + timedelta(minutes=5) + @pytest.fixture(scope="module", autouse=True) def setup_data(): @@ -92,6 +98,13 @@ def setup_data(): } cwms.store_location(location) + ts_json = { + "name": TEST_TSID_DELETE, + "office-id": TEST_OFFICE, + "units": "ft", + "values": [[EPOCH_MS, 99, 0]], + } + ts.store_timeseries(ts_json) yield for ts_id in TSIDS: try: @@ -109,21 +122,14 @@ def init_session(): def test_store_timeseries(): - now = datetime.now(timezone.utc).replace(microsecond=0) - now_epoch_ms = int(now.timestamp() * 1000) - iso_now = now.isoformat() ts_json = { "name": TEST_TSID_STORE, "office-id": TEST_OFFICE, "units": "ft", - "values": [[now_epoch_ms, 99, 0]], - "begin": iso_now, - "end": iso_now, - "version-date": iso_now, - "time-zone": "UTC", + "values": [[EPOCH_MS, 99, 0]], } ts.store_timeseries(ts_json) - data = ts.get_timeseries(TEST_TSID_STORE, TEST_OFFICE).json + data = ts.get_timeseries(TEST_TSID_STORE, TEST_OFFICE, begin=BEGIN, end=END).json assert data["name"] == TEST_TSID_STORE assert data["office-id"] == TEST_OFFICE assert data["units"] == "ft" @@ -131,7 +137,7 @@ def test_store_timeseries(): def test_get_timeseries(): - data = ts.get_timeseries(TEST_TSID_STORE, TEST_OFFICE).json + data = ts.get_timeseries(TEST_TSID_STORE, TEST_OFFICE, begin=BEGIN, end=END).json assert data["name"] == TEST_TSID_STORE assert data["office-id"] == TEST_OFFICE assert data["units"] == "ft" @@ -139,10 +145,9 @@ def test_get_timeseries(): def test_timeseries_df_to_json(): - dt = datetime(2023, 1, 1, 0, 0, tzinfo=timezone.utc) df = pd.DataFrame( { - "date-time": [dt], + "date-time": [DT], "value": [42], "quality-code": [0], } @@ -155,16 +160,14 @@ def test_timeseries_df_to_json(): assert json_out["office-id"] == office, "Incorrect office-id in output" assert json_out["units"] == units, "Incorrect units in output" assert json_out["values"] == [ - [dt.isoformat(), 42, 0] + [DT.isoformat(), 42, 0] ], "Values do not match expected" def test_store_multi_timeseries_df(): - now = datetime.now(timezone.utc).replace(microsecond=0) - TS_ID_REV_TEST = TEST_TSID_MULTI.replace("Raw-Multi", "Raw-Rev-Test") df = pd.DataFrame( { - "date-time": [now, now], + "date-time": [DT, DT], "value": [7, 8], "quality-code": [0, 0], "ts_id": [TEST_TSID_MULTI, TS_ID_REV_TEST], @@ -172,8 +175,12 @@ def test_store_multi_timeseries_df(): } ) ts.store_multi_timeseries_df(df, TEST_OFFICE) - data1 = ts.get_timeseries(TEST_TSID_MULTI, TEST_OFFICE, multithread=False).json - data2 = ts.get_timeseries(TS_ID_REV_TEST, TEST_OFFICE, multithread=False).json + data1 = ts.get_timeseries( + TEST_TSID_MULTI, TEST_OFFICE, multithread=False, begin=BEGIN, end=END + ).json + data2 = ts.get_timeseries( + TS_ID_REV_TEST, TEST_OFFICE, multithread=False, begin=BEGIN, end=END + ).json assert data1["name"] == TEST_TSID_MULTI assert data1["office-id"] == TEST_OFFICE assert data1["units"] == "ft" @@ -229,8 +236,9 @@ def test_get_multi_timeseries_chunk_df(): def test_get_multi_timeseries_df(): - TS_ID_REV_TEST = TEST_TSID_MULTI.replace("Raw-Multi", "Raw-Rev-Test") - df = ts.get_multi_timeseries_df([TEST_TSID_MULTI, TS_ID_REV_TEST], TEST_OFFICE) + df = ts.get_multi_timeseries_df( + [TEST_TSID_MULTI, TS_ID_REV_TEST], TEST_OFFICE, begin=BEGIN, end=END + ) assert df is not None, "Returned DataFrame is None" assert not df.empty, "Returned DataFrame is empty" assert any( @@ -324,10 +332,6 @@ def test_read_timeseries_chunk_ts(): def test_delete_timeseries(): - TS_ID_REV_TEST = TEST_TSID_MULTI.replace("Raw-Multi", "Raw-Rev-Test") - now = datetime.now(timezone.utc).replace(microsecond=0) - begin = now - timedelta(minutes=15) - end = now + timedelta(minutes=15) - ts.delete_timeseries(TS_ID_REV_TEST, TEST_OFFICE, begin, end) - result = ts.get_timeseries(TS_ID_REV_TEST, TEST_OFFICE) + ts.delete_timeseries(TEST_TSID_DELETE, TEST_OFFICE, BEGIN, END) + result = ts.get_timeseries(TEST_TSID_DELETE, TEST_OFFICE) assert result is None or result.json.get("values", []) == [] From 2ceae55a0d57d962badedf5e1353cfcfe8bb89d0 Mon Sep 17 00:00:00 2001 From: Eric Novotny Date: Fri, 27 Feb 2026 12:33:58 -0800 Subject: [PATCH 3/3] fix black version --- .github/ISSUE_TEMPLATE/bug-report.yml | 10 +++++----- .github/workflows/code-check.yml | 3 +++ 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml index 119b79b4..b7c563d7 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.yml +++ b/.github/ISSUE_TEMPLATE/bug-report.yml @@ -52,7 +52,8 @@ body: attributes: label: Describe the bug description: What went wrong? - placeholder: 'A clear and concise description of the bug... Did you get different data output? Did something work but fail in other ways?' + placeholder: 'A clear and concise description of the bug... Did you get different + data output? Did something work but fail in other ways?' validations: required: true @@ -60,10 +61,9 @@ body: id: logs attributes: label: REDACTED relevant log output - description: - Please copy and paste any relevant log output and/or the stack trace. Removing sensitive - information. This will be automatically formatted into code, so no need for - backticks. + description: Please copy and paste any relevant log output and/or the stack + trace. Removing sensitive information. This will be automatically formatted + into code, so no need for backticks. render: shell validations: required: true diff --git a/.github/workflows/code-check.yml b/.github/workflows/code-check.yml index f28ba958..a1dd56aa 100644 --- a/.github/workflows/code-check.yml +++ b/.github/workflows/code-check.yml @@ -16,4 +16,7 @@ jobs: # formatted. The code is not automatically reformatted like it is when running the # pre-commit hooks. - uses: psf/black@stable + with: + # Specify the desired Black version + version: "24.10.0" - uses: isort/isort-action@v1