From 75807bf91dd2e4ea3bab8bcb5a9f80a0b53c7fd3 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 17 Mar 2025 16:35:36 -0600 Subject: [PATCH 1/8] Removed uranium 238 from the list of isotopes From Bonnie Frey's feedback that is about 99% of the sample, but not the total --- backend/connectors/mappings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/connectors/mappings.py b/backend/connectors/mappings.py index 2fe4267..d2468d0 100644 --- a/backend/connectors/mappings.py +++ b/backend/connectors/mappings.py @@ -170,7 +170,7 @@ "Total Sulfate", ], TDS: ["Total dissolved solids"], - URANIUM: ["Uranium", "Uranium-238"], + URANIUM: ["Uranium"], PH: ["pH"], } # BOR =============================================================================== From b58923bcc3047f159e97c883a630bae69a835da1 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Tue, 18 Mar 2025 10:40:48 -0600 Subject: [PATCH 2/8] Add water level data from WQP --- CHANGELOG.md | 9 +++- README.md | 4 +- backend/config.py | 4 +- backend/connectors/wqp/source.py | 64 +++++++++++++++++++-------- backend/connectors/wqp/transformer.py | 6 ++- frontend/cli.py | 2 +- 6 files changed, 64 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b8c891..501273c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,14 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## Unrelease +## Unreleased: 0.8.0 + +### Added +- water level for WQP + +### Changed + +### Fixed ## 0.7.0 diff --git a/README.md b/README.md index 705dd92..9fa7ab7 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ Data comes from the following sources. We are continuously adding new sources as - [USGS (NWIS)](https://waterdata.usgs.gov/nwis) - Available data: `water levels` - [Water Quality Portal (WQP)](https://www.waterqualitydata.us/) - - Available data: `water quality` + - Available data: `water levels`, `water quality` ## Usage @@ -66,7 +66,7 @@ where `{parameter}` is the name of the parameter whose data is to be retrieved, | **nmose-roswell** | X | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | | **nwis** | X | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | | **pvacd** | X | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | -| **wqp** | - | X | X | X | X | X | X | X | X | X | X | X | X | X | X | X | +| **wqp** | X | X | X | X | X | X | X | X | X | X | X | X | X | X | X | X | ### Output The `--output` option is required and used to set the output type: diff --git a/backend/config.py b/backend/config.py index b09b88f..ecf69b3 100644 --- a/backend/config.py +++ b/backend/config.py @@ -57,7 +57,7 @@ CABQWaterLevelSource, ) from .connectors.usgs.source import NWISSiteSource, NWISWaterLevelSource -from .connectors.wqp.source import WQPSiteSource, WQPAnalyteSource +from .connectors.wqp.source import WQPSiteSource, WQPAnalyteSource, WQPWaterLevelSource SOURCE_KEYS = ( "bernco", @@ -244,6 +244,8 @@ def water_level_sources(self): sources.append((EBIDSiteSource(), EBIDWaterLevelSource())) if self.use_source_cabq: sources.append((CABQSiteSource(), CABQWaterLevelSource())) + if self.use_source_wqp: + sources.append((WQPSiteSource(), WQPWaterLevelSource())) for s, ss in sources: s.set_config(self) diff --git a/backend/connectors/wqp/source.py b/backend/connectors/wqp/source.py index 4a0f55d..be68031 100644 --- a/backend/connectors/wqp/source.py +++ b/backend/connectors/wqp/source.py @@ -20,12 +20,6 @@ from backend.connectors import NM_STATE_BOUNDING_POLYGON from backend.connectors.mappings import WQP_ANALYTE_MAPPING from backend.constants import ( - TDS, - URANIUM, - NITRATE, - SULFATE, - ARSENIC, - CHLORIDE, PARAMETER_NAME, PARAMETER_VALUE, PARAMETER_UNITS, @@ -33,11 +27,12 @@ SOURCE_PARAMETER_UNITS, DT_MEASURED, ) -from backend.connectors.wqp.transformer import WQPSiteTransformer, WQPAnalyteTransformer +from backend.connectors.wqp.transformer import WQPSiteTransformer, WQPAnalyteTransformer, WQPWaterLevelTransformer from backend.source import ( - BaseSource, BaseSiteSource, BaseAnalyteSource, + BaseWaterLevelSource, + BaseParameterSource, make_site_list, get_most_recent, get_analyte_search_param, @@ -93,6 +88,10 @@ def get_records(self): params["characteristicName"] = get_analyte_search_param( config.parameter, WQP_ANALYTE_MAPPING ) + else: + # every record with pCode 30210 (depth in m) has a corresponding + # record with pCode 72019 (depth in ft) but not vice versa + params["pCode"] = "30210" params.update(get_date_range(config)) @@ -103,17 +102,13 @@ def get_records(self): return parse_tsv(text) -class WQPAnalyteSource(BaseAnalyteSource): - transformer_klass = WQPAnalyteTransformer - - def __repr__(self): - return "WQPAnalyteSource" +class WQPParameterSource(BaseParameterSource): def _extract_parameter_record(self, record): record[PARAMETER_NAME] = self.config.parameter record[PARAMETER_VALUE] = record["ResultMeasureValue"] - record[PARAMETER_UNITS] = self.config.analyte_output_units - record[DT_MEASURED] = record["ActivityStartDate"] + record[PARAMETER_UNITS] = self._parameter_units_hook() + record[DT_MEASURED] = f"{record['ActivityStartDate']} {record['ActivityStartTime/Time']}" record[SOURCE_PARAMETER_NAME] = record["CharacteristicName"] record[SOURCE_PARAMETER_UNITS] = record["ResultMeasure/MeasureUnitCode"] return record @@ -148,22 +143,53 @@ def _extract_most_recent(self, records): } def get_records(self, site_record): + config = self.config sites = make_site_list(site_record) params = { "siteid": sites, "mimeType": "tsv", - "characteristicName": get_analyte_search_param( - self.config.parameter, WQP_ANALYTE_MAPPING - ), } params.update(get_date_range(self.config)) + if config.parameter.lower() != "waterlevels": + params["characteristicName"] = get_analyte_search_param( + config.parameter, WQP_ANALYTE_MAPPING + ) + else: + # every record with pCode 30210 (depth in m) has a corresponding + # record with pCode 72019 (depth in ft) but not vice versa + params["pCode"] = "30210" + + params.update(get_date_range(config)) + text = self._execute_text_request( - "https://www.waterqualitydata.us/data/Result/search?", params + "https://www.waterqualitydata.us/data/Result/search?", params, timeout=30 ) if text: return parse_tsv(text) + def _parameter_units_hook(self): + raise NotImplementedError( + f"{self.__class__.__name__} must implement _parameter_units_hook" + ) + +class WQPAnalyteSource(BaseAnalyteSource, WQPParameterSource): + transformer_klass = WQPAnalyteTransformer + + def __repr__(self): + return "WQPAnalyteSource" + + def _parameter_units_hook(self): + return self.config.analyte_output_units + +class WQPWaterLevelSource(BaseWaterLevelSource, WQPParameterSource): + transformer_klass = WQPWaterLevelTransformer + + def __repr__(self): + return "WQPWaterLevelSource" + + def _parameter_units_hook(self): + return self.config.waterlevel_output_units # ============= EOF ============================================= diff --git a/backend/connectors/wqp/transformer.py b/backend/connectors/wqp/transformer.py index e611724..ede8c22 100644 --- a/backend/connectors/wqp/transformer.py +++ b/backend/connectors/wqp/transformer.py @@ -16,7 +16,7 @@ import pprint from backend.record import SiteRecord, AnalyteSummaryRecord -from backend.transformer import BaseTransformer, SiteTransformer, AnalyteTransformer +from backend.transformer import BaseTransformer, SiteTransformer, AnalyteTransformer, WaterLevelTransformer class WQPSiteTransformer(SiteTransformer): @@ -44,4 +44,8 @@ class WQPAnalyteTransformer(AnalyteTransformer): source_tag = "WQP" +class WQPWaterLevelTransformer(WaterLevelTransformer): + source_tag = "WQP" + + # ============= EOF ============================================= diff --git a/frontend/cli.py b/frontend/cli.py index 2d4f278..e03ac0b 100644 --- a/frontend/cli.py +++ b/frontend/cli.py @@ -264,10 +264,10 @@ def weave( config.use_source_nmose_roswell = no_nmose_roswell config.use_source_nwis = no_nwis config.use_source_pvacd = no_pvacd + config.use_source_wqp = no_wqp config.use_source_bor = False config.use_source_nmed_dwb = False - config.use_source_wqp = False elif parameter == "carbonate": config.use_source_nmbgmr_amp = no_nmbgmr_amp From a06ce25df31542c069038c5cb0cd64ae23310e79 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Tue, 18 Mar 2025 13:42:02 -0600 Subject: [PATCH 3/8] Fix inheritance order for WQP water levels --- backend/connectors/wqp/source.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/backend/connectors/wqp/source.py b/backend/connectors/wqp/source.py index be68031..2ead0ec 100644 --- a/backend/connectors/wqp/source.py +++ b/backend/connectors/wqp/source.py @@ -174,7 +174,7 @@ def _parameter_units_hook(self): f"{self.__class__.__name__} must implement _parameter_units_hook" ) -class WQPAnalyteSource(BaseAnalyteSource, WQPParameterSource): +class WQPAnalyteSource(WQPParameterSource, BaseAnalyteSource): transformer_klass = WQPAnalyteTransformer def __repr__(self): @@ -183,7 +183,8 @@ def __repr__(self): def _parameter_units_hook(self): return self.config.analyte_output_units -class WQPWaterLevelSource(BaseWaterLevelSource, WQPParameterSource): +# inherit from WQPParameterSource first so that its _extract_souce_parameter_units method is used instead of BaseWaterLevelSource's method +class WQPWaterLevelSource(WQPParameterSource, BaseWaterLevelSource): transformer_klass = WQPWaterLevelTransformer def __repr__(self): From 362b023fc69390d7f88345070daeef937a6a89f5 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Tue, 18 Mar 2025 14:22:25 -0600 Subject: [PATCH 4/8] Work on transfering OSE Roswell to ST2 --- backend/connectors/st2/source.py | 17 +++++++++++++++++ backend/connectors/st2/transformer.py | 7 +++++++ 2 files changed, 24 insertions(+) diff --git a/backend/connectors/st2/source.py b/backend/connectors/st2/source.py index 5cd76b0..cc78e57 100644 --- a/backend/connectors/st2/source.py +++ b/backend/connectors/st2/source.py @@ -24,6 +24,8 @@ CABQ_BOUNDING_POLYGON, ) from backend.connectors.st2.transformer import ( + NMOSERoswellSiteTransformer, + NMOSERoswellWaterLevelTransformer, PVACDSiteTransformer, PVACDWaterLevelTransformer, EBIDSiteTransformer, @@ -64,6 +66,14 @@ def _get_filters(self): return [f"properties/agency eq '{self.agency}'"] +class NMOSERoswellSiteSource(ST2SiteSource): + transformer_klass = NMOSERoswellSiteTransformer + agency = "OSE-Roswell" + + def __repr__(self): + return "NMOSERoswellSiteSource" + + class PVACDSiteSource(ST2SiteSource): transformer_klass = PVACDSiteTransformer agency = "PVACD" @@ -171,6 +181,13 @@ def get_records(self, site_record, *args, **kw): # break return records +class NMOSERoswellWaterLevelSource(ST2WaterLevelSource): + transformer_klass = NMOSERoswellWaterLevelTransformer + agency = "OSE-Roswell" + + def __repr__(self): + return "NMOSERoswellWaterLevelSource" + class PVACDWaterLevelSource(ST2WaterLevelSource): transformer_klass = PVACDWaterLevelTransformer diff --git a/backend/connectors/st2/transformer.py b/backend/connectors/st2/transformer.py index 782d6d9..5049899 100644 --- a/backend/connectors/st2/transformer.py +++ b/backend/connectors/st2/transformer.py @@ -26,6 +26,10 @@ ) +class NMOSERoswellSiteTransformer(STSiteTransformer): + source_id = "ST2/NMOSE-Roswell" + + class PVACDSiteTransformer(STSiteTransformer): source_id = "ST2/PVACD" @@ -103,6 +107,9 @@ def _transform_elevation(self, elevation, record): # return rec +class NMOSERoswellWaterLevelTransformer(WaterLevelTransformer): + source_tag = "ST2/NMOSE-Roswell" + class PVACDWaterLevelTransformer(WaterLevelTransformer): source_tag = "ST2/PVACD" From 4368c28f38c5216f74e7bcc0dc728295ae5bd851 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Tue, 18 Mar 2025 15:04:26 -0600 Subject: [PATCH 5/8] Use ST2 for OSE Roswell --- backend/config.py | 23 ++++------------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/backend/config.py b/backend/config.py index ecf69b3..ec9be97 100644 --- a/backend/config.py +++ b/backend/config.py @@ -55,6 +55,8 @@ BernCoWaterLevelSource, CABQSiteSource, CABQWaterLevelSource, + NMOSERoswellSiteSource, + NMOSERoswellWaterLevelSource, ) from .connectors.usgs.source import NWISSiteSource, NWISWaterLevelSource from .connectors.wqp.source import WQPSiteSource, WQPAnalyteSource, WQPWaterLevelSource @@ -90,7 +92,7 @@ def get_source(source): elif source == "nmose_isc_seven_rivers": return ISCSevenRiversSiteSource() elif source == "nmose_roswell": - return OSERoswellSiteSource(HONDO_RESOURCE_ID) + return NMOSERoswellSiteSource() elif source == "nwis": return NWISSiteSource() elif source == "pvacd": @@ -218,24 +220,7 @@ def water_level_sources(self): sources.append((NWISSiteSource(), NWISWaterLevelSource())) if self.use_source_nmose_roswell: - sources.append( - ( - OSERoswellSiteSource(HONDO_RESOURCE_ID), - OSERoswellWaterLevelSource(HONDO_RESOURCE_ID), - ) - ) - sources.append( - ( - OSERoswellSiteSource(FORT_SUMNER_RESOURCE_ID), - OSERoswellWaterLevelSource(FORT_SUMNER_RESOURCE_ID), - ) - ) - sources.append( - ( - OSERoswellSiteSource(ROSWELL_RESOURCE_ID), - OSERoswellWaterLevelSource(ROSWELL_RESOURCE_ID), - ) - ) + sources.append((NMOSERoswellSiteSource(), NMOSERoswellWaterLevelSource())) if self.use_source_pvacd: sources.append((PVACDSiteSource(), PVACDWaterLevelSource())) if self.use_source_bernco: From 2bfbe624223ebc39c0c428009676325c7826844e Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Wed, 19 Mar 2025 10:05:26 -0600 Subject: [PATCH 6/8] Bump version to 0.8.0 --- CHANGELOG.md | 1 + setup.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 501273c..eff71cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), - water level for WQP ### Changed +- NM OSE Roswell data is now pulled from ST2 and not CKAN ### Fixed diff --git a/setup.py b/setup.py index e9d3a6f..c8b1657 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ setup( name="nmuwd", - version="0.7.0", + version="0..0", author="Jake Ross", description="New Mexico Water Data Integration Engine", long_description=long_description, From 233d7a9f15e268672f93e823a87cf32bbb702aba Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 20 Mar 2025 15:22:09 -0600 Subject: [PATCH 7/8] Decrease WQP chunk size to 50 to help with timeout errors --- backend/connectors/wqp/source.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/connectors/wqp/source.py b/backend/connectors/wqp/source.py index 2ead0ec..22e63ef 100644 --- a/backend/connectors/wqp/source.py +++ b/backend/connectors/wqp/source.py @@ -56,7 +56,7 @@ def get_date_range(config): class WQPSiteSource(BaseSiteSource): transformer_klass = WQPSiteTransformer - chunk_size = 100 + chunk_size = 50 bounding_polygon = NM_STATE_BOUNDING_POLYGON diff --git a/setup.py b/setup.py index c8b1657..1b99d57 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ setup( name="nmuwd", - version="0..0", + version="0.7.1", author="Jake Ross", description="New Mexico Water Data Integration Engine", long_description=long_description, From eccf6f17742b744ee0e25ec0af2f06ffd80eb19c Mon Sep 17 00:00:00 2001 From: jacob-a-brown Date: Thu, 20 Mar 2025 21:23:38 +0000 Subject: [PATCH 8/8] Formatting changes --- backend/connectors/st2/source.py | 1 + backend/connectors/st2/transformer.py | 1 + backend/connectors/wqp/source.py | 17 +++++++++++++---- backend/connectors/wqp/transformer.py | 7 ++++++- 4 files changed, 21 insertions(+), 5 deletions(-) diff --git a/backend/connectors/st2/source.py b/backend/connectors/st2/source.py index cc78e57..69739f4 100644 --- a/backend/connectors/st2/source.py +++ b/backend/connectors/st2/source.py @@ -181,6 +181,7 @@ def get_records(self, site_record, *args, **kw): # break return records + class NMOSERoswellWaterLevelSource(ST2WaterLevelSource): transformer_klass = NMOSERoswellWaterLevelTransformer agency = "OSE-Roswell" diff --git a/backend/connectors/st2/transformer.py b/backend/connectors/st2/transformer.py index 5049899..eaf62f1 100644 --- a/backend/connectors/st2/transformer.py +++ b/backend/connectors/st2/transformer.py @@ -110,6 +110,7 @@ def _transform_elevation(self, elevation, record): class NMOSERoswellWaterLevelTransformer(WaterLevelTransformer): source_tag = "ST2/NMOSE-Roswell" + class PVACDWaterLevelTransformer(WaterLevelTransformer): source_tag = "ST2/PVACD" diff --git a/backend/connectors/wqp/source.py b/backend/connectors/wqp/source.py index 22e63ef..4987fee 100644 --- a/backend/connectors/wqp/source.py +++ b/backend/connectors/wqp/source.py @@ -27,7 +27,11 @@ SOURCE_PARAMETER_UNITS, DT_MEASURED, ) -from backend.connectors.wqp.transformer import WQPSiteTransformer, WQPAnalyteTransformer, WQPWaterLevelTransformer +from backend.connectors.wqp.transformer import ( + WQPSiteTransformer, + WQPAnalyteTransformer, + WQPWaterLevelTransformer, +) from backend.source import ( BaseSiteSource, BaseAnalyteSource, @@ -108,7 +112,9 @@ def _extract_parameter_record(self, record): record[PARAMETER_NAME] = self.config.parameter record[PARAMETER_VALUE] = record["ResultMeasureValue"] record[PARAMETER_UNITS] = self._parameter_units_hook() - record[DT_MEASURED] = f"{record['ActivityStartDate']} {record['ActivityStartTime/Time']}" + record[DT_MEASURED] = ( + f"{record['ActivityStartDate']} {record['ActivityStartTime/Time']}" + ) record[SOURCE_PARAMETER_NAME] = record["CharacteristicName"] record[SOURCE_PARAMETER_UNITS] = record["ResultMeasure/MeasureUnitCode"] return record @@ -174,6 +180,7 @@ def _parameter_units_hook(self): f"{self.__class__.__name__} must implement _parameter_units_hook" ) + class WQPAnalyteSource(WQPParameterSource, BaseAnalyteSource): transformer_klass = WQPAnalyteTransformer @@ -182,15 +189,17 @@ def __repr__(self): def _parameter_units_hook(self): return self.config.analyte_output_units - + + # inherit from WQPParameterSource first so that its _extract_souce_parameter_units method is used instead of BaseWaterLevelSource's method class WQPWaterLevelSource(WQPParameterSource, BaseWaterLevelSource): transformer_klass = WQPWaterLevelTransformer def __repr__(self): return "WQPWaterLevelSource" - + def _parameter_units_hook(self): return self.config.waterlevel_output_units + # ============= EOF ============================================= diff --git a/backend/connectors/wqp/transformer.py b/backend/connectors/wqp/transformer.py index ede8c22..4764dc0 100644 --- a/backend/connectors/wqp/transformer.py +++ b/backend/connectors/wqp/transformer.py @@ -16,7 +16,12 @@ import pprint from backend.record import SiteRecord, AnalyteSummaryRecord -from backend.transformer import BaseTransformer, SiteTransformer, AnalyteTransformer, WaterLevelTransformer +from backend.transformer import ( + BaseTransformer, + SiteTransformer, + AnalyteTransformer, + WaterLevelTransformer, +) class WQPSiteTransformer(SiteTransformer):