From ab3c246874512c72a195d8a2551abccb88a3b0b2 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Fri, 14 Feb 2025 13:44:27 -0700 Subject: [PATCH 1/6] Add well data back to NMBGMR site records --- backend/connectors/nmbgmr/source.py | 27 ++++++++++-------------- backend/connectors/nmbgmr/transformer.py | 8 +++---- 2 files changed, 15 insertions(+), 20 deletions(-) diff --git a/backend/connectors/nmbgmr/source.py b/backend/connectors/nmbgmr/source.py index 243dc22..e837569 100644 --- a/backend/connectors/nmbgmr/source.py +++ b/backend/connectors/nmbgmr/source.py @@ -84,24 +84,19 @@ def get_records(self): sites = self._execute_json_request( _make_url("locations"), params, tag="features", timeout=30 ) + for site in sites: + print(f"Obtaining well data for {site['properties']['point_id']}") + well_data = self._execute_json_request( + _make_url("wells"), + params={"pointid": site["properties"]["point_id"]}, + tag="", + ) + site["properties"]["formation"] = well_data["formation"] + site["properties"]["well_depth"] = well_data["well_depth_ftbgs"] + site["properties"]["well_depth_units"] = FEET + return sites - # loop through the responses and add well information for each location - # this may be slow because of the number of sites that need to be queried - # but it is necessary to get the well information. With further - # development, this could be faster if one can batch the requests - # to /wells - # for site in sites: - # well_info = self._execute_json_request( - # _make_url("/wells"), - # params={"pointid": site["properties"]["point_id"]}, - # tag="", - # ) - # site["properties"]["formation"] = well_info["formation"] - # site["properties"]["well_depth"] = well_info["well_depth_ftbgs"] - # site["properties"]["well_depth_units"] = "ft" - - class NMBGMRAnalyteSource(BaseAnalyteSource): transformer_klass = NMBGMRAnalyteTransformer diff --git a/backend/connectors/nmbgmr/transformer.py b/backend/connectors/nmbgmr/transformer.py index b84f6b2..31c4285 100644 --- a/backend/connectors/nmbgmr/transformer.py +++ b/backend/connectors/nmbgmr/transformer.py @@ -25,7 +25,7 @@ class NMBGMRSiteTransformer(SiteTransformer): def _transform(self, record): - props = record["properties"] + props = record["properties"] rec = { "source": "NMBGMR", "id": props["point_id"], @@ -38,9 +38,9 @@ def _transform(self, record): "vertical_datum": props["altitude_datum"], "usgs_site_id": props["site_id"], "alternate_site_id": props["alternate_site_id"], - # "formation": props["formation"], - # "well_depth": props["well_depth"]["value"], - # "well_depth_units": props["well_depth"]["units"], + "formation": props["formation"], + "well_depth": props["well_depth"], + "well_depth_units": props["well_depth_units"], } return rec From 79ab18e832cd42d3d4b998208c28d2f9e3b4896f Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Fri, 14 Feb 2025 13:44:45 -0700 Subject: [PATCH 2/6] Subtract stickup_height from top of casing to get elevation for CABQ --- README.md | 5 ++-- backend/connectors/st2/transformer.py | 40 ++++++++++++++++++++++++++- 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 0f25255..2314f70 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,6 @@ Data comes from the following sources. We are continuously adding new sources as - Available data: `water quality` - [City of Albuquerque (CABQ)](https://st2.newmexicowaterdata.org/FROST-Server/v1.1/Locations?$filter=properties/agency%20eq%20%27CABQ%27) - Available data: `water levels` - - Note: the elevation data is the top of casing, not ground surface elevation - [Elephant Butte Irrigation District (EBID)](https://st2.newmexicowaterdata.org/FROST-Server/v1.1/Locations?$filter=properties/agency%20eq%20%27EBID%27) - Available data: `water levels` - [New Mexico Bureau of Geology and Mineral Resources (NMBGMR) Aquifer Mapping Program (AMP)](https://waterdata.nmt.edu/) @@ -124,7 +123,7 @@ A log of the inputs and processes, called `die.log`, is also saved to the output | most_recent_value | value of the most recent record | float | Y | | most_recent_units | units of the most recent record | string | Y | -* elevation at top of casing for CABQ +*CABQ elevation is calculated as [elevation at top of casing] - [stickup height]; if stickup height < 0 the measuring point is assumed to be beneath the ground surface #### Sites Table @@ -145,7 +144,7 @@ A log of the inputs and processes, called `die.log`, is also saved to the output | aquifer | aquifer from which the well draws water | string | N | | well_depth | depth of well | float | N | -** elevation at top of casing for CABQ +**CABQ elevation is calculated as [elevation at top of casing] - [stickup height]; if stickup height < 0 the measuring point is assumed to be beneath the ground surface #### Time Series Table(s) diff --git a/backend/connectors/st2/transformer.py b/backend/connectors/st2/transformer.py index f898a25..d9bdfab 100644 --- a/backend/connectors/st2/transformer.py +++ b/backend/connectors/st2/transformer.py @@ -14,10 +14,11 @@ # limitations under the License. # =============================================================================== import pprint +import sys from backend.connectors.st_connector import STSiteTransformer from backend.record import SiteRecord, WaterLevelRecord -from backend.transformer import BaseTransformer, WaterLevelTransformer, SiteTransformer +from backend.transformer import BaseTransformer, WaterLevelTransformer, SiteTransformer, convert_units class PVACDSiteTransformer(STSiteTransformer): @@ -58,6 +59,43 @@ class EBIDSiteTransformer(STSiteTransformer): class CABQSiteTransformer(STSiteTransformer): source_id = "ST2/CABQ" + def _transform(self, record): + if self.source_id is None: + raise ValueError(f"{self.__class__.__name__} Source ID not set") + + coordinates = record.location["coordinates"] + + lat = coordinates[1] + lng = coordinates[0] + # if not self.contained(lng, lat): + # print("not contained") + # return + + ele = None + if len(coordinates) == 3: + ele = coordinates[2] + + try: + thing = record.things._entities[0] + stickup_height_ft = thing._properties["stickup_height"]["value"] + stickup_height_m, conversion_factor, warning_msg = convert_units(stickup_height_ft, "ft", "m", "stickup_height", "stickup_height") + ele = ele - stickup_height_m + except KeyError: + self.config.warn(f"No stickup_height for {record.id}") + + + rec = { + "source": self.source_id, + "id": record.id, + "name": record.name, + "latitude": lat, + "longitude": lng, + "elevation": ele, + "elevation_units": "m", + "horizontal_datum": "WGS84", + } + + return self._transform_hook(rec) # class ST2WaterLevelTransformer(WaterLevelTransformer): # source_tag = "ST2" From 0a7062591a561c2ea525a1d1f0044c44189c0b2b Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Fri, 14 Feb 2025 13:45:15 -0700 Subject: [PATCH 3/6] Bump to version 0.6.0 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d212929..5975a97 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ setup( name="nmuwd", - version="0.5.2", + version="0.6.0", author="Jake Ross", description="New Mexico Water Data Integration Engine", long_description=long_description, From 8ec677c403b053d749569ccd71e569969fdcf94a Mon Sep 17 00:00:00 2001 From: jacob-a-brown Date: Fri, 14 Feb 2025 20:46:30 +0000 Subject: [PATCH 4/6] Formatting changes --- backend/connectors/nmbgmr/source.py | 3 ++- backend/connectors/nmbgmr/transformer.py | 2 +- backend/connectors/st2/transformer.py | 13 ++++++++++--- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/backend/connectors/nmbgmr/source.py b/backend/connectors/nmbgmr/source.py index e837569..6a2e3b6 100644 --- a/backend/connectors/nmbgmr/source.py +++ b/backend/connectors/nmbgmr/source.py @@ -94,9 +94,10 @@ def get_records(self): site["properties"]["formation"] = well_data["formation"] site["properties"]["well_depth"] = well_data["well_depth_ftbgs"] site["properties"]["well_depth_units"] = FEET - + return sites + class NMBGMRAnalyteSource(BaseAnalyteSource): transformer_klass = NMBGMRAnalyteTransformer diff --git a/backend/connectors/nmbgmr/transformer.py b/backend/connectors/nmbgmr/transformer.py index 31c4285..dd1163e 100644 --- a/backend/connectors/nmbgmr/transformer.py +++ b/backend/connectors/nmbgmr/transformer.py @@ -25,7 +25,7 @@ class NMBGMRSiteTransformer(SiteTransformer): def _transform(self, record): - props = record["properties"] + props = record["properties"] rec = { "source": "NMBGMR", "id": props["point_id"], diff --git a/backend/connectors/st2/transformer.py b/backend/connectors/st2/transformer.py index d9bdfab..5ebc51d 100644 --- a/backend/connectors/st2/transformer.py +++ b/backend/connectors/st2/transformer.py @@ -18,7 +18,12 @@ from backend.connectors.st_connector import STSiteTransformer from backend.record import SiteRecord, WaterLevelRecord -from backend.transformer import BaseTransformer, WaterLevelTransformer, SiteTransformer, convert_units +from backend.transformer import ( + BaseTransformer, + WaterLevelTransformer, + SiteTransformer, + convert_units, +) class PVACDSiteTransformer(STSiteTransformer): @@ -78,12 +83,13 @@ def _transform(self, record): try: thing = record.things._entities[0] stickup_height_ft = thing._properties["stickup_height"]["value"] - stickup_height_m, conversion_factor, warning_msg = convert_units(stickup_height_ft, "ft", "m", "stickup_height", "stickup_height") + stickup_height_m, conversion_factor, warning_msg = convert_units( + stickup_height_ft, "ft", "m", "stickup_height", "stickup_height" + ) ele = ele - stickup_height_m except KeyError: self.config.warn(f"No stickup_height for {record.id}") - rec = { "source": self.source_id, "id": record.id, @@ -97,6 +103,7 @@ def _transform(self, record): return self._transform_hook(rec) + # class ST2WaterLevelTransformer(WaterLevelTransformer): # source_tag = "ST2" From e72103178eb63c5fcaff7b0c7d39f8212df8c9a9 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Fri, 14 Feb 2025 14:37:01 -0700 Subject: [PATCH 5/6] Make _transform_elevation method for ST2SiteTransformer CABQ needs to subtract the stickup_height from the elevation to get ground surface elevation. This method allows the _transform method to be universal and then these edge cases can be accounted for with different sources. --- backend/connectors/st2/transformer.py | 35 +++------------------------ backend/connectors/st_connector.py | 4 +++ 2 files changed, 8 insertions(+), 31 deletions(-) diff --git a/backend/connectors/st2/transformer.py b/backend/connectors/st2/transformer.py index 5ebc51d..4bb8dba 100644 --- a/backend/connectors/st2/transformer.py +++ b/backend/connectors/st2/transformer.py @@ -64,45 +64,18 @@ class EBIDSiteTransformer(STSiteTransformer): class CABQSiteTransformer(STSiteTransformer): source_id = "ST2/CABQ" - def _transform(self, record): - if self.source_id is None: - raise ValueError(f"{self.__class__.__name__} Source ID not set") - - coordinates = record.location["coordinates"] - - lat = coordinates[1] - lng = coordinates[0] - # if not self.contained(lng, lat): - # print("not contained") - # return - - ele = None - if len(coordinates) == 3: - ele = coordinates[2] - + def _transform_elevation(self, elevation, record): + if elevation: try: thing = record.things._entities[0] stickup_height_ft = thing._properties["stickup_height"]["value"] stickup_height_m, conversion_factor, warning_msg = convert_units( stickup_height_ft, "ft", "m", "stickup_height", "stickup_height" ) - ele = ele - stickup_height_m + elevation = elevation - stickup_height_m except KeyError: self.config.warn(f"No stickup_height for {record.id}") - - rec = { - "source": self.source_id, - "id": record.id, - "name": record.name, - "latitude": lat, - "longitude": lng, - "elevation": ele, - "elevation_units": "m", - "horizontal_datum": "WGS84", - } - - return self._transform_hook(rec) - + return elevation # class ST2WaterLevelTransformer(WaterLevelTransformer): # source_tag = "ST2" diff --git a/backend/connectors/st_connector.py b/backend/connectors/st_connector.py index 92cac48..d6b78ea 100644 --- a/backend/connectors/st_connector.py +++ b/backend/connectors/st_connector.py @@ -148,6 +148,9 @@ class STSiteTransformer(SiteTransformer): source_id: str check_contained = False # API returns only records within the bounds + def _transform_elevation(self, elevation, record): + return elevation + def _transform_hook(self, rec): return rec @@ -166,6 +169,7 @@ def _transform(self, record): ele = None if len(coordinates) == 3: ele = coordinates[2] + ele = self._transform_elevation(ele, record) rec = { "source": self.source_id, From 0f33bdb442c47d3c13f2ac3ed3199a94ef82f412 Mon Sep 17 00:00:00 2001 From: jacob-a-brown Date: Fri, 14 Feb 2025 21:41:01 +0000 Subject: [PATCH 6/6] Formatting changes --- backend/connectors/st2/transformer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/connectors/st2/transformer.py b/backend/connectors/st2/transformer.py index 4bb8dba..782d6d9 100644 --- a/backend/connectors/st2/transformer.py +++ b/backend/connectors/st2/transformer.py @@ -77,6 +77,7 @@ def _transform_elevation(self, elevation, record): self.config.warn(f"No stickup_height for {record.id}") return elevation + # class ST2WaterLevelTransformer(WaterLevelTransformer): # source_tag = "ST2"