Merge pull request #33 from DataIntegrationGroup/pre-production

Pre production

Author: jirhiker

README.md

@@ -24,7 +24,6 @@ Data comes from the following sources. We are continuously adding new sources as
   - Available data: `water quality`
 - [City of Albuquerque (CABQ)](https://st2.newmexicowaterdata.org/FROST-Server/v1.1/Locations?$filter=properties/agency%20eq%20%27CABQ%27)
   - Available data: `water levels`
-  - Note: the elevation data is the top of casing, not ground surface elevation
 - [Elephant Butte Irrigation District (EBID)](https://st2.newmexicowaterdata.org/FROST-Server/v1.1/Locations?$filter=properties/agency%20eq%20%27EBID%27)
   - Available data: `water levels`
 - [New Mexico Bureau of Geology and Mineral Resources (NMBGMR) Aquifer Mapping Program (AMP)](https://waterdata.nmt.edu/)
@@ -124,7 +123,7 @@ A log of the inputs and processes, called `die.log`, is also saved to the output
 | most_recent_value | value of the most recent record  | float | Y |
 | most_recent_units | units of the most recent record | string | Y |
 
-<sup>* elevation at top of casing for CABQ</sup>
+<sup>*CABQ elevation is calculated as [elevation at top of casing] - [stickup height]; if stickup height < 0 the measuring point is assumed to be beneath the ground surface</sup>
 
 #### Sites Table
 
@@ -145,7 +144,7 @@ A log of the inputs and processes, called `die.log`, is also saved to the output
 | aquifer | aquifer from which the well draws water | string | N |
 | well_depth | depth of well | float | N |
 
-<sup>** elevation at top of casing for CABQ</sup>
+<sup>**CABQ elevation is calculated as [elevation at top of casing] - [stickup height]; if stickup height < 0 the measuring point is assumed to be beneath the ground surface</sup>
 
 #### Time Series Table(s)
 

backend/connectors/nmbgmr/source.py

@@ -84,22 +84,18 @@ def get_records(self):
         sites = self._execute_json_request(
             _make_url("locations"), params, tag="features", timeout=30
         )
-        return sites
+        for site in sites:
+            print(f"Obtaining well data for {site['properties']['point_id']}")
+            well_data = self._execute_json_request(
+                _make_url("wells"),
+                params={"pointid": site["properties"]["point_id"]},
+                tag="",
+            )
+            site["properties"]["formation"] = well_data["formation"]
+            site["properties"]["well_depth"] = well_data["well_depth_ftbgs"]
+            site["properties"]["well_depth_units"] = FEET
 
-        # loop through the responses and add well information for each location
-        # this may be slow because of the number of sites that need to be queried
-        # but it is necessary to get the well information. With further
-        # development, this could be faster if one can batch the requests
-        # to /wells
-        # for site in sites:
-        #     well_info = self._execute_json_request(
-        #         _make_url("/wells"),
-        #         params={"pointid": site["properties"]["point_id"]},
-        #         tag="",
-        #     )
-        #     site["properties"]["formation"] = well_info["formation"]
-        #     site["properties"]["well_depth"] = well_info["well_depth_ftbgs"]
-        #     site["properties"]["well_depth_units"] = "ft"
+        return sites
 
 
 class NMBGMRAnalyteSource(BaseAnalyteSource):

backend/connectors/nmbgmr/transformer.py

@@ -38,9 +38,9 @@ def _transform(self, record):
             "vertical_datum": props["altitude_datum"],
             "usgs_site_id": props["site_id"],
             "alternate_site_id": props["alternate_site_id"],
-            # "formation": props["formation"],
-            # "well_depth": props["well_depth"]["value"],
-            # "well_depth_units": props["well_depth"]["units"],
+            "formation": props["formation"],
+            "well_depth": props["well_depth"],
+            "well_depth_units": props["well_depth_units"],
         }
         return rec
 

backend/connectors/st2/transformer.py

@@ -14,10 +14,16 @@
 # limitations under the License.
 # ===============================================================================
 import pprint
+import sys
 
 from backend.connectors.st_connector import STSiteTransformer
 from backend.record import SiteRecord, WaterLevelRecord
-from backend.transformer import BaseTransformer, WaterLevelTransformer, SiteTransformer
+from backend.transformer import (
+    BaseTransformer,
+    WaterLevelTransformer,
+    SiteTransformer,
+    convert_units,
+)
 
 
 class PVACDSiteTransformer(STSiteTransformer):
@@ -58,6 +64,19 @@ class EBIDSiteTransformer(STSiteTransformer):
 class CABQSiteTransformer(STSiteTransformer):
     source_id = "ST2/CABQ"
 
+    def _transform_elevation(self, elevation, record):
+        if elevation:
+            try:
+                thing = record.things._entities[0]
+                stickup_height_ft = thing._properties["stickup_height"]["value"]
+                stickup_height_m, conversion_factor, warning_msg = convert_units(
+                    stickup_height_ft, "ft", "m", "stickup_height", "stickup_height"
+                )
+                elevation = elevation - stickup_height_m
+            except KeyError:
+                self.config.warn(f"No stickup_height for {record.id}")
+        return elevation
+
 
 # class ST2WaterLevelTransformer(WaterLevelTransformer):
 #     source_tag = "ST2"

backend/connectors/st_connector.py

@@ -148,6 +148,9 @@ class STSiteTransformer(SiteTransformer):
     source_id: str
     check_contained = False  # API returns only records within the bounds
 
+    def _transform_elevation(self, elevation, record):
+        return elevation
+
     def _transform_hook(self, rec):
         return rec
 
@@ -166,6 +169,7 @@ def _transform(self, record):
         ele = None
         if len(coordinates) == 3:
             ele = coordinates[2]
+            ele = self._transform_elevation(ele, record)
 
         rec = {
             "source": self.source_id,

setup.py

@@ -21,7 +21,7 @@
 
 setup(
     name="nmuwd",
-    version="0.5.2",
+    version="0.6.0",
     author="Jake Ross",
     description="New Mexico Water Data Integration Engine",
     long_description=long_description,