Merge pull request #34 from DataIntegrationGroup/dev/jab

Author: jirhiker

CHANGELOG.md

@@ -0,0 +1,34 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## Unrelease
+
+
+## 0.7.0
+
+### Added
+
+- CHANGELOG.md to document changes to the DIE
+- ppb to mg/L unit conversion
+
+### Changed
+
+- Report analyte sources as `False` if source does not have measurements for those analytes
+  - e.g. BoR has analyte measurements, but does not have `carbonate` measurements, so if the user invokes `die weave carbonate ...` Config will print to the command line `user_source_bor: False` even if the user does not specify `--no-bor`
+
+### Fixed
+
+- Decreased NMBGMR chunk size from 100 to 10 to prevent ReadTimeoutErrors from occurring while gathering water level data
+
+## 0.6.0 - 2025-02-14
+
+### Added 
+
+- CABQ water level data
+- EBID water level data
+- `source_parameter_name`, `source_parameter_units`, and `conversion_factor` to all time series tables
+- `well_depth`, `well_depth_units`, and `formation` back to NMBGMR summary and site tables
+- UNIT_CONVERSIONS.md to document unit conversions done by the DIE

README.md

@@ -54,23 +54,19 @@ where `{parameter}` is the name of the parameter whose data is to be retrieved,
 
 
 #### Available Parameters
-The following parameters are currently available for retrieval:
-- waterlevels
-- arsenic
-- bicarbonate
-- calcium
-- carbonate
-- chloride
-- fluoride
-- magnesium
-- nitrate
-- ph
-- potassium
-- silica
-- sodium
-- sulfate
-- tds
-- uranium
+|                            | waterlevels | arsenic | bicarbonate | calcium | carbonate | chloride | fluoride | magnesium | nitrate | ph  | potassium | silica | sodium | sulfate | tds | uranium |
+| -------------------------- | ----------- | ------- | ----------- | ------- | --------- | -------- | -------- | --------- | ------- | --- | --------- | ------ | ------ | ------- | --- | ------- |
+| **bernco**                 | X           | -       | -           | -       | -         | -        | -        | -         | -       | -   | -         | -      | -      | -       | -   | -       |
+| **bor**                    | -           | X       | -           | X       | -         | X        | X        | X         | X       | X   | X         | X      | X      | X       | X   | X       |
+| **cabq**                   | X           | -       | -           | -       | -         | -        | -        | -         | -       | -   | -         | -      | -      | -       | -   | -       |
+| **ebid**                   | X           | -       | -           | -       | -         | -        | -        | -         | -       | -   | -         | -      | -      | -       | -   | -       |
+| **nmbgmr-amp**             | X           | X       | X           | X       | X         | X        | X        | X         | X       | X   | X         | X      | X      | X       | X   | X       |
+| **nmed-dwb**               | -           | X       | X           | X       | -         | X        | X        | X         | X       | X   | X         | X      | X      | X       | X   | X       |
+| **nmose-isc-seven-rivers** | X           | -       | X           | X       | -         | X        | X        | X         | X       | X   | X         | X      | X      | X       | X   | -       |
+| **nmose-roswell**          | X           | -       | -           | -       | -         | -        | -        | -         | -       | -   | -         | -      | -      | -       | -   | -       |
+| **nwis**                   | X           | -       | -           | -       | -         | -        | -        | -         | -       | -   | -         | -      | -      | -       | -   | -       |
+| **pvacd**                  | X           | -       | -           | -       | -         | -        | -        | -         | -       | -   | -         | -      | -      | -       | -   | -       |
+| **wqp**                    | -           | X       | X           | X       | X         | X        | X        | X         | X       | X   | X         | X      | X      | X       | X   | X       |
 
 ### Output
 The `--output` option is required and used to set the output type:

backend/connectors/nmbgmr/source.py

@@ -52,7 +52,7 @@ def _make_url(endpoint):
 
 class NMBGMRSiteSource(BaseSiteSource):
     transformer_klass = NMBGMRSiteTransformer
-    chunk_size = 100
+    chunk_size = 10
     bounding_polygon = NM_STATE_BOUNDING_POLYGON
 
     def __repr__(self):
@@ -94,6 +94,9 @@ def get_records(self):
             site["properties"]["formation"] = well_data["formation"]
             site["properties"]["well_depth"] = well_data["well_depth_ftbgs"]
             site["properties"]["well_depth_units"] = FEET
+            # site["properties"]["formation"] = None
+            # site["properties"]["well_depth"] = None
+            # site["properties"]["well_depth_units"] = FEET
 
         return sites
 

backend/constants.py

@@ -14,7 +14,7 @@
 # limitations under the License.
 # ===============================================================================
 
-TDS = "tds"
+WATERLEVELS = "waterlevels"
 ARSENIC = "arsenic"
 BICARBONATE = "bicarbonate"
 CALCIUM = "calcium"
@@ -23,20 +23,19 @@
 FLUORIDE = "fluoride"
 MAGNESIUM = "magnesium"
 NITRATE = "nitrate"
+PH = "ph"
 POTASSIUM = "potassium"
 SILICA = "silica"
 SODIUM = "sodium"
 SULFATE = "sulfate"
+TDS = "tds"
 URANIUM = "uranium"
-WATERLEVELS = "waterlevels"
-
-
-PH = "ph"
 
 
 MILLIGRAMS_PER_LITER = "mg/L"
 MICROGRAMS_PER_LITER = "ug/L"
 PARTS_PER_MILLION = "ppm"
+PARTS_PER_BILLION = "ppb"
 TONS_PER_ACRE_FOOT = "tons/ac ft"
 FEET = "ft"
 METERS = "m"
@@ -65,13 +64,13 @@
         FLUORIDE,
         MAGNESIUM,
         NITRATE,
+        PH,
         POTASSIUM,
         SILICA,
         SODIUM,
         SULFATE,
         TDS,
         URANIUM,
-        PH,
     ]
 )
 

backend/transformer.py

@@ -23,6 +23,7 @@
 from backend.constants import (
     MILLIGRAMS_PER_LITER,
     PARTS_PER_MILLION,
+    PARTS_PER_BILLION,
     FEET,
     METERS,
     TONS_PER_ACRE_FOOT,
@@ -181,6 +182,7 @@ def convert_units(
     mgl = MILLIGRAMS_PER_LITER.lower()
     ugl = MICROGRAMS_PER_LITER.lower()
     ppm = PARTS_PER_MILLION.lower()
+    ppb = PARTS_PER_BILLION.lower()
     tpaf = TONS_PER_ACRE_FOOT.lower()
     ft = FEET.lower()
     m = METERS.lower()
@@ -210,7 +212,7 @@ def convert_units(
             conversion_factor = 0.004427
         elif input_units == "pci/l":
             conversion_factor = 0.00149
-        elif input_units == ugl:
+        elif input_units in (ugl, ppb):
             conversion_factor = 0.001
         elif input_units == tpaf:
             conversion_factor = 735.47
@@ -239,7 +241,7 @@ def convert_units(
         return input_value, conversion_factor, warning
 
 
-def standardize_datetime(dt):
+def standardize_datetime(dt, record_id):
     if isinstance(dt, tuple):
         dt = [di for di in dt if di is not None]
         dt = " ".join(dt)
@@ -274,7 +276,7 @@ def standardize_datetime(dt):
                 except ValueError as e:
                     pass
         else:
-            raise ValueError(f"Failed to parse datetime {dt}")
+            raise ValueError(f"Failed to parse datetime {dt} for {record_id}")
 
     if fmt == "%Y-%m-%d":
         return dt.strftime("%Y-%m-%d"), ""
@@ -405,13 +407,13 @@ def do_transform(
         # standardize datetime
         dt = record.get(DT_MEASURED)
         if dt:
-            d, t = standardize_datetime(dt)
+            d, t = standardize_datetime(dt, record["id"])
             record["date_measured"] = d
             record["time_measured"] = t
         else:
             mrd = record.get("most_recent_datetime")
             if mrd:
-                d, t = standardize_datetime(mrd)
+                d, t = standardize_datetime(mrd, record["id"])
                 record["date_measured"] = d
                 record["time_measured"] = t
 
@@ -661,7 +663,7 @@ def _transform(self, record, site_record):
         rec = {}
 
         if self.config.output_summary:
-            self._transform_most_recents(record)
+            self._transform_most_recents(record, site_record.id)
 
             parameter, units = self._get_parameter_name_and_units()
             rec.update(
@@ -693,9 +695,9 @@ def _transform(self, record, site_record):
         rec.update(source_id)
         return rec
 
-    def _transform_most_recents(self, record):
+    def _transform_most_recents(self, record, site_id):
         # convert most_recents
-        dt, tt = standardize_datetime(record["most_recent_datetime"])
+        dt, tt = standardize_datetime(record["most_recent_datetime"], site_id)
         record["most_recent_date"] = dt
         record["most_recent_time"] = tt
         parameter_name, unit = self._get_parameter_name_and_units()

frontend/cli.py

@@ -255,7 +255,7 @@ def weave(
     config.output_timeseries_separated = timeseries_separated
 
     # sources
-    if parameter.lower() == "waterlevels":
+    if parameter == "waterlevels":
         config.use_source_bernco = no_bernco
         config.use_source_cabq = no_cabq
         config.use_source_ebid = no_ebid
@@ -268,7 +268,49 @@ def weave(
         config.use_source_bor = False
         config.use_source_nmed_dwb = False
         config.use_source_wqp = False
-    else:
+
+    elif parameter == "carbonate":
+        config.use_source_nmbgmr_amp = no_nmbgmr_amp
+        config.use_source_wqp = no_wqp
+
+        config.use_source_bor = False
+        config.use_source_bernco = False
+        config.use_source_cabq = False
+        config.use_source_ebid = False
+        config.use_source_nmed_dwb = False
+        config.use_source_nmose_isc_seven_rivers = False
+        config.use_source_nmose_roswell = False
+        config.use_source_nwis = False
+        config.use_source_pvacd = False
+
+    elif parameter in ["arsenic", "uranium"]:
+        config.use_source_bor = no_bor
+        config.use_source_nmbgmr_amp = no_nmbgmr_amp
+        config.use_source_nmed_dwb = no_nmed_dwb
+        config.use_source_wqp = no_wqp
+
+        config.use_source_bernco = False
+        config.use_source_cabq = False
+        config.use_source_ebid = False
+        config.use_source_nmose_isc_seven_rivers = False
+        config.use_source_nmose_roswell = False
+        config.use_source_nwis = False
+        config.use_source_pvacd = False
+
+    elif parameter in [
+        "bicarbonate",
+        "calcium",
+        "chloride",
+        "fluoride",
+        "magnesium",
+        "nitrate",
+        "ph",
+        "potassium",
+        "silica",
+        "sodium",
+        "sulfate",
+        "tds",
+    ]:
         config.use_source_bor = no_bor
         config.use_source_nmbgmr_amp = no_nmbgmr_amp
         config.use_source_nmed_dwb = no_nmed_dwb

setup.py

@@ -21,7 +21,7 @@
 
 setup(
     name="nmuwd",
-    version="0.6.0",
+    version="0.7.0",
     author="Jake Ross",
     description="New Mexico Water Data Integration Engine",
     long_description=long_description,