Merge pull request #46 from DataIntegrationGroup/dev/jab

Author: jirhiker

.github/dependabot.yml

@@ -0,0 +1,11 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
+
+version: 2
+updates:
+  - package-ecosystem: "pip" # See documentation for possible values
+    directory: "/" # Location of package manifests
+    schedule:
+      interval: "weekly"

.github/workflows/cicd.yml

@@ -5,9 +5,9 @@ name: CI/CD
 
 on:
   push:
-    branches: [ "main", "feature/jir", "dev/jab"]
+    branches: [ "pre-production", "feature/jir", "dev/jab"]
   pull_request:
-    branches: [ "main"]
+    branches: [ "pre-production"]
 
 permissions:
   contents: read
@@ -18,26 +18,39 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
+
     - name: Set up Python 3.10
-      uses: actions/setup-python@v3
+      uses: actions/setup-python@v4
       with:
         python-version: "3.10"
         cache: "pip"
+
+    - name: Cache pip dependencies
+      uses: actions/cache@v3
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
+        restore-keys: |
+          ${{ runner.os }}-pip-
+
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
         pip install flake8 pytest mypy
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+
     - name: Lint with flake8
       run: |
         # stop the build if there are Python syntax errors or undefined names
         flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
         # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
         flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+
     - name: Type check with mypy
       run: |
         mypy . --ignore-missing-imports
+
     - name: Test with pytest
       run: |
         pytest -s -rx tests

.github/workflows/publish-to-pypi.yml

@@ -3,10 +3,12 @@ name: Publish Python 🐍 distributions 📦 to PyPI and TestPyPI
 # pushes only occur on successful pull request merges
 # this makes it so that the main branch gets published to PyPi, not the
 # target branch from the pull request (pre-production)
+
+# pushing tags should be the only way to trigger this workflow
 on:
   push:
-    branches:
-      - main
+    tags:
+      - '*'
 
 jobs:
   build-and-publish-if-merged:
@@ -20,7 +22,7 @@ jobs:
         uses: actions/setup-python@v1
         with:
           python-version: 3.9
-          cache: 'pip'
+
       - name: Install pypa/build
         run: >-
             python -m

CHANGELOG.md

@@ -4,7 +4,24 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## Unreleased: 0.8.0
+## 0.9.2
+
+### Added
+- `--sites-only` flag to only retrieve site data
+- `--output-format` flag to write out sites/summary tables as csv or geojson.
+  - options are `csv` or `geojson`
+  - timeseries data is always written to a csv
+- NM OSE POD data for sites.
+  - can be removed from output with `--no-nmose-pod`
+- `--output-dir` to change the output directory to a location other than `.` (the current working directory)
+
+### Changed
+- `output` to `output-type` for CLI
+
+### Fixed
+- a bug with `--site-limit`. it now exports the number of sets requested by the 
+
+## 0.8.0
 
 ### Added
 - water level for WQP

README.md

@@ -2,7 +2,7 @@
 [![Format code](https://github.com/DataIntegrationGroup/PyWeaver/actions/workflows/format_code.yml/badge.svg?branch=main)](https://github.com/DataIntegrationGroup/PyWeaver/actions/workflows/format_code.yml)
 [![Publish Python 🐍 distributions 📦 to PyPI and TestPyPI](https://github.com/DataIntegrationGroup/PyWeaver/actions/workflows/publish-to-pypi.yml/badge.svg)](https://github.com/DataIntegrationGroup/PyWeaver/actions/workflows/publish-to-pypi.yml)
 [![CI/CD](https://github.com/DataIntegrationGroup/PyWeaver/actions/workflows/cicd.yml/badge.svg)](https://github.com/DataIntegrationGroup/PyWeaver/actions/workflows/cicd.yml)
-
+[![Dependabot Updates](https://github.com/DataIntegrationGroup/DataIntegrationEngine/actions/workflows/dependabot/dependabot-updates/badge.svg)](https://github.com/DataIntegrationGroup/DataIntegrationEngine/actions/workflows/dependabot/dependabot-updates)
 
 ![NMWDI](https://newmexicowaterdata.org/wp-content/uploads/2023/11/newmexicowaterdatalogoNov2023.png)
 ![NMBGMR](https://waterdata.nmt.edu/static/nmbgmr_logo_resized.png)
@@ -71,22 +71,22 @@ where `{parameter}` is the name of the parameter whose data is to be retrieved,
 | **pvacd**                  | X           | -       | -           | -       | -         | -        | -        | -         | -       | -   | -         | -      | -      | -       | -   | -       |
 | **wqp**                    | X           | X       | X           | X       | X         | X        | X        | X         | X       | X   | X         | X      | X      | X       | X   | X       |
 
-### Output
-The `--output` option is required and used to set the output type:
+### Output Type
+The `--output-type` option is required and used to set the output type:
 
 ```
---output summary
+--output-type summary
 ```
 - A summary table consisting of location information as well as summary statistics for the parameter of interest for every location that has observations.
 
 ```
---output timeseries_unified
+--output-type timeseries_unified
 ```
 - A single table consisting of time series data for all locations for the parameter of interest.
 - A single table of site data that contains information such as latitude, longitude, and elevation
 
 ```
---output timeseries_separated
+--output-type timeseries_separated
 ```
 - Separate time series tables for all locations for the parameter of interest.
 - A single table of site data that contains information such as latitude, longitude, and elevation
@@ -146,6 +146,7 @@ A log of the inputs and processes, called `die.log`, is also saved to the output
 | formation | geologic formation in which the well terminates | string | N |
 | aquifer | aquifer from which the well draws water | string | N |
 | well_depth | depth of well | float | N |
+| well_depth_units | units of well depth. Defaults to ft | string | N |
 
 <sup>**CABQ elevation is calculated as [elevation at top of casing] - [stickup height]; if stickup height < 0 the measuring point is assumed to be beneath the ground surface</sup>
 
@@ -180,7 +181,7 @@ The Data Integration Engine enables the user to obtain groundwater level and gro
 - `--no-pvacd` to exclude Pecos Valley Artesian Convservancy District (PVACD) data
 - `--no-wqp` to exclude Water Quality Portal (WQP) data
 
-### Geographic Filters
+### Geographic Filters [In Development]
 
 The following flags can be used to geographically filter data:
 
@@ -192,7 +193,11 @@ The following flags can be used to geographically filter data:
 -- bbox 'x1 y1, x2 y2'
 ```
 
-### Date Filters
+```
+-- wkt {wkt polygon or multipolygon}
+```
+
+### Date Filters [In Development]
 
 The following flags can be used to filter by dates:
 
@@ -214,12 +219,12 @@ die sources {parameter}
 
 to print the sources that report that parameter to the terminal.
 
-### Wells [In Development]
+### Sites
 
 Use
 
 ```
-die wells
+die sites
 ```
 
-to print wells to the terminal.
+to export site information only

backend/__init__.py

@@ -1,7 +1,16 @@
 from enum import Enum
+from os import environ
 
 
 class OutputFormat(str, Enum):
     GEOJSON = "geojson"
     CSV = "csv"
-    GEOSERVER = "geoserver"
+    GEOSERVER = "geoserver"
+
+
+def get_bool_env_variable(var: str) -> bool:
+    env_var = environ.get(var, None)
+    if env_var is None or env_var.strip().lower() not in ["true", "1", "yes"]:
+        return False
+    else:
+        return True

backend/config.py

@@ -18,6 +18,8 @@
 from datetime import datetime, timedelta
 from enum import Enum
 import shapely.wkt
+import yaml
+
 from . import OutputFormat
 from .bounding_polygons import get_county_polygon
 from .connectors.nmbgmr.source import (
@@ -68,7 +70,8 @@
 )
 from .connectors.usgs.source import NWISSiteSource, NWISWaterLevelSource
 from .connectors.wqp.source import WQPSiteSource, WQPAnalyteSource, WQPWaterLevelSource
-from .logger import Loggable
+from backend.logger import Loggable
+
 
 SOURCE_DICT = {
     "bernco": BernCoSiteSource,
@@ -98,9 +101,6 @@ def get_source(source):
         return klass()
 
 
-
-
-
 class Config(Loggable):
     site_limit: int = 0
     dry: bool = False
@@ -149,18 +149,14 @@ class Config(Loggable):
     analyte_output_units: str = MILLIGRAMS_PER_LITER
     waterlevel_output_units: str = FEET
 
-    # use_csv: bool = True
-    # use_geojson: bool = False
-
-    output_format: OutputFormat = OutputFormat.CSV
+    output_format: str = OutputFormat.CSV
 
-    yes: bool = True
+    yes: bool = False
 
     def __init__(self, model=None, payload=None, path=None):
         # need to initialize logger
         super().__init__()
 
-        self.bbox = {}
         if path:
             payload = self._load_from_yaml(path)
 
@@ -186,24 +182,27 @@ def __init__(self, model=None, payload=None, path=None):
                     if value is not None:
                         setattr(self, f"use_source_{sk}", value)
 
-            for attr in ("wkt", "county", "bbox",
-                         "output_summary",
-                         "output_timeseries_unified",
-                         "output_timeseries_separated",
-                         "start_date",
-                         "end_date",
-                         "parameter",
-                         "output_name",
-                         "dry",
-                         "latest_water_level_only",
-                         "output_format",
-                         "use_cloud_storage",
-                         "yes"):
+            for attr in (
+                "wkt",
+                "county",
+                "bbox",
+                "output_summary",
+                "output_timeseries_unified",
+                "output_timeseries_separated",
+                "start_date",
+                "end_date",
+                "parameter",
+                "output_name",
+                "dry",
+                "latest_water_level_only",
+                "output_format",
+                "use_cloud_storage",
+                "yes",
+            ):
                 if attr in payload:
                     setattr(self, attr, payload[attr])
 
     def _load_from_yaml(self, path):
-        import yaml
         path = os.path.abspath(path)
         if os.path.exists(path):
             self.log(f"Loading config from {path}")
@@ -221,7 +220,6 @@ def get_config_and_false_agencies(self):
                 "ebid",
                 "nmbgmr_amp",
                 "nmose_isc_seven_rivers",
-                "nmose_pod",
                 "nmose_roswell",
                 "nwis",
                 "pvacd",
@@ -450,7 +448,7 @@ def _report_attributes(title, attrs):
                 "output_horizontal_datum",
                 "output_elevation_units",
                 "use_cloud_storage",
-                "output_format"
+                "output_format",
             ),
         )
 
@@ -573,4 +571,6 @@ def output_path(self):
     def get(self, attr):
         if self._payload:
             return self._payload.get(attr)
+
+
 # ============= EOF =============================================