Merge pull request #609 from DataIntegrationGroup/jab-bdms-626-field-activity

BDMS 626: groundwater level field activity & test fixes

Author: jacob-a-brown

schemas/well_inventory.py

@@ -34,6 +34,7 @@
     MonitoringStatus,
     SampleMethod,
     DataQuality,
+    GroundwaterLevelReason,
 )
 from phonenumbers import NumberParseException
 from pydantic import (
@@ -182,6 +183,10 @@ def validator(v):
 DataQualityField: TypeAlias = Annotated[
     Optional[DataQuality], BeforeValidator(flexible_lexicon_validator(DataQuality))
 ]
+GroundwaterLevelReasonField: TypeAlias = Annotated[
+    Optional[GroundwaterLevelReason],
+    BeforeValidator(flexible_lexicon_validator(GroundwaterLevelReason)),
+]
 PostalCodeField: TypeAlias = Annotated[
     Optional[str], BeforeValidator(postal_code_or_none)
 ]
@@ -326,7 +331,7 @@ class WellInventoryRow(BaseModel):
         default=None,
         validation_alias=AliasChoices("mp_height", "mp_height_ft"),
     )
-    level_status: Optional[str] = None
+    level_status: GroundwaterLevelReasonField = None
     depth_to_water_ft: OptionalFloat = None
     data_quality: DataQualityField = None
     water_level_notes: Optional[str] = None

services/well_inventory_csv.py

@@ -445,7 +445,7 @@ def _find_existing_imported_well(
             .where(
                 Thing.name == model.well_name_point_id,
                 Thing.thing_type == "water well",
-                FieldActivity.activity_type == "well inventory",
+                FieldActivity.activity_type == "groundwater level",
                 Sample.sample_name == sample_name,
             )
             .order_by(Thing.id.asc())
@@ -795,14 +795,23 @@ def _add_csv_row(session: Session, group: Group, model: WellInventoryRow, user)
             session.add(parameter)
             session.flush()
 
+        # create FieldActivity
+        gwl_field_activity = FieldActivity(
+            field_event=fe,
+            activity_type="groundwater level",
+            notes="Groundwater level measurement activity conducted during well inventory field event.",
+        )
+        session.add(gwl_field_activity)
+        session.flush()
+
         # create Sample
         sample_method = (
             model.sample_method.value
             if hasattr(model.sample_method, "value")
             else (model.sample_method or "Unknown")
         )
         sample = Sample(
-            field_activity_id=fa.id,
+            field_activity_id=gwl_field_activity.id,
             sample_date=model.measurement_date_time,
             sample_name=f"{well.name}-WL-{model.measurement_date_time.strftime('%Y%m%d%H%M')}",
             sample_matrix="groundwater",
@@ -813,13 +822,19 @@ def _add_csv_row(session: Session, group: Group, model: WellInventoryRow, user)
         session.flush()
 
         # create Observation
+        # TODO: groundwater_level_reason may be conditionally required for null depth_to_water_ft - handle accordingly
         observation = Observation(
             sample_id=sample.id,
             parameter_id=parameter.id,
             value=model.depth_to_water_ft,
             unit="ft",
             observation_datetime=model.measurement_date_time,
             measuring_point_height=model.mp_height,
+            groundwater_level_reason=(
+                model.level_status.value
+                if hasattr(model.level_status, "value")
+                else None
+            ),
             nma_data_quality=(
                 model.data_quality.value
                 if hasattr(model.data_quality, "value")

tests/test_well_inventory.py

@@ -14,6 +14,7 @@
 import pytest
 from cli.service_adapter import well_inventory_csv
 from core.constants import SRID_UTM_ZONE_13N, SRID_WGS84
+from core.enums import Role, ContactType
 from db import (
     Base,
     Location,
@@ -67,7 +68,7 @@ def isolate_well_inventory_tables():
     _reset_well_inventory_tables()
 
 
-def test_well_inventory_db_contents():
+def test_well_inventory_db_contents_no_waterlevels():
     """
     Test that the well inventory upload creates the correct database contents.
 
@@ -457,6 +458,77 @@ def test_well_inventory_db_contents():
                     assert participant.participant.name == file_content["field_staff_2"]
 
 
+def test_well_inventory_db_contents_with_waterlevels(tmp_path):
+    """
+    Tests that the following records are made:
+
+    - field event
+    - field activity for well inventory
+    - field activity for water level measurement
+    - field participants
+    - contact
+    - location
+    - thing
+    - sample
+    - observation
+
+    """
+    row = _minimal_valid_well_inventory_row()
+    row.update(
+        {
+            "water_level_date_time": "2025-02-15T10:30:00",
+            "depth_to_water_ft": "8",
+            "sample_method": "Steel-tape measurement",
+            "data_quality": "Water level accurate to within two hundreths of a foot",
+            "water_level_notes": "Attempted measurement",
+            "mp_height_ft": 2.5,
+            "level_status": "Water level not affected",
+        }
+    )
+    file_path = tmp_path / "well-inventory-blank-depth.csv"
+    with file_path.open("w", encoding="utf-8", newline="") as f:
+        writer = csv.DictWriter(f, fieldnames=list(row.keys()))
+        writer.writeheader()
+        writer.writerow(row)
+
+    result = well_inventory_csv(file_path)
+    assert result.exit_code == 0, result.stderr
+
+    with session_ctx() as session:
+        field_events = session.query(FieldEvent).all()
+        field_activities = session.query(FieldActivity).all()
+        field_event_participants = session.query(FieldEventParticipant).all()
+        contacts = session.query(Contact).all()
+        locations = session.query(Location).all()
+        things = session.query(Thing).all()
+        samples = session.query(Sample).all()
+        observations = session.query(Observation).all()
+
+        assert len(field_events) == 1
+        assert len(field_activities) == 2
+        activity_types = {fa.activity_type for fa in field_activities}
+        assert activity_types == {
+            "well inventory",
+            "groundwater level",
+        }, f"Unexpected activity types: {activity_types}"
+        gwl_field_activity = next(
+            (fa for fa in field_activities if fa.activity_type == "groundwater level"),
+            None,
+        )
+        assert gwl_field_activity is not None
+
+        assert len(field_event_participants) == 1
+        assert len(contacts) == 1
+        assert len(locations) == 1
+        assert len(things) == 1
+        assert len(samples) == 1
+        sample = samples[0]
+        assert sample.field_activity == gwl_field_activity
+        assert len(observations) == 1
+        observation = observations[0]
+        assert observation.sample == sample
+
+
 def test_blank_depth_to_water_still_creates_water_level_records(tmp_path):
     """Blank depth-to-water is treated as missing while preserving the attempted measurement."""
     row = _minimal_valid_well_inventory_row()
@@ -486,9 +558,9 @@ def test_blank_depth_to_water_still_creates_water_level_records(tmp_path):
 
         assert len(samples) == 1
         assert len(observations) == 1
-        assert samples[0].sample_date == datetime.fromisoformat("2025-02-15T10:30:00")
+        assert samples[0].sample_date == datetime.fromisoformat("2025-02-15T10:30:00Z")
         assert observations[0].observation_datetime == datetime.fromisoformat(
-            "2025-02-15T10:30:00"
+            "2025-02-15T10:30:00Z"
         )
         assert observations[0].value is None
         assert observations[0].measuring_point_height == 2.5
@@ -779,8 +851,8 @@ def test_make_contact_with_full_info(self):
         model.contact_special_requests_notes = "Call before visiting"
         model.contact_1_name = "John Doe"
         model.contact_1_organization = "Test Org"
-        model.contact_1_role = "Owner"
-        model.contact_1_type = "Primary"
+        model.contact_1_role = Role.Owner
+        model.contact_1_type = ContactType.Primary
         model.contact_1_email_1 = "john@example.com"
         model.contact_1_email_1_type = "Work"
         model.contact_1_email_2 = None
@@ -866,8 +938,8 @@ def test_make_contact_with_organization_only(self):
         model.contact_special_requests_notes = None
         model.contact_1_name = None
         model.contact_1_organization = "Test Org"
-        model.contact_1_role = None
-        model.contact_1_type = None
+        model.contact_1_role = Role.Owner
+        model.contact_1_type = ContactType.Primary
         model.contact_1_email_1 = None
         model.contact_1_email_1_type = None
         model.contact_1_email_2 = None
@@ -1094,7 +1166,7 @@ def test_water_level_aliases_are_mapped(self):
                 "sample_method": "Steel-tape measurement",
                 "water_level_date_time": "2025-02-15T10:30:00",
                 "mp_height_ft": 2.5,
-                "level_status": "Static",
+                "level_status": "Other conditions exist that would affect the level (remarks)",
                 "depth_to_water_ft": 11.2,
                 "data_quality": "Water level accurate to within two hundreths of a foot",
                 "water_level_notes": "Initial reading",
@@ -1131,6 +1203,8 @@ def test_blank_contact_organization_is_treated_as_none(self):
         row = _minimal_valid_well_inventory_row()
         row["contact_1_name"] = "Test Contact"
         row["contact_1_organization"] = ""
+        row["contact_1_role"] = "Owner"
+        row["contact_1_type"] = "Primary"
 
         model = WellInventoryRow(**row)