Merge pull request #466 from DataIntegrationGroup/water-level-csv-refactor

BDMS 412/529/530: water level csv implementation

Author: jirhiker

api/observation.py

@@ -41,7 +41,7 @@
     UpdateWaterChemistryObservation,
 )
 from schemas.transducer import TransducerObservationWithBlockResponse
-from schemas.water_level_csv import WaterLevelBulkUploadResponse
+from schemas.water_level_csv import WaterLevelBulkUploadPayload
 from services.crud_helper import model_deleter, model_adder
 from services.observation_helper import (
     get_observations,
@@ -90,8 +90,8 @@ async def add_water_chemistry_observation(
 
 @router.post(
     "/groundwater-level/bulk-upload",
-    response_model=WaterLevelBulkUploadResponse,
-    status_code=HTTP_200_OK,
+    response_model=WaterLevelBulkUploadPayload,
+    status_code=HTTP_201_CREATED,
 )
 async def bulk_upload_groundwater_levels(
     user: amp_admin_dependency,
@@ -107,7 +107,9 @@ async def bulk_upload_groundwater_levels(
     result = bulk_upload_water_levels(contents)
 
     if result.exit_code != 0:
-        raise HTTPException(status_code=HTTP_400_BAD_REQUEST, detail=result.payload)
+        raise HTTPException(
+            status_code=HTTP_400_BAD_REQUEST, detail=result.payload.model_dump()
+        )
 
     return result.payload
 

cli/cli.py

@@ -83,7 +83,6 @@ def water_levels_bulk_upload(file_path: str, output_format: str | None):
     """
     parse and upload a csv
     """
-    # TODO: use the same helper function used by api to parse and upload a WL csv
     from cli.service_adapter import water_levels_csv
 
     pretty_json = (output_format or "").lower() == "json"

schemas/water_level_csv.py

@@ -13,30 +13,140 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ===============================================================================
-from pydantic import BaseModel
+from datetime import datetime
+from pydantic import BaseModel, ConfigDict, field_validator, Field
+from typing import Any
 
+from core.enums import SampleMethod, GroundwaterLevelReason, GroundwaterLevelAccuracy
 
-class WaterLevelBulkUploadSummary(BaseModel):
-    total_rows_processed: int
-    total_rows_imported: int
-    validation_errors_or_warnings: int
+
+class WaterLevelCsvRow(BaseModel):
+    """
+    This class defines the schema for a single row in the water level CSV upload.
+    """
+
+    model_config = ConfigDict(extra="ignore", str_strip_whitespace=True)
+
+    well_name_point_id: str = Field(
+        description="Name/PointID of the well where the measurement was taken."
+    )
+    field_event_date_time: datetime = Field(
+        description="Date and time when the field event occurred."
+    )
+    field_staff: str = Field(description="Name of the person who led the field event.")
+    field_staff_2: str | None = Field(
+        description="Name of the second person who participated in the field event.",
+        default=None,
+    )
+    field_staff_3: str | None = Field(
+        description="Name of the third person who participated in the field event.",
+        default=None,
+    )
+    water_level_date_time: datetime = Field(
+        description="Date and time when the water level measurement was taken."
+    )
+    measuring_person: str = Field(
+        description="Person who took the water level measurement. They must be one of the field staff"
+    )
+    sample_method: SampleMethod = Field(
+        description="Method used to measure the water level."
+    )
+    mp_height: float = Field(
+        description="Measuring point height relative to the ground surface in feet."
+    )
+    level_status: GroundwaterLevelReason = Field(
+        description="Status of the water level."
+    )
+    depth_to_water_ft: float = Field(description="Depth to water in feet.")
+    data_quality: GroundwaterLevelAccuracy = Field(
+        description="A description of the accuracy of the data."
+    )
+    water_level_notes: str | None = Field(
+        description="Additional notes about the water level measurement.", default=None
+    )
+
+    @field_validator("water_level_notes", mode="before")
+    @classmethod
+    def _empty_to_none(cls, value: str | None) -> str | None:
+        if value is None:
+            return None
+        if isinstance(value, str) and value.strip() == "":
+            return None
+        return value
+
+    @field_validator("measuring_person")
+    @classmethod
+    def ensure_measuring_person_is_field_staff(
+        cls, value: str, values: dict[str, Any]
+    ) -> str:
+        data = values.data
+        field_staffs = [
+            data.get("field_staff"),
+            data.get("field_staff_2"),
+            data.get("field_staff_3"),
+        ]
+        if value not in field_staffs:
+            raise ValueError("measuring_person must be one of the field staff")
+        return value
+
+
+class WaterLevelBulkUploadRow(WaterLevelCsvRow):
+    """
+    This class extends WaterLevelCsvRow to include resolved database objects
+    for easier processing during bulk upload.
+    """
+
+    well: Any = Field(description="The Thing object representing the well.")
+    field_staff_contact: Any = Field(
+        description="The Contact object for the field staff."
+    )
+    field_staff_2_contact: Any | None = Field(
+        description="The Contact object for the second field staff."
+    )
+    field_staff_3_contact: Any | None = Field(
+        description="The Contact object for the third field staff."
+    )
+    measuring_person_field_staff_index: int = Field(
+        description="The index of the field staff who is the measuring person: 1, 2, or 3."
+    )
 
 
-class WaterLevelBulkUploadRow(BaseModel):
+class WaterLevelCreatedRow(BaseModel):
+    """
+    This class defines the structure of a successfully created water level row
+    during bulk upload.
+    """
+
     well_name_point_id: str
     field_event_id: int
     field_activity_id: int
+    field_event_participant_1_id: int
+    field_event_participant_2_id: int | None
+    field_event_participant_3_id: int | None
     sample_id: int
     observation_id: int
-    measurement_date_time: str
-    level_status: str
-    data_quality: str
+    water_level_date_time: str
+    groundwater_level_reason: str
+    groundwater_level_accuracy: str
 
 
-class WaterLevelBulkUploadResponse(BaseModel):
+class WaterLevelBulkUploadSummary(BaseModel):
+    total_rows_processed: int
+    total_rows_imported: int
+    total_validation_errors_or_warnings: int
+
+
+class WaterLevelBulkUploadPayload(BaseModel):
     summary: WaterLevelBulkUploadSummary
-    water_levels: list[WaterLevelBulkUploadRow]
+    water_levels: list[WaterLevelCreatedRow]
     validation_errors: list[str]
 
 
+class WaterLevelBulkUploadResponse(BaseModel):
+    exit_code: int
+    stdout: str
+    stderr: str
+    payload: WaterLevelBulkUploadPayload
+
+
 # ============= EOF =============================================