From ef96f7b22621f54eef93aede5fcff705aa543ec0 Mon Sep 17 00:00:00 2001 From: Kelsey Smuczynski Date: Thu, 26 Feb 2026 13:45:17 -0700 Subject: [PATCH 1/4] feat(core): add legacy site notes field to lexicon - Added "Site Notes (legacy)" term with associated category `note_type` and definition. --- core/lexicon.json | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/core/lexicon.json b/core/lexicon.json index 2f325282..2a37686e 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -8206,6 +8206,13 @@ "term": "OwnerComment", "definition": "Legacy owner comments field" }, + { + "categories": [ + "note_type" + ], + "term": "Site Notes (legacy)", + "definition": "Legacy site notes field from WaterLevels" + }, { "categories": [ "well_pump_type" From b533da4481825e9a5fec4543f0ff4074b8bd4302 Mon Sep 17 00:00:00 2001 From: Kelsey Smuczynski Date: Thu, 26 Feb 2026 13:45:55 -0700 Subject: [PATCH 2/4] feat(db, schemas): add support for legacy site notes - Added `site_notes` property to `Thing` model and schema to handle "Site Notes (legacy)". - Ensured alignment with existing note retrieval and schema structure. --- db/thing.py | 4 ++++ schemas/thing.py | 1 + 2 files changed, 5 insertions(+) diff --git a/db/thing.py b/db/thing.py index db2419c3..c3c7c02d 100644 --- a/db/thing.py +++ b/db/thing.py @@ -434,6 +434,10 @@ def sampling_procedure_notes(self): def construction_notes(self): return self._get_notes("Construction") + @property + def site_notes(self): + return self._get_notes("Site Notes (legacy)") + @property def well_status(self) -> str | None: """ diff --git a/schemas/thing.py b/schemas/thing.py index fceba6c0..ad109bf0 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -211,6 +211,7 @@ class BaseThingResponse(BaseResponseModel): monitoring_frequencies: list[MonitoringFrequencyResponse] = [] general_notes: list[NoteResponse] = [] sampling_procedure_notes: list[NoteResponse] = [] + site_notes: list[NoteResponse] = [] @field_validator("monitoring_frequencies", mode="before") def remove_records_with_end_date(cls, monitoring_frequencies): From a58fc7543af4f54ae41e61e628694525a342f798 Mon Sep 17 00:00:00 2001 From: Kelsey Smuczynski Date: Thu, 26 Feb 2026 13:46:35 -0700 Subject: [PATCH 3/4] feat(transfers): add support for legacy SiteNotes handling - Insert legacy `SiteNotes` in the `Notes` table during transfer process. - Updated stats to track the number of `notes_created`. --- transfers/waterlevels_transfer.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/transfers/waterlevels_transfer.py b/transfers/waterlevels_transfer.py index 9c45cf26..50bd24b9 100644 --- a/transfers/waterlevels_transfer.py +++ b/transfers/waterlevels_transfer.py @@ -33,6 +33,7 @@ Contact, FieldEventParticipant, Parameter, + Notes, ) from db.engine import session_ctx from transfers.transferer import Transferer @@ -158,6 +159,7 @@ def _transfer_hook(self, session: Session) -> None: "observations_created": 0, "contacts_created": 0, "contacts_reused": 0, + "notes_created": 0, } gwd = self.cleaned_df.groupby(["PointID"]) @@ -396,6 +398,26 @@ def _transfer_hook(self, session: Session) -> None: session.execute(insert(Observation), observation_rows) stats["observations_created"] += len(observation_rows) + # Site Notes (legacy) + site_notes = { + prep["row"].SiteNotes + for prep in prepared_rows + if hasattr(prep["row"], "SiteNotes") + and prep["row"].SiteNotes + and str(prep["row"].SiteNotes).strip() + } + for note_content in site_notes: + session.add( + Notes( + target_table="thing", + target_id=thing_id, + note_type="Site Notes (legacy)", + content=str(note_content).strip(), + release_status="public", + ) + ) + stats["notes_created"] += 1 + session.commit() session.expunge_all() stats["groups_processed"] += 1 From e6a8f2176de3805d19050bc24f1c760621972e5d Mon Sep 17 00:00:00 2001 From: Kelsey Smuczynski Date: Thu, 26 Feb 2026 16:16:28 -0700 Subject: [PATCH 4/4] fix(transfers): handle duplicate legacy SiteNotes with date context - Track legacy `SiteNotes` by content and date to preserve context for duplicates. - Updated insertion to prepend the date to note content for uniqueness. --- transfers/waterlevels_transfer.py | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/transfers/waterlevels_transfer.py b/transfers/waterlevels_transfer.py index 50bd24b9..5ab4819a 100644 --- a/transfers/waterlevels_transfer.py +++ b/transfers/waterlevels_transfer.py @@ -399,20 +399,32 @@ def _transfer_hook(self, session: Session) -> None: stats["observations_created"] += len(observation_rows) # Site Notes (legacy) - site_notes = { - prep["row"].SiteNotes - for prep in prepared_rows - if hasattr(prep["row"], "SiteNotes") - and prep["row"].SiteNotes - and str(prep["row"].SiteNotes).strip() - } - for note_content in site_notes: + # If there are duplicate notes for a single point ID, we only create one note. + # However, if some duplicates are "time stamped" (meaning they are attached to + # rows with different dates), we should ideally preserve that context. + # The current implementation prepends the date to the note content + # to ensure that duplicate content from different dates remains distinct. + unique_notes: dict[str, datetime] = {} + for prep in prepared_rows: + if hasattr(prep["row"], "SiteNotes") and prep["row"].SiteNotes: + content = str(prep["row"].SiteNotes).strip() + if content: + dt = prep["dt_utc"] + # We keep all notes that have different content OR different dates + # Actually, if content is same but date is different, we want to see it. + # So we key by (content, date) + key = (content, dt.date()) + if key not in unique_notes: + unique_notes[key] = dt + + for (content, _), dt in unique_notes.items(): + date_prefix = dt.strftime("%Y-%m-%d") session.add( Notes( target_table="thing", target_id=thing_id, note_type="Site Notes (legacy)", - content=str(note_content).strip(), + content=f"{date_prefix}: {content}", release_status="public", ) )