From bab253f4f11bb1f01da5e60e23a8229f9df77779 Mon Sep 17 00:00:00 2001 From: jakeross Date: Tue, 10 Feb 2026 20:31:57 +1100 Subject: [PATCH 01/12] fix: improve data handling in initializers and transferer, streamline constructor in radionuclides --- ..._add_sample_point_fields_to_minor_trace.py | 37 +++++++++++++++++++ core/initializers.py | 4 +- db/nma_legacy.py | 8 ++++ tests/test_minor_trace_chemistry_transfer.py | 2 + tests/test_nma_chemistry_lineage.py | 6 +++ transfers/minor_trace_chemistry_transfer.py | 15 ++++++++ transfers/radionuclides.py | 8 +--- transfers/transferer.py | 11 +++--- 8 files changed, 78 insertions(+), 13 deletions(-) create mode 100644 alembic/versions/e71807682f57_add_sample_point_fields_to_minor_trace.py diff --git a/alembic/versions/e71807682f57_add_sample_point_fields_to_minor_trace.py b/alembic/versions/e71807682f57_add_sample_point_fields_to_minor_trace.py new file mode 100644 index 000000000..e089272ba --- /dev/null +++ b/alembic/versions/e71807682f57_add_sample_point_fields_to_minor_trace.py @@ -0,0 +1,37 @@ +"""add sample point fields to minor trace + +Revision ID: e71807682f57 +Revises: h1b2c3d4e5f6 +Create Date: 2026-02-10 20:07:25.586385 + +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = "e71807682f57" +down_revision: Union[str, Sequence[str], None] = "h1b2c3d4e5f6" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + op.add_column( + "NMA_MinorTraceChemistry", + sa.Column("nma_SamplePtID", postgresql.UUID(as_uuid=True), nullable=False), + ) + op.add_column( + "NMA_MinorTraceChemistry", + sa.Column("nma_SamplePointID", sa.String(length=10), nullable=False), + ) + + +def downgrade() -> None: + """Downgrade schema.""" + op.drop_column("NMA_MinorTraceChemistry", "nma_SamplePointID") + op.drop_column("NMA_MinorTraceChemistry", "nma_SamplePtID") diff --git a/core/initializers.py b/core/initializers.py index 4ffbfb744..c3fe058fc 100644 --- a/core/initializers.py +++ b/core/initializers.py @@ -168,6 +168,7 @@ def init_lexicon(path: str = None) -> None: ) association_rows = [] + seen_links = set() for term_dict in terms: term_id = existing_terms.get(term_dict["term"]) if term_id is None: @@ -177,8 +178,9 @@ def init_lexicon(path: str = None) -> None: if category_id is None: continue key = (term_id, category_id) - if key in existing_links: + if key in existing_links or key in seen_links: continue + seen_links.add(key) association_rows.append( {"term_id": term_id, "category_id": category_id} ) diff --git a/db/nma_legacy.py b/db/nma_legacy.py index 557c415ad..c603633c9 100644 --- a/db/nma_legacy.py +++ b/db/nma_legacy.py @@ -774,6 +774,8 @@ class NMA_MinorTraceChemistry(Base): - nma_global_id: Original UUID PK, now UNIQUE for audit - chemistry_sample_info_id: Integer FK to NMA_Chemistry_SampleInfo.id - nma_chemistry_sample_info_uuid: Legacy UUID FK for audit + - nma_sample_pt_id: Legacy UUID FK (SamplePtID) for audit + - nma_sample_point_id: Legacy SamplePointID string - nma_wclab_id: Legacy WCLab_ID string (audit) """ @@ -807,6 +809,12 @@ class NMA_MinorTraceChemistry(Base): ) # Additional columns + nma_sample_pt_id: Mapped[Optional[uuid.UUID]] = mapped_column( + "nma_SamplePtID", UUID(as_uuid=True), nullable=False + ) + nma_sample_point_id: Mapped[Optional[str]] = mapped_column( + "nma_SamplePointID", String(10), nullable=False + ) analyte: Mapped[Optional[str]] = mapped_column("analyte", String(50)) symbol: Mapped[Optional[str]] = mapped_column("symbol", String(10)) sample_value: Mapped[Optional[float]] = mapped_column("sample_value", Float) diff --git a/tests/test_minor_trace_chemistry_transfer.py b/tests/test_minor_trace_chemistry_transfer.py index 2d38e1a19..10959f797 100644 --- a/tests/test_minor_trace_chemistry_transfer.py +++ b/tests/test_minor_trace_chemistry_transfer.py @@ -36,3 +36,5 @@ def test_row_to_dict_includes_wclab_id(): row_dict = transfer._row_to_dict(row) assert row_dict["nma_WCLab_ID"] == "LAB-123" + assert row_dict["nma_SamplePtID"] == sample_pt_id + assert row_dict["nma_SamplePointID"] == "POINT-1" diff --git a/tests/test_nma_chemistry_lineage.py b/tests/test_nma_chemistry_lineage.py index 4ad4a8ea7..a66812900 100644 --- a/tests/test_nma_chemistry_lineage.py +++ b/tests/test_nma_chemistry_lineage.py @@ -134,6 +134,8 @@ def test_nma_minor_trace_chemistry_columns(): "id", # Integer PK "nma_global_id", # Legacy UUID "chemistry_sample_info_id", # Integer FK + "nma_sample_pt_id", # Legacy UUID FK + "nma_sample_point_id", # Legacy sample point id # from legacy "analyte", "sample_value", @@ -173,6 +175,8 @@ def test_nma_minor_trace_chemistry_save_all_columns(shared_thing): mtc = NMA_MinorTraceChemistry( nma_global_id=_next_global_id(), chemistry_sample_info=sample_info, + nma_sample_pt_id=sample_info.nma_sample_pt_id, + nma_sample_point_id=sample_info.nma_sample_point_id, analyte="As", sample_value=0.015, units="mg/L", @@ -193,6 +197,8 @@ def test_nma_minor_trace_chemistry_save_all_columns(shared_thing): assert mtc.id is not None # Integer PK assert mtc.nma_global_id is not None # Legacy UUID assert mtc.chemistry_sample_info_id == sample_info.id # Integer FK + assert mtc.nma_sample_pt_id == sample_info.nma_sample_pt_id + assert mtc.nma_sample_point_id == sample_info.nma_sample_point_id assert mtc.analyte == "As" assert mtc.sample_value == 0.015 assert mtc.units == "mg/L" diff --git a/transfers/minor_trace_chemistry_transfer.py b/transfers/minor_trace_chemistry_transfer.py index ed1d16da7..c6dcf491d 100644 --- a/transfers/minor_trace_chemistry_transfer.py +++ b/transfers/minor_trace_chemistry_transfer.py @@ -25,6 +25,8 @@ - nma_global_id: Legacy UUID PK (GlobalID), UNIQUE for audit - chemistry_sample_info_id: Integer FK to NMA_Chemistry_SampleInfo.id - nma_chemistry_sample_info_uuid: Legacy UUID FK for audit +- nma_sample_pt_id: Legacy UUID FK (SamplePtID) for audit +- nma_sample_point_id: Legacy SamplePointID string """ from __future__ import annotations @@ -147,6 +149,8 @@ def _transfer_hook(self, session: Session) -> None: set_={ "chemistry_sample_info_id": excluded.chemistry_sample_info_id, "nma_chemistry_sample_info_uuid": excluded.nma_chemistry_sample_info_uuid, + "nma_SamplePtID": excluded.nma_SamplePtID, + "nma_SamplePointID": excluded.nma_SamplePointID, "sample_value": excluded.sample_value, "units": excluded.units, "symbol": excluded.symbol, @@ -176,6 +180,15 @@ def _row_to_dict(self, row) -> Optional[dict[str, Any]]: ) return None + sample_point_id = self._safe_str(row, "SamplePointID") + if sample_point_id is None: + self._capture_error( + getattr(row, "SamplePointID", None), + f"Missing SamplePointID for SamplePtID: {legacy_sample_pt_id}", + "SamplePointID", + ) + return None + # Look up Integer FK from cache chemistry_sample_info_id = self._sample_info_cache.get(legacy_sample_pt_id) if chemistry_sample_info_id is None: @@ -203,6 +216,8 @@ def _row_to_dict(self, row) -> Optional[dict[str, Any]]: "chemistry_sample_info_id": chemistry_sample_info_id, # Legacy UUID FK for audit "nma_chemistry_sample_info_uuid": legacy_sample_pt_id, + "nma_SamplePtID": legacy_sample_pt_id, + "nma_SamplePointID": sample_point_id, # Data columns "analyte": self._safe_str(row, "Analyte"), "sample_value": self._safe_float(row, "SampleValue"), diff --git a/transfers/radionuclides.py b/transfers/radionuclides.py index 8b4ad9dfc..1a8713ec8 100644 --- a/transfers/radionuclides.py +++ b/transfers/radionuclides.py @@ -38,7 +38,6 @@ from db import NMA_Radionuclides from transfers.logger import logger from transfers.transferer import ChemistryTransferer -from transfers.util import read_csv class RadionuclidesTransferer(ChemistryTransferer): @@ -50,15 +49,10 @@ class RadionuclidesTransferer(ChemistryTransferer): source_table = "Radionuclides" - def __init__(self, *args, batch_size: int = 1000, **kwargs): + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._parse_dates = ["AnalysisDate"] - def _get_dfs(self) -> tuple[pd.DataFrame, pd.DataFrame]: - input_df = read_csv(self.source_table, parse_dates=self._parse_dates) - cleaned_df = self._filter_to_valid_sample_infos(input_df) - return input_df, cleaned_df - def _transfer_hook(self, session: Session) -> None: row_dicts = [] skipped_global_id = 0 diff --git a/transfers/transferer.py b/transfers/transferer.py index e6fe93e34..afef86e34 100644 --- a/transfers/transferer.py +++ b/transfers/transferer.py @@ -314,7 +314,7 @@ def _build_sample_info_cache(self) -> None: ) def _get_dfs(self) -> tuple[pd.DataFrame, pd.DataFrame]: - input_df = read_csv(self.source_table, parse_dates=self._parse_dates) + input_df = self._read_csv(self.source_table, parse_dates=self._parse_dates) cleaned_df = self._filter_to_valid_sample_infos(input_df) return input_df, cleaned_df @@ -332,10 +332,10 @@ def _filter_to_valid_sample_infos(self, df: pd.DataFrame) -> pd.DataFrame: inverted_df = df[~mask].copy() if not inverted_df.empty: for _, row in inverted_df.iterrows(): - pointid = row["SamplePointID"] + sample_pt_id = row.get("SamplePtID") self._capture_error( - pointid, - f"No matching ChemistrySampleInfo for SamplePtID: {pointid}", + sample_pt_id, + f"No matching ChemistrySampleInfo for SamplePtID: {sample_pt_id}", "SamplePtID", ) @@ -343,8 +343,9 @@ def _filter_to_valid_sample_infos(self, df: pd.DataFrame) -> pd.DataFrame: if before_count > after_count: skipped = before_count - after_count + table_name = self.source_table or self.__class__.__name__ logger.warning( - f"Filtered out {skipped} FieldParameters records without matching " + f"Filtered out {skipped} {table_name} records without matching " f"ChemistrySampleInfo ({after_count} valid, {skipped} orphan records prevented)" ) From e75b396529b37611b8fde7cc6ac092f1f9021587 Mon Sep 17 00:00:00 2001 From: jakeross Date: Tue, 10 Feb 2026 20:46:11 +1100 Subject: [PATCH 02/12] fix: remove legacy SamplePtID references and update schema to use SamplePointID --- .../e71807682f57_add_sample_point_fields_to_minor_trace.py | 6 ------ db/nma_legacy.py | 4 ---- tests/test_minor_trace_chemistry_transfer.py | 1 - tests/test_nma_chemistry_lineage.py | 3 --- transfers/minor_trace_chemistry_transfer.py | 3 --- 5 files changed, 17 deletions(-) diff --git a/alembic/versions/e71807682f57_add_sample_point_fields_to_minor_trace.py b/alembic/versions/e71807682f57_add_sample_point_fields_to_minor_trace.py index e089272ba..3ce78b238 100644 --- a/alembic/versions/e71807682f57_add_sample_point_fields_to_minor_trace.py +++ b/alembic/versions/e71807682f57_add_sample_point_fields_to_minor_trace.py @@ -10,7 +10,6 @@ import sqlalchemy as sa from alembic import op -from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision: str = "e71807682f57" @@ -21,10 +20,6 @@ def upgrade() -> None: """Upgrade schema.""" - op.add_column( - "NMA_MinorTraceChemistry", - sa.Column("nma_SamplePtID", postgresql.UUID(as_uuid=True), nullable=False), - ) op.add_column( "NMA_MinorTraceChemistry", sa.Column("nma_SamplePointID", sa.String(length=10), nullable=False), @@ -34,4 +29,3 @@ def upgrade() -> None: def downgrade() -> None: """Downgrade schema.""" op.drop_column("NMA_MinorTraceChemistry", "nma_SamplePointID") - op.drop_column("NMA_MinorTraceChemistry", "nma_SamplePtID") diff --git a/db/nma_legacy.py b/db/nma_legacy.py index c603633c9..e5f199d0a 100644 --- a/db/nma_legacy.py +++ b/db/nma_legacy.py @@ -774,7 +774,6 @@ class NMA_MinorTraceChemistry(Base): - nma_global_id: Original UUID PK, now UNIQUE for audit - chemistry_sample_info_id: Integer FK to NMA_Chemistry_SampleInfo.id - nma_chemistry_sample_info_uuid: Legacy UUID FK for audit - - nma_sample_pt_id: Legacy UUID FK (SamplePtID) for audit - nma_sample_point_id: Legacy SamplePointID string - nma_wclab_id: Legacy WCLab_ID string (audit) """ @@ -809,9 +808,6 @@ class NMA_MinorTraceChemistry(Base): ) # Additional columns - nma_sample_pt_id: Mapped[Optional[uuid.UUID]] = mapped_column( - "nma_SamplePtID", UUID(as_uuid=True), nullable=False - ) nma_sample_point_id: Mapped[Optional[str]] = mapped_column( "nma_SamplePointID", String(10), nullable=False ) diff --git a/tests/test_minor_trace_chemistry_transfer.py b/tests/test_minor_trace_chemistry_transfer.py index 10959f797..87b6a1d7c 100644 --- a/tests/test_minor_trace_chemistry_transfer.py +++ b/tests/test_minor_trace_chemistry_transfer.py @@ -36,5 +36,4 @@ def test_row_to_dict_includes_wclab_id(): row_dict = transfer._row_to_dict(row) assert row_dict["nma_WCLab_ID"] == "LAB-123" - assert row_dict["nma_SamplePtID"] == sample_pt_id assert row_dict["nma_SamplePointID"] == "POINT-1" diff --git a/tests/test_nma_chemistry_lineage.py b/tests/test_nma_chemistry_lineage.py index a66812900..78ec4c6d8 100644 --- a/tests/test_nma_chemistry_lineage.py +++ b/tests/test_nma_chemistry_lineage.py @@ -134,7 +134,6 @@ def test_nma_minor_trace_chemistry_columns(): "id", # Integer PK "nma_global_id", # Legacy UUID "chemistry_sample_info_id", # Integer FK - "nma_sample_pt_id", # Legacy UUID FK "nma_sample_point_id", # Legacy sample point id # from legacy "analyte", @@ -175,7 +174,6 @@ def test_nma_minor_trace_chemistry_save_all_columns(shared_thing): mtc = NMA_MinorTraceChemistry( nma_global_id=_next_global_id(), chemistry_sample_info=sample_info, - nma_sample_pt_id=sample_info.nma_sample_pt_id, nma_sample_point_id=sample_info.nma_sample_point_id, analyte="As", sample_value=0.015, @@ -197,7 +195,6 @@ def test_nma_minor_trace_chemistry_save_all_columns(shared_thing): assert mtc.id is not None # Integer PK assert mtc.nma_global_id is not None # Legacy UUID assert mtc.chemistry_sample_info_id == sample_info.id # Integer FK - assert mtc.nma_sample_pt_id == sample_info.nma_sample_pt_id assert mtc.nma_sample_point_id == sample_info.nma_sample_point_id assert mtc.analyte == "As" assert mtc.sample_value == 0.015 diff --git a/transfers/minor_trace_chemistry_transfer.py b/transfers/minor_trace_chemistry_transfer.py index c6dcf491d..af7913a69 100644 --- a/transfers/minor_trace_chemistry_transfer.py +++ b/transfers/minor_trace_chemistry_transfer.py @@ -25,7 +25,6 @@ - nma_global_id: Legacy UUID PK (GlobalID), UNIQUE for audit - chemistry_sample_info_id: Integer FK to NMA_Chemistry_SampleInfo.id - nma_chemistry_sample_info_uuid: Legacy UUID FK for audit -- nma_sample_pt_id: Legacy UUID FK (SamplePtID) for audit - nma_sample_point_id: Legacy SamplePointID string """ @@ -149,7 +148,6 @@ def _transfer_hook(self, session: Session) -> None: set_={ "chemistry_sample_info_id": excluded.chemistry_sample_info_id, "nma_chemistry_sample_info_uuid": excluded.nma_chemistry_sample_info_uuid, - "nma_SamplePtID": excluded.nma_SamplePtID, "nma_SamplePointID": excluded.nma_SamplePointID, "sample_value": excluded.sample_value, "units": excluded.units, @@ -216,7 +214,6 @@ def _row_to_dict(self, row) -> Optional[dict[str, Any]]: "chemistry_sample_info_id": chemistry_sample_info_id, # Legacy UUID FK for audit "nma_chemistry_sample_info_uuid": legacy_sample_pt_id, - "nma_SamplePtID": legacy_sample_pt_id, "nma_SamplePointID": sample_point_id, # Data columns "analyte": self._safe_str(row, "Analyte"), From 27bb085a76a74868e86bbb6735d32df1e77027e3 Mon Sep 17 00:00:00 2001 From: jakeross Date: Tue, 10 Feb 2026 21:03:26 +1100 Subject: [PATCH 03/12] fix: add nma_sample_point_id to NMA_MinorTraceChemistry instances in tests --- tests/integration/test_admin_minor_trace_chemistry.py | 1 + tests/test_nma_chemistry_lineage.py | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/tests/integration/test_admin_minor_trace_chemistry.py b/tests/integration/test_admin_minor_trace_chemistry.py index fcdcd539a..f5cf0d0fa 100644 --- a/tests/integration/test_admin_minor_trace_chemistry.py +++ b/tests/integration/test_admin_minor_trace_chemistry.py @@ -104,6 +104,7 @@ def minor_trace_chemistry_record(): chemistry = NMA_MinorTraceChemistry( nma_global_id=uuid.uuid4(), chemistry_sample_info_id=sample_info.id, # Integer FK + nma_sample_point_id=sample_info.nma_sample_point_id, analyte="Arsenic", symbol="As", sample_value=0.005, diff --git a/tests/test_nma_chemistry_lineage.py b/tests/test_nma_chemistry_lineage.py index 78ec4c6d8..f0853958d 100644 --- a/tests/test_nma_chemistry_lineage.py +++ b/tests/test_nma_chemistry_lineage.py @@ -401,6 +401,7 @@ def test_assign_sample_info_to_mtc(shared_thing): mtc = NMA_MinorTraceChemistry( nma_global_id=_next_global_id(), chemistry_sample_info=sample_info, # OO: assign object + nma_sample_point_id=sample_info.nma_sample_point_id, analyte="Pb", ) session.add(mtc) @@ -433,6 +434,7 @@ def test_append_mtc_to_sample_info(shared_thing): mtc = NMA_MinorTraceChemistry( nma_global_id=_next_global_id(), + nma_sample_point_id=sample_info.nma_sample_point_id, analyte="Fe", ) sample_info.minor_trace_chemistries.append(mtc) @@ -454,6 +456,7 @@ def test_mtc_requires_chemistry_sample_info(): with session_ctx() as session: mtc = NMA_MinorTraceChemistry( nma_global_id=_next_global_id(), + nma_sample_point_id=_next_sample_point_id(), analyte="Cu", # No chemistry_sample_info_id - should fail ) @@ -487,6 +490,7 @@ def test_full_lineage_navigation(shared_thing): mtc = NMA_MinorTraceChemistry( nma_global_id=_next_global_id(), chemistry_sample_info=sample_info, + nma_sample_point_id=sample_info.nma_sample_point_id, analyte="Zn", ) session.add(mtc) @@ -524,6 +528,7 @@ def test_reverse_lineage_navigation(shared_thing): mtc = NMA_MinorTraceChemistry( nma_global_id=_next_global_id(), chemistry_sample_info=sample_info, + nma_sample_point_id=sample_info.nma_sample_point_id, analyte="Mn", ) session.add(mtc) @@ -560,6 +565,7 @@ def test_cascade_delete_sample_info_deletes_mtc(shared_thing): mtc = NMA_MinorTraceChemistry( nma_global_id=_next_global_id(), chemistry_sample_info=sample_info, + nma_sample_point_id=sample_info.nma_sample_point_id, analyte="Cd", ) session.add(mtc) @@ -681,11 +687,13 @@ def test_multiple_mtc_per_sample_info(shared_thing): mtc1 = NMA_MinorTraceChemistry( nma_global_id=_next_global_id(), chemistry_sample_info=sample_info, + nma_sample_point_id=sample_info.nma_sample_point_id, analyte="As", ) mtc2 = NMA_MinorTraceChemistry( nma_global_id=_next_global_id(), chemistry_sample_info=sample_info, + nma_sample_point_id=sample_info.nma_sample_point_id, analyte="Pb", ) session.add_all([mtc1, mtc2]) From addee9d04e01eef5951e2ecb2c49033c519e134d Mon Sep 17 00:00:00 2001 From: jakeross Date: Tue, 10 Feb 2026 21:09:12 +1100 Subject: [PATCH 04/12] fix: remove limit handling from data transfer methods in field_parameters_transfer and minor_trace_chemistry_transfer --- transfers/field_parameters_transfer.py | 3 --- transfers/minor_trace_chemistry_transfer.py | 3 --- 2 files changed, 6 deletions(-) diff --git a/transfers/field_parameters_transfer.py b/transfers/field_parameters_transfer.py index 3a894222e..adc8f23f4 100644 --- a/transfers/field_parameters_transfer.py +++ b/transfers/field_parameters_transfer.py @@ -57,10 +57,7 @@ def _transfer_hook(self, session: Session) -> None: Uses ON CONFLICT DO UPDATE on nma_GlobalID (legacy UUID PK, now UNIQUE). """ - limit = self.flags.get("LIMIT", 0) df = self.cleaned_df - if limit > 0: - df = df.head(limit) row_dicts = [] for row in df.itertuples(): diff --git a/transfers/minor_trace_chemistry_transfer.py b/transfers/minor_trace_chemistry_transfer.py index af7913a69..c19fe2509 100644 --- a/transfers/minor_trace_chemistry_transfer.py +++ b/transfers/minor_trace_chemistry_transfer.py @@ -116,10 +116,7 @@ def _transfer_hook(self, session: Session) -> None: Uses ON CONFLICT DO UPDATE on nma_GlobalID (the legacy UUID PK, now UNIQUE). """ - limit = self.flags.get("LIMIT", 0) df = self.cleaned_df - if limit > 0: - df = df.head(limit) # Convert rows to dicts row_dicts = [] From 6890e45412203468b0096da3f65ce1d599e13ac2 Mon Sep 17 00:00:00 2001 From: Jake Ross Date: Thu, 12 Feb 2026 11:20:07 -0700 Subject: [PATCH 05/12] Update alembic/versions/e71807682f57_add_sample_point_fields_to_minor_trace.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- ..._add_sample_point_fields_to_minor_trace.py | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/alembic/versions/e71807682f57_add_sample_point_fields_to_minor_trace.py b/alembic/versions/e71807682f57_add_sample_point_fields_to_minor_trace.py index 3ce78b238..531286dd8 100644 --- a/alembic/versions/e71807682f57_add_sample_point_fields_to_minor_trace.py +++ b/alembic/versions/e71807682f57_add_sample_point_fields_to_minor_trace.py @@ -20,12 +20,32 @@ def upgrade() -> None: """Upgrade schema.""" + # Step 1: add the column as nullable with a temporary default so existing rows get a value. op.add_column( "NMA_MinorTraceChemistry", - sa.Column("nma_SamplePointID", sa.String(length=10), nullable=False), + sa.Column( + "nma_SamplePointID", + sa.String(length=10), + nullable=True, + server_default="", + ), ) + # Step 2: enforce NOT NULL now that all existing rows have a non-NULL value. + op.alter_column( + "NMA_MinorTraceChemistry", + "nma_SamplePointID", + existing_type=sa.String(length=10), + nullable=False, + ) + # Step 3: drop the temporary default so future inserts must supply a value explicitly. + op.alter_column( + "NMA_MinorTraceChemistry", + "nma_SamplePointID", + existing_type=sa.String(length=10), + server_default=None, + ) def downgrade() -> None: """Downgrade schema.""" op.drop_column("NMA_MinorTraceChemistry", "nma_SamplePointID") From 3aa742dd628a7dcff3c89c5bb16d8c4e2eb5e388 Mon Sep 17 00:00:00 2001 From: jirhiker Date: Thu, 12 Feb 2026 18:20:26 +0000 Subject: [PATCH 06/12] Formatting changes --- .../e71807682f57_add_sample_point_fields_to_minor_trace.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/alembic/versions/e71807682f57_add_sample_point_fields_to_minor_trace.py b/alembic/versions/e71807682f57_add_sample_point_fields_to_minor_trace.py index 531286dd8..4648235f2 100644 --- a/alembic/versions/e71807682f57_add_sample_point_fields_to_minor_trace.py +++ b/alembic/versions/e71807682f57_add_sample_point_fields_to_minor_trace.py @@ -46,6 +46,8 @@ def upgrade() -> None: existing_type=sa.String(length=10), server_default=None, ) + + def downgrade() -> None: """Downgrade schema.""" op.drop_column("NMA_MinorTraceChemistry", "nma_SamplePointID") From 0eb415d75f447cc803737bf45a00f66bbf0a114a Mon Sep 17 00:00:00 2001 From: Jake Ross Date: Thu, 12 Feb 2026 11:25:06 -0700 Subject: [PATCH 07/12] Update db/nma_legacy.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- db/nma_legacy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/nma_legacy.py b/db/nma_legacy.py index e5f199d0a..f07942b15 100644 --- a/db/nma_legacy.py +++ b/db/nma_legacy.py @@ -808,7 +808,7 @@ class NMA_MinorTraceChemistry(Base): ) # Additional columns - nma_sample_point_id: Mapped[Optional[str]] = mapped_column( + nma_sample_point_id: Mapped[str] = mapped_column( "nma_SamplePointID", String(10), nullable=False ) analyte: Mapped[Optional[str]] = mapped_column("analyte", String(50)) From 732c79aa12dad399a653c058ad3d14fb4c71dd1e Mon Sep 17 00:00:00 2001 From: Jake Ross Date: Thu, 12 Feb 2026 11:25:21 -0700 Subject: [PATCH 08/12] Update transfers/minor_trace_chemistry_transfer.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- transfers/minor_trace_chemistry_transfer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transfers/minor_trace_chemistry_transfer.py b/transfers/minor_trace_chemistry_transfer.py index c19fe2509..53ff3a3db 100644 --- a/transfers/minor_trace_chemistry_transfer.py +++ b/transfers/minor_trace_chemistry_transfer.py @@ -145,7 +145,7 @@ def _transfer_hook(self, session: Session) -> None: set_={ "chemistry_sample_info_id": excluded.chemistry_sample_info_id, "nma_chemistry_sample_info_uuid": excluded.nma_chemistry_sample_info_uuid, - "nma_SamplePointID": excluded.nma_SamplePointID, + "nma_sample_point_id": excluded.nma_sample_point_id, "sample_value": excluded.sample_value, "units": excluded.units, "symbol": excluded.symbol, From 99124f566010465efc926782e6268fa971069830 Mon Sep 17 00:00:00 2001 From: Jake Ross Date: Thu, 12 Feb 2026 11:25:48 -0700 Subject: [PATCH 09/12] Update transfers/minor_trace_chemistry_transfer.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- transfers/minor_trace_chemistry_transfer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transfers/minor_trace_chemistry_transfer.py b/transfers/minor_trace_chemistry_transfer.py index 53ff3a3db..916845568 100644 --- a/transfers/minor_trace_chemistry_transfer.py +++ b/transfers/minor_trace_chemistry_transfer.py @@ -211,7 +211,7 @@ def _row_to_dict(self, row) -> Optional[dict[str, Any]]: "chemistry_sample_info_id": chemistry_sample_info_id, # Legacy UUID FK for audit "nma_chemistry_sample_info_uuid": legacy_sample_pt_id, - "nma_SamplePointID": sample_point_id, + "nma_sample_point_id": sample_point_id, # Data columns "analyte": self._safe_str(row, "Analyte"), "sample_value": self._safe_float(row, "SampleValue"), From 9ebf6bd0b4fe3bf3ab7c75aab2e91801898ece12 Mon Sep 17 00:00:00 2001 From: Jake Ross Date: Thu, 12 Feb 2026 11:29:05 -0700 Subject: [PATCH 10/12] Update alembic/versions/e71807682f57_add_sample_point_fields_to_minor_trace.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../e71807682f57_add_sample_point_fields_to_minor_trace.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alembic/versions/e71807682f57_add_sample_point_fields_to_minor_trace.py b/alembic/versions/e71807682f57_add_sample_point_fields_to_minor_trace.py index 4648235f2..c8cb463dc 100644 --- a/alembic/versions/e71807682f57_add_sample_point_fields_to_minor_trace.py +++ b/alembic/versions/e71807682f57_add_sample_point_fields_to_minor_trace.py @@ -27,7 +27,7 @@ def upgrade() -> None: "nma_SamplePointID", sa.String(length=10), nullable=True, - server_default="", + server_default=sa.text("''"), ), ) From b0e1c061545eec50cbb6dacb3ed5ab9ff5777fe4 Mon Sep 17 00:00:00 2001 From: jakeross Date: Thu, 12 Feb 2026 11:29:52 -0700 Subject: [PATCH 11/12] ```text fix: update error handling for missing SamplePointID in MinorTraceChemistryTransferer ``` --- tests/test_minor_trace_chemistry_transfer.py | 36 ++++++++++++++++++++ transfers/minor_trace_chemistry_transfer.py | 2 +- 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/tests/test_minor_trace_chemistry_transfer.py b/tests/test_minor_trace_chemistry_transfer.py index 87b6a1d7c..78f7c612c 100644 --- a/tests/test_minor_trace_chemistry_transfer.py +++ b/tests/test_minor_trace_chemistry_transfer.py @@ -37,3 +37,39 @@ def test_row_to_dict_includes_wclab_id(): row_dict = transfer._row_to_dict(row) assert row_dict["nma_WCLab_ID"] == "LAB-123" assert row_dict["nma_SamplePointID"] == "POINT-1" + + +def test_row_to_dict_missing_sample_point_id_returns_none_and_captures_error(): + # Bypass __init__ so we can stub the cache without hitting the DB. + transfer = MinorTraceChemistryTransferer.__new__(MinorTraceChemistryTransferer) + sample_pt_id = uuid.uuid4() + transfer._sample_info_cache = {sample_pt_id: 1} + transfer.flags = {} + transfer.errors = [] + + row = pd.Series( + { + "SamplePtID": str(sample_pt_id), + "GlobalID": str(uuid.uuid4()), + # SamplePointID intentionally missing + "Analyte": "Ca", + "SampleValue": 10.5, + "Units": "mg/L", + "Symbol": None, + "AnalysisMethod": "ICP", + "AnalysisDate": "2024-01-01 00:00:00.000", + "Notes": "note", + "AnalysesAgency": "Lab", + "Uncertainty": 0.1, + "Volume": "2", + "VolumeUnit": "L", + "WCLab_ID": "LAB-123", + } + ) + + row_dict = transfer._row_to_dict(row) + assert row_dict is None + assert len(transfer.errors) == 1 + error = transfer.errors[0] + assert error["field"] == "SamplePointID" + assert "Missing SamplePointID" in error["error"] diff --git a/transfers/minor_trace_chemistry_transfer.py b/transfers/minor_trace_chemistry_transfer.py index 916845568..230767929 100644 --- a/transfers/minor_trace_chemistry_transfer.py +++ b/transfers/minor_trace_chemistry_transfer.py @@ -178,7 +178,7 @@ def _row_to_dict(self, row) -> Optional[dict[str, Any]]: sample_point_id = self._safe_str(row, "SamplePointID") if sample_point_id is None: self._capture_error( - getattr(row, "SamplePointID", None), + legacy_sample_pt_id, f"Missing SamplePointID for SamplePtID: {legacy_sample_pt_id}", "SamplePointID", ) From a5827ab9a1f9b138c08137d90be856f06862df9e Mon Sep 17 00:00:00 2001 From: jakeross Date: Thu, 12 Feb 2026 11:36:24 -0700 Subject: [PATCH 12/12] ```text fix: update test to use nma_sample_point_id instead of nma_SamplePointID ``` --- tests/test_minor_trace_chemistry_transfer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_minor_trace_chemistry_transfer.py b/tests/test_minor_trace_chemistry_transfer.py index 78f7c612c..58ecc01ec 100644 --- a/tests/test_minor_trace_chemistry_transfer.py +++ b/tests/test_minor_trace_chemistry_transfer.py @@ -36,7 +36,7 @@ def test_row_to_dict_includes_wclab_id(): row_dict = transfer._row_to_dict(row) assert row_dict["nma_WCLab_ID"] == "LAB-123" - assert row_dict["nma_SamplePointID"] == "POINT-1" + assert row_dict["nma_sample_point_id"] == "POINT-1" def test_row_to_dict_missing_sample_point_id_returns_none_and_captures_error():