Merge pull request #537 from DataIntegrationGroup/BDMS-576-Verify-that-all-data-associated-with-relevant-PointIDs-was-transferred-from-SQL-to-Postgres-successfully

BDMS-576: Add transfer accountability tooling, duplicate WellData comparison, transfer hardening/perf, and nullable schema updates

Author: jirhiker

.github/workflows/tests.yml

@@ -104,6 +104,7 @@ jobs:
       BASE_URL: http://localhost:8000
       SESSION_SECRET_KEY: supersecretkeyforunittests
       AUTHENTIK_DISABLE_AUTHENTICATION: 1
+      DROP_AND_REBUILD_DB: 1
 
     services:
       postgis:

.gitignore

@@ -45,6 +45,7 @@ transfers/logs/*
 run_bdd-local.sh
 .pre-commit-config.local.yaml
 .serena/
+cli/logs
 
 # deployment files
 app.yaml

alembic/versions/43bc34504ee6_merge_migrations_after_staging_merge.py

@@ -1,7 +1,7 @@
 """Add unique index for NGWMN well construction
 
 Revision ID: 50d1c2a3b4c5
-Revises: 43bc34504ee6
+Revises: 3cb924ca51fd
 Create Date: 2026-01-31 00:27:12.204176
 
 """
@@ -12,7 +12,7 @@
 
 # revision identifiers, used by Alembic.
 revision: str = "50d1c2a3b4c5"
-down_revision: Union[str, Sequence[str], None] = "43bc34504ee6"
+down_revision: Union[str, Sequence[str], None] = "3cb924ca51fd"
 branch_labels: Union[str, Sequence[str], None] = None
 depends_on: Union[str, Sequence[str], None] = None
 

alembic/versions/50d1c2a3b4c5_add_unique_index_ngwmn_wellconstruction.py

@@ -0,0 +1,36 @@
+"""make measuring_point_history.measuring_point_height nullable
+
+Revision ID: 8c9d0e1f2a3b
+Revises: 5336a52336df
+Create Date: 2026-02-21 12:00:00.000000
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "8c9d0e1f2a3b"
+down_revision: Union[str, Sequence[str], None] = "5336a52336df"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.alter_column(
+        "measuring_point_history",
+        "measuring_point_height",
+        existing_type=sa.Numeric(),
+        nullable=True,
+    )
+
+
+def downgrade() -> None:
+    op.alter_column(
+        "measuring_point_history",
+        "measuring_point_height",
+        existing_type=sa.Numeric(),
+        nullable=False,
+    )

alembic/versions/8c9d0e1f2a3b_make_measuring_point_height_nullable.py

@@ -0,0 +1,36 @@
+"""make address.postal_code nullable
+
+Revision ID: 9a0b1c2d3e4f
+Revises: 8c9d0e1f2a3b
+Create Date: 2026-02-21 13:00:00.000000
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "9a0b1c2d3e4f"
+down_revision: Union[str, Sequence[str], None] = "8c9d0e1f2a3b"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.alter_column(
+        "address",
+        "postal_code",
+        existing_type=sa.String(length=20),
+        nullable=True,
+    )
+
+
+def downgrade() -> None:
+    op.alter_column(
+        "address",
+        "postal_code",
+        existing_type=sa.String(length=20),
+        nullable=False,
+    )

alembic/versions/9a0b1c2d3e4f_make_address_postal_code_nullable.py

@@ -0,0 +1,36 @@
+"""make deployment installation_date nullable
+
+Revision ID: a1b2c3d4e5f7
+Revises: 9a0b1c2d3e4f
+Create Date: 2026-02-21 14:32:00.000000
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "a1b2c3d4e5f7"
+down_revision: Union[str, Sequence[str], None] = "9a0b1c2d3e4f"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.alter_column(
+        "deployment",
+        "installation_date",
+        existing_type=sa.Date(),
+        nullable=True,
+    )
+
+
+def downgrade() -> None:
+    op.alter_column(
+        "deployment",
+        "installation_date",
+        existing_type=sa.Date(),
+        nullable=False,
+    )

alembic/versions/a1b2c3d4e5f7_make_deployment_installation_date_nullable.py

@@ -0,0 +1,48 @@
+"""make wellscreen depth fields nullable
+
+Revision ID: b3c4d5e6f7a8
+Revises: a1b2c3d4e5f7
+Create Date: 2026-02-21 15:20:00.000000
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "b3c4d5e6f7a8"
+down_revision: Union[str, Sequence[str], None] = "a1b2c3d4e5f7"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.alter_column(
+        "well_screen",
+        "screen_depth_top",
+        existing_type=sa.Float(),
+        nullable=True,
+    )
+    op.alter_column(
+        "well_screen",
+        "screen_depth_bottom",
+        existing_type=sa.Float(),
+        nullable=True,
+    )
+
+
+def downgrade() -> None:
+    op.alter_column(
+        "well_screen",
+        "screen_depth_bottom",
+        existing_type=sa.Float(),
+        nullable=False,
+    )
+    op.alter_column(
+        "well_screen",
+        "screen_depth_top",
+        existing_type=sa.Float(),
+        nullable=False,
+    )

alembic/versions/b3c4d5e6f7a8_make_wellscreen_depths_nullable.py

@@ -0,0 +1,48 @@
+"""make address.city and address.state nullable
+
+Revision ID: c4d5e6f7a8b9
+Revises: b3c4d5e6f7a8
+Create Date: 2026-02-21 16:30:00.000000
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "c4d5e6f7a8b9"
+down_revision: Union[str, Sequence[str], None] = "b3c4d5e6f7a8"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.alter_column(
+        "address",
+        "city",
+        existing_type=sa.String(length=100),
+        nullable=True,
+    )
+    op.alter_column(
+        "address",
+        "state",
+        existing_type=sa.String(length=50),
+        nullable=True,
+    )
+
+
+def downgrade() -> None:
+    op.alter_column(
+        "address",
+        "city",
+        existing_type=sa.String(length=100),
+        nullable=False,
+    )
+    op.alter_column(
+        "address",
+        "state",
+        existing_type=sa.String(length=50),
+        nullable=False,
+    )

alembic/versions/c4d5e6f7a8b9_make_address_city_state_nullable.py

@@ -0,0 +1,18 @@
+# API
+
+This directory contains FastAPI route modules grouped by resource/domain.
+
+## Structure
+
+- One module per domain (for example `thing.py`, `contact.py`, `observation.py`)
+- `api/ogc/` contains OGC-specific endpoints
+
+## Guidelines
+
+- Keep endpoints focused on transport concerns (request/response, status codes).
+- Put transfer/business logic in service or transfer modules.
+- Ensure response schemas match `schemas/` definitions.
+
+## Running locally
+
+Use project entrypoint from repo root (see top-level README for full setup).