Skip to content

Commit 0d51464

Browse files
jirhikerjirhiker
authored
Merge pull request #580 from DataIntegrationGroup/jir-feature-collections
Jir feature collections
2 parents a18c1d8 + 945d140 commit 0d51464

10 files changed

Lines changed: 768 additions & 28 deletions

‎.github/workflows/tests.yml‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ jobs:
3535
- name: Check out source repository
3636
uses: actions/checkout@v6.0.2
3737

38-
- name: Start database (PostGIS + pg_cron)
38+
- name: Start database (PostGIS)
3939
run: |
4040
docker compose build db
4141
docker compose up -d db
@@ -81,7 +81,6 @@ jobs:
8181
PGPASSWORD=postgres psql -h localhost -p 5432 -U postgres -tc "SELECT 1 FROM pg_database WHERE datname = 'ocotilloapi_test'" | grep -q 1 || \
8282
PGPASSWORD=postgres psql -h localhost -p 5432 -U postgres -c "CREATE DATABASE ocotilloapi_test"
8383
PGPASSWORD=postgres psql -h localhost -p 5432 -U postgres -d ocotilloapi_test -c "CREATE EXTENSION IF NOT EXISTS postgis"
84-
PGPASSWORD=postgres psql -h localhost -p 5432 -U postgres -d ocotilloapi_test -c "CREATE EXTENSION IF NOT EXISTS pg_cron"
8584
8685
- name: Run tests
8786
run: uv run pytest -vv --durations=20 --cov --cov-report=xml --junitxml=junit.xml --ignore=tests/transfers
@@ -121,7 +120,7 @@ jobs:
121120
- name: Check out source repository
122121
uses: actions/checkout@v6.0.2
123122

124-
- name: Start database (PostGIS + pg_cron)
123+
- name: Start database (PostGIS)
125124
run: |
126125
docker compose build db
127126
docker compose up -d db
@@ -167,7 +166,6 @@ jobs:
167166
PGPASSWORD=postgres psql -h localhost -p 5432 -U postgres -tc "SELECT 1 FROM pg_database WHERE datname = 'ocotilloapi_test'" | grep -q 1 || \
168167
PGPASSWORD=postgres psql -h localhost -p 5432 -U postgres -c "CREATE DATABASE ocotilloapi_test"
169168
PGPASSWORD=postgres psql -h localhost -p 5432 -U postgres -d ocotilloapi_test -c "CREATE EXTENSION IF NOT EXISTS postgis"
170-
PGPASSWORD=postgres psql -h localhost -p 5432 -U postgres -d ocotilloapi_test -c "CREATE EXTENSION IF NOT EXISTS pg_cron"
171169
172170
- name: Run BDD tests
173171
run: uv run behave tests/features --tags="@backend and @production and not @skip" --no-capture

‎alembic/versions/d5e6f7a8b9c0_create_pygeoapi_supporting_views.py‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -71,9 +71,7 @@ def _create_thing_view(view_id: str, thing_type: str) -> str:
7171
SELECT
7272
t.id,
7373
t.name,
74-
t.thing_type,
7574
t.first_visit_date,
76-
t.spring_type,
7775
t.nma_pk_welldata,
7876
t.well_depth,
7977
t.hole_depth,
@@ -87,6 +85,7 @@ def _create_thing_view(view_id: str, thing_type: str) -> str:
8785
t.formation_completion_code,
8886
t.nma_formation_zone,
8987
t.release_status,
88+
l.elevation,
9089
l.point
9190
FROM thing AS t
9291
JOIN latest_location AS ll ON ll.thing_id = t.id
@@ -152,7 +151,7 @@ def _create_avg_tds_view() -> str:
152151
SELECT
153152
csi.thing_id,
154153
mc.id AS major_chemistry_id,
155-
mc."AnalysisDate" AS analysis_date,
154+
COALESCE(mc."AnalysisDate", csi."CollectionDate")::date AS observation_date,
156155
mc."SampleValue" AS sample_value,
157156
mc."Units" AS units
158157
FROM "NMA_MajorChemistry" AS mc
@@ -176,8 +175,8 @@ def _create_avg_tds_view() -> str:
176175
t.thing_type,
177176
COUNT(to2.major_chemistry_id)::integer AS tds_observation_count,
178177
AVG(to2.sample_value)::double precision AS avg_tds_value,
179-
MIN(to2.analysis_date) AS first_tds_observation_datetime,
180-
MAX(to2.analysis_date) AS latest_tds_observation_datetime,
178+
MIN(to2.observation_date) AS first_tds_observation_date,
179+
MAX(to2.observation_date) AS last_tds_observation_date,
181180
l.point
182181
FROM tds_obs AS to2
183182
JOIN thing AS t ON t.id = to2.thing_id

‎alembic/versions/i2b3c4d5e6f7_add_latest_tds_pygeoapi_materialized_view.py‎

Lines changed: 229 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,229 @@
1+
"""add latest tds pygeoapi materialized view
2+
3+
Revision ID: i2b3c4d5e6f7
4+
Revises: d5e6f7a8b9c0
5+
Create Date: 2026-03-02 11:00:00.000000
6+
"""
7+
8+
from typing import Sequence, Union
9+
10+
from alembic import op
11+
from sqlalchemy import inspect, text
12+
13+
# revision identifiers, used by Alembic.
14+
revision: str = "i2b3c4d5e6f7"
15+
down_revision: Union[str, Sequence[str], None] = "d5e6f7a8b9c0"
16+
branch_labels: Union[str, Sequence[str], None] = None
17+
depends_on: Union[str, Sequence[str], None] = None
18+
19+
LATEST_LOCATION_CTE = """
20+
SELECT DISTINCT ON (lta.thing_id)
21+
lta.thing_id,
22+
lta.location_id,
23+
lta.effective_start
24+
FROM location_thing_association AS lta
25+
WHERE lta.effective_end IS NULL
26+
ORDER BY lta.thing_id, lta.effective_start DESC
27+
""".strip()
28+
29+
30+
def _create_latest_tds_view() -> str:
31+
return f"""
32+
CREATE VIEW ogc_latest_tds_wells AS
33+
WITH latest_location AS (
34+
{LATEST_LOCATION_CTE}
35+
),
36+
tds_obs AS (
37+
SELECT
38+
csi.thing_id,
39+
mc.id AS major_chemistry_id,
40+
COALESCE(mc."AnalysisDate", csi."CollectionDate") AS observation_datetime,
41+
mc."SampleValue" AS sample_value,
42+
mc."Units" AS units
43+
FROM "NMA_MajorChemistry" AS mc
44+
JOIN "NMA_Chemistry_SampleInfo" AS csi
45+
ON csi.id = mc.chemistry_sample_info_id
46+
JOIN thing AS t ON t.id = csi.thing_id
47+
WHERE
48+
t.thing_type = 'water well'
49+
AND mc."SampleValue" IS NOT NULL
50+
AND (
51+
lower(coalesce(mc."Analyte", '')) IN (
52+
'tds',
53+
'total dissolved solids'
54+
)
55+
OR lower(coalesce(mc."Symbol", '')) = 'tds'
56+
)
57+
),
58+
ranked_tds AS (
59+
SELECT
60+
to2.thing_id,
61+
to2.major_chemistry_id,
62+
to2.observation_datetime,
63+
to2.sample_value,
64+
to2.units,
65+
ROW_NUMBER() OVER (
66+
PARTITION BY to2.thing_id
67+
ORDER BY to2.observation_datetime DESC NULLS LAST, to2.major_chemistry_id DESC
68+
) AS rn
69+
FROM tds_obs AS to2
70+
)
71+
SELECT
72+
t.id AS id,
73+
t.name,
74+
t.thing_type,
75+
rt.major_chemistry_id,
76+
rt.observation_datetime::date AS latest_tds_observation_date,
77+
rt.sample_value AS latest_tds_value,
78+
rt.units AS latest_tds_units,
79+
l.point
80+
FROM ranked_tds AS rt
81+
JOIN thing AS t ON t.id = rt.thing_id
82+
JOIN latest_location AS ll ON ll.thing_id = t.id
83+
JOIN location AS l ON l.id = ll.location_id
84+
WHERE rt.rn = 1
85+
"""
86+
87+
88+
def _create_avg_tds_view() -> str:
89+
return f"""
90+
CREATE MATERIALIZED VIEW ogc_avg_tds_wells AS
91+
WITH latest_location AS (
92+
{LATEST_LOCATION_CTE}
93+
),
94+
tds_obs AS (
95+
SELECT
96+
csi.thing_id,
97+
mc.id AS major_chemistry_id,
98+
COALESCE(mc."AnalysisDate", csi."CollectionDate")::date AS observation_date,
99+
mc."SampleValue" AS sample_value,
100+
mc."Units" AS units
101+
FROM "NMA_MajorChemistry" AS mc
102+
JOIN "NMA_Chemistry_SampleInfo" AS csi
103+
ON csi.id = mc.chemistry_sample_info_id
104+
JOIN thing AS t ON t.id = csi.thing_id
105+
WHERE
106+
t.thing_type = 'water well'
107+
AND mc."SampleValue" IS NOT NULL
108+
AND (
109+
lower(coalesce(mc."Analyte", '')) IN (
110+
'tds',
111+
'total dissolved solids'
112+
)
113+
OR lower(coalesce(mc."Symbol", '')) = 'tds'
114+
)
115+
)
116+
SELECT
117+
t.id AS id,
118+
t.name,
119+
t.thing_type,
120+
COUNT(to2.major_chemistry_id)::integer AS tds_observation_count,
121+
AVG(to2.sample_value)::double precision AS avg_tds_value,
122+
MIN(to2.observation_date) AS first_tds_observation_date,
123+
MAX(to2.observation_date) AS last_tds_observation_date,
124+
l.point
125+
FROM tds_obs AS to2
126+
JOIN thing AS t ON t.id = to2.thing_id
127+
JOIN latest_location AS ll ON ll.thing_id = t.id
128+
JOIN location AS l ON l.id = ll.location_id
129+
GROUP BY t.id, t.name, t.thing_type, l.point
130+
"""
131+
132+
133+
def _create_avg_tds_view_with_datetime_columns() -> str:
134+
return f"""
135+
CREATE MATERIALIZED VIEW ogc_avg_tds_wells AS
136+
WITH latest_location AS (
137+
{LATEST_LOCATION_CTE}
138+
),
139+
tds_obs AS (
140+
SELECT
141+
csi.thing_id,
142+
mc.id AS major_chemistry_id,
143+
mc."AnalysisDate" AS analysis_date,
144+
mc."SampleValue" AS sample_value,
145+
mc."Units" AS units
146+
FROM "NMA_MajorChemistry" AS mc
147+
JOIN "NMA_Chemistry_SampleInfo" AS csi
148+
ON csi.id = mc.chemistry_sample_info_id
149+
JOIN thing AS t ON t.id = csi.thing_id
150+
WHERE
151+
t.thing_type = 'water well'
152+
AND mc."SampleValue" IS NOT NULL
153+
AND (
154+
lower(coalesce(mc."Analyte", '')) IN (
155+
'tds',
156+
'total dissolved solids'
157+
)
158+
OR lower(coalesce(mc."Symbol", '')) = 'tds'
159+
)
160+
)
161+
SELECT
162+
t.id AS id,
163+
t.name,
164+
t.thing_type,
165+
COUNT(to2.major_chemistry_id)::integer AS tds_observation_count,
166+
AVG(to2.sample_value)::double precision AS avg_tds_value,
167+
MIN(to2.analysis_date::date) AS first_tds_observation_date,
168+
MAX(to2.analysis_date::date) AS last_tds_observation_date,
169+
l.point
170+
FROM tds_obs AS to2
171+
JOIN thing AS t ON t.id = to2.thing_id
172+
JOIN latest_location AS ll ON ll.thing_id = t.id
173+
JOIN location AS l ON l.id = ll.location_id
174+
GROUP BY t.id, t.name, t.thing_type, l.point
175+
"""
176+
177+
178+
def upgrade() -> None:
179+
bind = op.get_bind()
180+
inspector = inspect(bind)
181+
existing_tables = set(inspector.get_table_names(schema="public"))
182+
required_tds = {
183+
"NMA_MajorChemistry",
184+
"NMA_Chemistry_SampleInfo",
185+
"thing",
186+
"location",
187+
"location_thing_association",
188+
}
189+
190+
if not required_tds.issubset(existing_tables):
191+
missing_tds_tables = sorted(t for t in required_tds if t not in existing_tables)
192+
missing_tds_tables_str = ", ".join(missing_tds_tables)
193+
raise RuntimeError(
194+
"Cannot create TDS views. The following required "
195+
f"tables are missing: {missing_tds_tables_str}"
196+
)
197+
198+
op.execute(text("DROP MATERIALIZED VIEW IF EXISTS ogc_avg_tds_wells"))
199+
op.execute(text("DROP VIEW IF EXISTS ogc_latest_tds_wells"))
200+
op.execute(text("DROP MATERIALIZED VIEW IF EXISTS ogc_latest_tds_wells"))
201+
202+
op.execute(text(_create_avg_tds_view()))
203+
op.execute(
204+
text(
205+
"COMMENT ON MATERIALIZED VIEW ogc_avg_tds_wells IS "
206+
"'Average TDS per well from major chemistry results for pygeoapi.'"
207+
)
208+
)
209+
op.execute(
210+
text("CREATE UNIQUE INDEX ux_ogc_avg_tds_wells_id " "ON ogc_avg_tds_wells (id)")
211+
)
212+
213+
op.execute(text(_create_latest_tds_view()))
214+
op.execute(
215+
text(
216+
"COMMENT ON VIEW ogc_latest_tds_wells IS "
217+
"'Latest TDS per well from major chemistry results for pygeoapi.'"
218+
)
219+
)
220+
221+
222+
def downgrade() -> None:
223+
op.execute(text("DROP MATERIALIZED VIEW IF EXISTS ogc_avg_tds_wells"))
224+
op.execute(text("DROP VIEW IF EXISTS ogc_latest_tds_wells"))
225+
op.execute(text("DROP MATERIALIZED VIEW IF EXISTS ogc_latest_tds_wells"))
226+
op.execute(text(_create_avg_tds_view_with_datetime_columns()))
227+
op.execute(
228+
text("CREATE UNIQUE INDEX ux_ogc_avg_tds_wells_id " "ON ogc_avg_tds_wells (id)")
229+
)

0 commit comments

Comments
 (0)