From 29a07fa16387ef1999261f5fd600bd6ddc7c9b92 Mon Sep 17 00:00:00 2001 From: jross Date: Thu, 27 Mar 2025 10:02:16 -0600 Subject: [PATCH 01/26] started sites only --- backend/bounding_polygons.py | 1 + backend/config.py | 15 ++- backend/connectors/nmbgmr/source.py | 41 ++++---- backend/connectors/nmbgmr/transformer.py | 6 +- backend/connectors/nmenv/source.py | 39 +++++--- backend/connectors/wqp/source.py | 18 ++-- backend/unifier.py | 116 ++++++++++++++++------- frontend/cli.py | 4 +- 8 files changed, 157 insertions(+), 83 deletions(-) diff --git a/backend/bounding_polygons.py b/backend/bounding_polygons.py index d9cd100..32cbbe3 100644 --- a/backend/bounding_polygons.py +++ b/backend/bounding_polygons.py @@ -174,6 +174,7 @@ def get_state_polygon(state): # private helpers ============================ def _make_shape(obj, as_wkt): poly = shape(obj["geometry"]) + poly = poly.simplify(0.1) if as_wkt: return poly.wkt return poly diff --git a/backend/config.py b/backend/config.py index ec9be97..89c4f59 100644 --- a/backend/config.py +++ b/backend/config.py @@ -116,6 +116,8 @@ class Config(Loggable): county: str = "" wkt: str = "" + sites_only = False + # sources use_source_bernco: bool = True use_source_bor: bool = True @@ -186,6 +188,15 @@ def __init__(self, model=None, payload=None): for s in SOURCE_KEYS: setattr(self, f"use_source_{s}", s in payload.get("sources", [])) + def finalize(self): + self.update_output_name() + self.make_output_path() + + def all_site_sources(self): + sources = self.water_level_sources() + sources.extend(self.analyte_sources()) + return sources + def analyte_sources(self): sources = [] @@ -384,7 +395,7 @@ def _validate_county(self): return True - def _update_output_name(self): + def update_output_name(self): """ Generate a unique output name based on existing directories in the output directory. @@ -419,7 +430,7 @@ def _update_output_name(self): self.output_name = output_name - def _make_output_path(self): + def make_output_path(self): if not os.path.exists(self.output_path): os.mkdir(self.output_path) diff --git a/backend/connectors/nmbgmr/source.py b/backend/connectors/nmbgmr/source.py index d75adae..a91e29e 100644 --- a/backend/connectors/nmbgmr/source.py +++ b/backend/connectors/nmbgmr/source.py @@ -73,30 +73,33 @@ def get_records(self): if config.site_limit: params["limit"] = config.site_limit - if config.parameter.lower() != "waterlevels": - params["parameter"] = get_analyte_search_param( - config.parameter, NMBGMR_ANALYTE_MAPPING - ) - else: - params["parameter"] = "Manual groundwater levels" + if not config.sites_only: + + if config.parameter.lower() != "waterlevels": + params["parameter"] = get_analyte_search_param( + config.parameter, NMBGMR_ANALYTE_MAPPING + ) + else: + params["parameter"] = "Manual groundwater levels" # tags="features" because the response object is a GeoJSON sites = self._execute_json_request( _make_url("locations"), params, tag="features", timeout=30 ) - for site in sites: - print(f"Obtaining well data for {site['properties']['point_id']}") - well_data = self._execute_json_request( - _make_url("wells"), - params={"pointid": site["properties"]["point_id"]}, - tag="", - ) - site["properties"]["formation"] = well_data["formation"] - site["properties"]["well_depth"] = well_data["well_depth_ftbgs"] - site["properties"]["well_depth_units"] = FEET - # site["properties"]["formation"] = None - # site["properties"]["well_depth"] = None - # site["properties"]["well_depth_units"] = FEET + if not config.sites_only: + for site in sites: + print(f"Obtaining well data for {site['properties']['point_id']}") + well_data = self._execute_json_request( + _make_url("wells"), + params={"pointid": site["properties"]["point_id"]}, + tag="", + ) + site["properties"]["formation"] = well_data["formation"] + site["properties"]["well_depth"] = well_data["well_depth_ftbgs"] + site["properties"]["well_depth_units"] = FEET + # site["properties"]["formation"] = None + # site["properties"]["well_depth"] = None + # site["properties"]["well_depth_units"] = FEET return sites diff --git a/backend/connectors/nmbgmr/transformer.py b/backend/connectors/nmbgmr/transformer.py index dd1163e..420c7f6 100644 --- a/backend/connectors/nmbgmr/transformer.py +++ b/backend/connectors/nmbgmr/transformer.py @@ -38,9 +38,9 @@ def _transform(self, record): "vertical_datum": props["altitude_datum"], "usgs_site_id": props["site_id"], "alternate_site_id": props["alternate_site_id"], - "formation": props["formation"], - "well_depth": props["well_depth"], - "well_depth_units": props["well_depth_units"], + "formation": props.get("formation", ""), + "well_depth": props.get("well_depth", ""), + "well_depth_units": props.get("well_depth_units", ""), } return rec diff --git a/backend/connectors/nmenv/source.py b/backend/connectors/nmenv/source.py index 335fd73..b28413f 100644 --- a/backend/connectors/nmenv/source.py +++ b/backend/connectors/nmenv/source.py @@ -47,29 +47,42 @@ def health(self): return self.get_records(top=10, analyte="TDS") def get_records(self, *args, **kw): + analyte = None if "analyte" in kw: analyte = kw["analyte"] elif self.config: analyte = self.config.parameter - analyte = get_analyte_search_param(analyte, DWB_ANALYTE_MAPPING) - if analyte is None: - return [] - service = self.get_service() - ds = service.datastreams() - q = ds.query() - fs = [f"ObservedProperty/id eq {analyte}"] - if self.config: + if self.config.sites_only: + ds = service.things() + q = ds.query() + fs = [] if self.config.has_bounds(): fs.append( - f"st_within(Thing/Location/location, geography'{self.config.bounding_wkt()}')" + f"st_within(Locations/location, geography'{self.config.bounding_wkt()}')" ) - - q = q.filter(" and ".join(fs)) - q = q.expand("Thing/Locations") - return [ds.thing.locations.entities[0] for ds in q.list()] + q = q.expand("Locations") + q = q.filter(" and ".join(fs)) + return [thing.locations.entities[0] for thing in q.list()] + else: + analyte = get_analyte_search_param(analyte, DWB_ANALYTE_MAPPING) + if analyte is None: + return [] + + ds = service.datastreams() + q = ds.query() + fs = [f"ObservedProperty/id eq {analyte}"] + if self.config: + if self.config.has_bounds(): + fs.append( + f"st_within(Thing/Location/location, geography'{self.config.bounding_wkt()}')" + ) + + q = q.filter(" and ".join(fs)) + q = q.expand("Thing/Locations") + return [di.thing.locations.entities[0] for di in q.list()] class DWBAnalyteSource(STAnalyteSource): diff --git a/backend/connectors/wqp/source.py b/backend/connectors/wqp/source.py index 4987fee..996b3aa 100644 --- a/backend/connectors/wqp/source.py +++ b/backend/connectors/wqp/source.py @@ -87,15 +87,15 @@ def get_records(self): } if config.has_bounds(): params["bBox"] = ",".join([str(b) for b in config.bbox_bounding_points()]) - - if config.parameter.lower() != "waterlevels": - params["characteristicName"] = get_analyte_search_param( - config.parameter, WQP_ANALYTE_MAPPING - ) - else: - # every record with pCode 30210 (depth in m) has a corresponding - # record with pCode 72019 (depth in ft) but not vice versa - params["pCode"] = "30210" + if not config.sites_only: + if config.parameter.lower() != "waterlevels": + params["characteristicName"] = get_analyte_search_param( + config.parameter, WQP_ANALYTE_MAPPING + ) + else: + # every record with pCode 30210 (depth in m) has a corresponding + # record with pCode 72019 (depth in ft) but not vice versa + params["pCode"] = "30210" params.update(get_date_range(config)) diff --git a/backend/unifier.py b/backend/unifier.py index 9523da9..36dd3b5 100644 --- a/backend/unifier.py +++ b/backend/unifier.py @@ -73,6 +73,16 @@ def unify_waterlevels(config): return True +def unify_sites_only(config): + print("Unifying sites only\n") + + # config.report() -- report is done in cli.py, no need to do it twice + config.validate() + + if not config.dry: + _unify_parameter(config, config.all_site_sources()) + + return True def _perister_factory(config): """ @@ -136,39 +146,44 @@ def _site_wrapper(site_source, parameter_source, persister, config): start_ind = 1 end_ind = 0 first_flag = True - for sites in site_source.chunks(sites): - if site_limit and sites_with_records_count == site_limit: - break - - if type(sites) == list: - if first_flag: - end_ind += len(sites) - first_flag = False + + if config.sites_only: + persister.sites.extend(sites) + else: + for sites in site_source.chunks(sites): + if site_limit and sites_with_records_count == site_limit: + break + + if type(sites) == list: + n = len(sites) + if first_flag: + first_flag = False + else: + start_ind = end_ind + 1 + + end_ind += n + + if use_summarize: + summary_records = parameter_source.read( + sites, use_summarize, start_ind, end_ind + ) + if summary_records: + persister.records.extend(summary_records) else: - start_ind = end_ind + 1 - end_ind += len(sites) - - if use_summarize: - summary_records = parameter_source.read( - sites, use_summarize, start_ind, end_ind - ) - if summary_records: - persister.records.extend(summary_records) - else: - results = parameter_source.read( - sites, use_summarize, start_ind, end_ind - ) - # no records are returned if there is no site record for parameter - # or if the record isn't clean (doesn't have the correct fields) - # don't count these sites to apply to site_limit - if results is None or len(results) == 0: - continue - - for site, records in results: - persister.timeseries.append((site, records)) - persister.sites.append(site) - - sites_with_records_count += 1 + results = parameter_source.read( + sites, use_summarize, start_ind, end_ind + ) + # no records are returned if there is no site record for parameter + # or if the record isn't clean (doesn't have the correct fields) + # don't count these sites to apply to site_limit + if results is None or len(results) == 0: + continue + + for site, records in results: + persister.timeseries.append((site, records)) + persister.sites.append(site) + + sites_with_records_count += 1 except BaseException: import traceback @@ -191,6 +206,8 @@ def _unify_parameter( elif config.output_timeseries_unified: persister.dump_timeseries_unified(config.output_path) persister.dump_sites(config.output_path) + elif config.sites_only: + persister.dump_sites(config.output_path) else: # config.output_timeseries_separated persister.dump_timeseries_separated(config.output_path) persister.dump_sites(config.output_path) @@ -297,13 +314,41 @@ def waterlevel_unification_test(): cfg.use_source_nwis = False cfg.use_source_nmbgmr = False cfg.use_source_iscsevenrivers = False - # cfg.use_source_pvacd = False - cfg.use_source_oseroswell = False + cfg.use_source_pvacd = False + # cfg.use_source_oseroswell = False cfg.use_source_bernco = False + cfg.use_source_iscsevenrivers = False + cfg.use_source_nmose_isc_seven_rivers = False + cfg.use_source_ebid = False # cfg.site_limit = 10 unify_waterlevels(cfg) +def site_unification_test(): + cfg = Config() + cfg.county = "chaves" + + + cfg.output_summary = False + cfg.output_name = "sitesonly" + cfg.sites_only = True + # cfg.output_summary = True + # cfg.output_single_timeseries = True + + cfg.use_source_nwis = False + cfg.use_source_nmbgmr = False + cfg.use_source_iscsevenrivers = False + cfg.use_source_pvacd = False + # cfg.use_source_oseroswell = False + cfg.use_source_bernco = False + cfg.use_source_iscsevenrivers = False + cfg.use_source_nmose_isc_seven_rivers = False + cfg.use_source_ebid = False + + cfg.finalize() + + unify_sites_only(cfg) + def get_datastream(siteid): import httpx @@ -329,7 +374,8 @@ def get_datastreams(): # shandler = logging.StreamHandler() # get_sources(Config()) setup_logging() - waterlevel_unification_test() + site_unification_test() + # waterlevel_unification_test() # analyte_unification_test() # print(health_check("nwis")) # generate_site_bounds() diff --git a/frontend/cli.py b/frontend/cli.py index e03ac0b..8dd330c 100644 --- a/frontend/cli.py +++ b/frontend/cli.py @@ -228,10 +228,10 @@ def weave( config.parameter = parameter # make sure config.output_name is properly set - config._update_output_name() + config.update_output_name() # make output_path now so that die.log can be written to it live - config._make_output_path() + config.make_output_path() # setup logging here so that the path can be set to config.output_path setup_logging(path=config.output_path) From 3510008c8f67b9e2e083b708edaa75fcde69f1b5 Mon Sep 17 00:00:00 2001 From: jross Date: Mon, 31 Mar 2025 11:15:59 -0600 Subject: [PATCH 02/26] sites only --- backend/connectors/nmenv/source.py | 3 +- backend/unifier.py | 40 +++++++------- frontend/cli.py | 83 +++++++++++------------------- frontend/cronjob_worker.sh | 3 ++ 4 files changed, 53 insertions(+), 76 deletions(-) create mode 100644 frontend/cronjob_worker.sh diff --git a/backend/connectors/nmenv/source.py b/backend/connectors/nmenv/source.py index b28413f..3646073 100644 --- a/backend/connectors/nmenv/source.py +++ b/backend/connectors/nmenv/source.py @@ -64,7 +64,8 @@ def get_records(self, *args, **kw): f"st_within(Locations/location, geography'{self.config.bounding_wkt()}')" ) q = q.expand("Locations") - q = q.filter(" and ".join(fs)) + if fs: + q = q.filter(" and ".join(fs)) return [thing.locations.entities[0] for thing in q.list()] else: analyte = get_analyte_search_param(analyte, DWB_ANALYTE_MAPPING) diff --git a/backend/unifier.py b/backend/unifier.py index 36dd3b5..0646ace 100644 --- a/backend/unifier.py +++ b/backend/unifier.py @@ -40,22 +40,11 @@ def health_check(source: BaseSiteSource) -> bool: return bool(source.health()) -def unify_sites(config): - print("Unifying sites\n") - - # def func(config, persister): - # for source in config.site_sources(): - # s = source() - # persister.load(s.read(config)) - - # _unify_wrapper(config, func) - - def unify_analytes(config): print("Unifying analytes\n") # config.report() -- report is done in cli.py, no need to do it twice config.validate() - + config.finalize() if not config.dry: _unify_parameter(config, config.analyte_sources()) @@ -67,17 +56,18 @@ def unify_waterlevels(config): # config.report() -- report is done in cli.py, no need to do it twice config.validate() - + config.finalize() if not config.dry: _unify_parameter(config, config.water_level_sources()) return True -def unify_sites_only(config): +def unify_sites(config): print("Unifying sites only\n") # config.report() -- report is done in cli.py, no need to do it twice config.validate() + config.finalize() if not config.dry: _unify_parameter(config, config.all_site_sources()) @@ -335,19 +325,25 @@ def site_unification_test(): # cfg.output_summary = True # cfg.output_single_timeseries = True - cfg.use_source_nwis = False - cfg.use_source_nmbgmr = False - cfg.use_source_iscsevenrivers = False - cfg.use_source_pvacd = False - # cfg.use_source_oseroswell = False cfg.use_source_bernco = False - cfg.use_source_iscsevenrivers = False - cfg.use_source_nmose_isc_seven_rivers = False + cfg.use_source_bor = False + cfg.use_source_cabq = False cfg.use_source_ebid = False + cfg.use_source_nmbgmr_amp = False + cfg.use_source_nmed_dwb = False + cfg.use_source_nmose_isc_seven_rivers = False + cfg.use_source_nmose_roswell = False + cfg.use_source_nwis = False + cfg.use_source_pvacd = False + cfg.use_source_wqp = False + + cfg.use_source_nmed_dwb = True + + cfg.finalize() - unify_sites_only(cfg) + unify_sites(cfg) def get_datastream(siteid): diff --git a/frontend/cli.py b/frontend/cli.py index 8dd330c..760f303 100644 --- a/frontend/cli.py +++ b/frontend/cli.py @@ -227,11 +227,11 @@ def weave( config = setup_config(f"{parameter}", bbox, county, site_limit, dry) config.parameter = parameter - # make sure config.output_name is properly set - config.update_output_name() - - # make output_path now so that die.log can be written to it live - config.make_output_path() + # # make sure config.output_name is properly set + # config.update_output_name() + # + # # make output_path now so that die.log can be written to it live + # config.make_output_path() # setup logging here so that the path can be set to config.output_path setup_logging(path=config.output_path) @@ -249,53 +249,33 @@ def weave( summary = False timeseries_unified = False timeseries_separated = True + else: + click.echo(f"Invalid output type: {output}") + return config.output_summary = summary config.output_timeseries_unified = timeseries_unified config.output_timeseries_separated = timeseries_separated + false_agencies = [] + config_agencies = [] # sources if parameter == "waterlevels": - config.use_source_bernco = no_bernco - config.use_source_cabq = no_cabq - config.use_source_ebid = no_ebid - config.use_source_nmbgmr_amp = no_nmbgmr_amp - config.use_source_nmose_isc_seven_rivers = no_nmose_isc_seven_rivers - config.use_source_nmose_roswell = no_nmose_roswell - config.use_source_nwis = no_nwis - config.use_source_pvacd = no_pvacd - config.use_source_wqp = no_wqp - - config.use_source_bor = False - config.use_source_nmed_dwb = False + config_agencies =["bernco", "cabq", "ebid", "nmbgmr_amp", "nmed_dwb", + "nmose_isc_seven_rivers", "nmose_roswell", "nwis", "pvacd"] + + false_agencies = ['bor', 'nmed_dwb'] elif parameter == "carbonate": - config.use_source_nmbgmr_amp = no_nmbgmr_amp - config.use_source_wqp = no_wqp - - config.use_source_bor = False - config.use_source_bernco = False - config.use_source_cabq = False - config.use_source_ebid = False - config.use_source_nmed_dwb = False - config.use_source_nmose_isc_seven_rivers = False - config.use_source_nmose_roswell = False - config.use_source_nwis = False - config.use_source_pvacd = False + config_agencies = ['nmbgmr_amp', 'wqp'] + false_agencies = ['bor', 'bernco', 'cabq', 'ebid', 'nmed_dwb', + 'nmose_isc_seven_rivers', 'nmose_roswell', 'nwis', 'pvacd'] elif parameter in ["arsenic", "uranium"]: - config.use_source_bor = no_bor - config.use_source_nmbgmr_amp = no_nmbgmr_amp - config.use_source_nmed_dwb = no_nmed_dwb - config.use_source_wqp = no_wqp - - config.use_source_bernco = False - config.use_source_cabq = False - config.use_source_ebid = False - config.use_source_nmose_isc_seven_rivers = False - config.use_source_nmose_roswell = False - config.use_source_nwis = False - config.use_source_pvacd = False + config_agencies = ['bor', 'nmbgmr_amp', 'nmed_dwb', 'wqp'] + false_agencies = ['bernco', 'cabq', 'ebid', 'nmose_isc_seven_rivers', + 'nmose_roswell', 'nwis', 'pvacd'] + elif parameter in [ "bicarbonate", @@ -311,19 +291,16 @@ def weave( "sulfate", "tds", ]: - config.use_source_bor = no_bor - config.use_source_nmbgmr_amp = no_nmbgmr_amp - config.use_source_nmed_dwb = no_nmed_dwb - config.use_source_nmose_isc_seven_rivers = no_nmose_isc_seven_rivers - config.use_source_wqp = no_wqp - - config.use_source_bernco = False - config.use_source_cabq = False - config.use_source_ebid = False - config.use_source_nmose_roswell = False - config.use_source_nwis = False - config.use_source_pvacd = False + config_agencies = ['bor', 'nmbgmr_amp', 'nmed_dwb','nmose_isc_seven_rivers', 'wqp'] + false_agencies = ['bernco', 'cabq', 'ebid', 'nmose_roswell', 'nwis', 'pvacd'] + + if false_agencies: + for agency in false_agencies: + setattr(config, f"use_source_{agency}", False) + if config_agencies: + for agency in config_agencies: + setattr(config, f"use_source_{agency}", getattr(locals(),f'no_{agency}')) # dates config.start_date = start_date config.end_date = end_date diff --git a/frontend/cronjob_worker.sh b/frontend/cronjob_worker.sh new file mode 100644 index 0000000..4a3925f --- /dev/null +++ b/frontend/cronjob_worker.sh @@ -0,0 +1,3 @@ + + +die weave \ No newline at end of file From e5ec261485d17f13d64bfdd52b9b46ce6f2f09a5 Mon Sep 17 00:00:00 2001 From: jross Date: Mon, 31 Mar 2025 11:49:26 -0600 Subject: [PATCH 03/26] sites only cli --- frontend/cli.py | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/frontend/cli.py b/frontend/cli.py index 760f303..534b225 100644 --- a/frontend/cli.py +++ b/frontend/cli.py @@ -321,11 +321,34 @@ def weave( @cli.command() @add_options(SPATIAL_OPTIONS) -def wells(bbox, county): +@add_options(ALL_SOURCE_OPTIONS) +@add_options(DEBUG_OPTIONS) +def wells(bbox, county, + no_bernco, + no_bor, + no_cabq, + no_ebid, + no_nmbgmr_amp, + no_nmed_dwb, + no_nmose_isc_seven_rivers, + no_nmose_roswell, + no_nwis, + no_pvacd, + no_wqp, + site_limit, + dry,): """ Get locations """ - config = setup_config("sites", bbox, county) + + + config = setup_config("sites", bbox, county, site_limit, dry) + config_agencies = ["bernco", "bor", "cabq", "ebid", "nmbgmr_amp", "nmed_dwb", + "nmose_isc_seven_rivers", "nmose_roswell", "nwis", "pvacd", + "wqp"] + for agency in config_agencies: + setattr(config, f"use_source_{agency}", getattr(locals(),f'no_{agency}')) + unify_sites(config) From 5feace88541f9ecbb77621dd3479bb152bfb8eab Mon Sep 17 00:00:00 2001 From: jross Date: Mon, 31 Mar 2025 11:54:29 -0600 Subject: [PATCH 04/26] sites only cli --- frontend/cli.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/frontend/cli.py b/frontend/cli.py index 534b225..eb09581 100644 --- a/frontend/cli.py +++ b/frontend/cli.py @@ -298,9 +298,10 @@ def weave( for agency in false_agencies: setattr(config, f"use_source_{agency}", False) + lcs = locals() if config_agencies: for agency in config_agencies: - setattr(config, f"use_source_{agency}", getattr(locals(),f'no_{agency}')) + setattr(config, f"use_source_{agency}", lcs.get(f'no_{agency}', False)) # dates config.start_date = start_date config.end_date = end_date @@ -341,13 +342,13 @@ def wells(bbox, county, Get locations """ - config = setup_config("sites", bbox, county, site_limit, dry) config_agencies = ["bernco", "bor", "cabq", "ebid", "nmbgmr_amp", "nmed_dwb", "nmose_isc_seven_rivers", "nmose_roswell", "nwis", "pvacd", "wqp"] + lcs = locals() for agency in config_agencies: - setattr(config, f"use_source_{agency}", getattr(locals(),f'no_{agency}')) + setattr(config, f"use_source_{agency}", lcs.get(f'no_{agency}', False)) unify_sites(config) From b77984678dd869905512ccbe33ba1d3cd156306f Mon Sep 17 00:00:00 2001 From: jross Date: Mon, 31 Mar 2025 11:56:59 -0600 Subject: [PATCH 05/26] sites only cli --- frontend/cli.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/frontend/cli.py b/frontend/cli.py index eb09581..37b51ae 100644 --- a/frontend/cli.py +++ b/frontend/cli.py @@ -350,6 +350,11 @@ def wells(bbox, county, for agency in config_agencies: setattr(config, f"use_source_{agency}", lcs.get(f'no_{agency}', False)) + config.report() + # prompt user to continue + if not click.confirm("Do you want to continue?", default=True): + return + unify_sites(config) From f9b6a87dededa1beb0cf2377cd579ddbeac1e41d Mon Sep 17 00:00:00 2001 From: jross Date: Mon, 31 Mar 2025 12:08:08 -0600 Subject: [PATCH 06/26] sites only cli --- frontend/cli.py | 1 + 1 file changed, 1 insertion(+) diff --git a/frontend/cli.py b/frontend/cli.py index 37b51ae..89b16f1 100644 --- a/frontend/cli.py +++ b/frontend/cli.py @@ -350,6 +350,7 @@ def wells(bbox, county, for agency in config_agencies: setattr(config, f"use_source_{agency}", lcs.get(f'no_{agency}', False)) + config.sites_only = True config.report() # prompt user to continue if not click.confirm("Do you want to continue?", default=True): From c7b1e3d27daf8e3dfb33359838d5732b2f3b0a18 Mon Sep 17 00:00:00 2001 From: jross Date: Mon, 31 Mar 2025 14:55:03 -0600 Subject: [PATCH 07/26] sites only cli --- frontend/cli.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/frontend/cli.py b/frontend/cli.py index 89b16f1..7201bd4 100644 --- a/frontend/cli.py +++ b/frontend/cli.py @@ -136,6 +136,12 @@ def cli(): default=False, help="Dry run. Do not execute unifier. Used by unit tests", ), + click.option( + "--yes", + is_flag=True, + default=False, + help="Do not ask for confirmation before running", + ), ] DT_OPTIONS = [ @@ -337,7 +343,8 @@ def wells(bbox, county, no_pvacd, no_wqp, site_limit, - dry,): + dry, + yes): """ Get locations """ @@ -352,9 +359,10 @@ def wells(bbox, county, config.sites_only = True config.report() - # prompt user to continue - if not click.confirm("Do you want to continue?", default=True): - return + if not yes: + # prompt user to continue + if not click.confirm("Do you want to continue?", default=True): + return unify_sites(config) From a9d671f77ebb839dff876b71f18229f2e44ac2fa Mon Sep 17 00:00:00 2001 From: jross Date: Mon, 31 Mar 2025 15:01:02 -0600 Subject: [PATCH 08/26] sites only cli --- backend/config.py | 1 + backend/unifier.py | 5 ++--- frontend/cli.py | 16 +++++++++++----- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/backend/config.py b/backend/config.py index 89c4f59..0396a73 100644 --- a/backend/config.py +++ b/backend/config.py @@ -189,6 +189,7 @@ def __init__(self, model=None, payload=None): setattr(self, f"use_source_{s}", s in payload.get("sources", [])) def finalize(self): + self._update_output_units() self.update_output_name() self.make_output_path() diff --git a/backend/unifier.py b/backend/unifier.py index 0646ace..3e46008 100644 --- a/backend/unifier.py +++ b/backend/unifier.py @@ -44,7 +44,7 @@ def unify_analytes(config): print("Unifying analytes\n") # config.report() -- report is done in cli.py, no need to do it twice config.validate() - config.finalize() + if not config.dry: _unify_parameter(config, config.analyte_sources()) @@ -56,7 +56,7 @@ def unify_waterlevels(config): # config.report() -- report is done in cli.py, no need to do it twice config.validate() - config.finalize() + if not config.dry: _unify_parameter(config, config.water_level_sources()) @@ -67,7 +67,6 @@ def unify_sites(config): # config.report() -- report is done in cli.py, no need to do it twice config.validate() - config.finalize() if not config.dry: _unify_parameter(config, config.all_site_sources()) diff --git a/frontend/cli.py b/frontend/cli.py index 7201bd4..8e1ebe1 100644 --- a/frontend/cli.py +++ b/frontend/cli.py @@ -239,9 +239,6 @@ def weave( # # make output_path now so that die.log can be written to it live # config.make_output_path() - # setup logging here so that the path can be set to config.output_path - setup_logging(path=config.output_path) - # output type if output == "summary": summary = True @@ -312,14 +309,16 @@ def weave( config.start_date = start_date config.end_date = end_date + config.finalize() + # setup logging here so that the path can be set to config.output_path + setup_logging(path=config.output_path) + if not dry: config.report() # prompt user to continue if not click.confirm("Do you want to continue?", default=True): return - config._update_output_units() - if parameter.lower() == "waterlevels": unify_waterlevels(config) else: @@ -328,9 +327,11 @@ def weave( @cli.command() @add_options(SPATIAL_OPTIONS) +@add_options(OUTPUT_OPTIONS) @add_options(ALL_SOURCE_OPTIONS) @add_options(DEBUG_OPTIONS) def wells(bbox, county, + output, no_bernco, no_bor, no_cabq, @@ -358,6 +359,11 @@ def wells(bbox, county, setattr(config, f"use_source_{agency}", lcs.get(f'no_{agency}', False)) config.sites_only = True + + config.finalize() + # setup logging here so that the path can be set to config.output_path + setup_logging(path=config.output_path) + config.report() if not yes: # prompt user to continue From 49e4f6845691e22a5e80265cbfd5819bc9ab44e0 Mon Sep 17 00:00:00 2001 From: jross Date: Mon, 31 Mar 2025 15:04:46 -0600 Subject: [PATCH 09/26] sites only cli --- frontend/cli.py | 61 +++++++++++++++++++++++++++---------------------- 1 file changed, 34 insertions(+), 27 deletions(-) diff --git a/frontend/cli.py b/frontend/cli.py index 8e1ebe1..580d153 100644 --- a/frontend/cli.py +++ b/frontend/cli.py @@ -23,6 +23,7 @@ from backend.logging import setup_logging + # setup_logging() @@ -180,7 +181,12 @@ def cli(): type=click.Choice(["summary", "timeseries_unified", "timeseries_separated"]), required=True, help="Output summary file, single unified timeseries file, or separated timeseries files", - ) + ), + click.option( + "--output-dir", + default=".", + help="Output root directory. Default is current directory", + ), ] @@ -205,25 +211,26 @@ def _add_options(func): @add_options(ALL_SOURCE_OPTIONS) @add_options(DEBUG_OPTIONS) def weave( - weave, - output, - start_date, - end_date, - bbox, - county, - no_bernco, - no_bor, - no_cabq, - no_ebid, - no_nmbgmr_amp, - no_nmed_dwb, - no_nmose_isc_seven_rivers, - no_nmose_roswell, - no_nwis, - no_pvacd, - no_wqp, - site_limit, - dry, + weave, + output, + output_dir, + start_date, + end_date, + bbox, + county, + no_bernco, + no_bor, + no_cabq, + no_ebid, + no_nmbgmr_amp, + no_nmed_dwb, + no_nmose_isc_seven_rivers, + no_nmose_roswell, + no_nwis, + no_pvacd, + no_wqp, + site_limit, + dry, ): """ Get parameter timeseries or summary data @@ -264,20 +271,20 @@ def weave( config_agencies = [] # sources if parameter == "waterlevels": - config_agencies =["bernco", "cabq", "ebid", "nmbgmr_amp", "nmed_dwb", - "nmose_isc_seven_rivers", "nmose_roswell", "nwis", "pvacd"] + config_agencies = ["bernco", "cabq", "ebid", "nmbgmr_amp", "nmed_dwb", + "nmose_isc_seven_rivers", "nmose_roswell", "nwis", "pvacd"] false_agencies = ['bor', 'nmed_dwb'] elif parameter == "carbonate": config_agencies = ['nmbgmr_amp', 'wqp'] false_agencies = ['bor', 'bernco', 'cabq', 'ebid', 'nmed_dwb', - 'nmose_isc_seven_rivers', 'nmose_roswell', 'nwis', 'pvacd'] + 'nmose_isc_seven_rivers', 'nmose_roswell', 'nwis', 'pvacd'] elif parameter in ["arsenic", "uranium"]: config_agencies = ['bor', 'nmbgmr_amp', 'nmed_dwb', 'wqp'] false_agencies = ['bernco', 'cabq', 'ebid', 'nmose_isc_seven_rivers', - 'nmose_roswell', 'nwis', 'pvacd'] + 'nmose_roswell', 'nwis', 'pvacd'] elif parameter in [ @@ -294,7 +301,7 @@ def weave( "sulfate", "tds", ]: - config_agencies = ['bor', 'nmbgmr_amp', 'nmed_dwb','nmose_isc_seven_rivers', 'wqp'] + config_agencies = ['bor', 'nmbgmr_amp', 'nmed_dwb', 'nmose_isc_seven_rivers', 'wqp'] false_agencies = ['bernco', 'cabq', 'ebid', 'nmose_roswell', 'nwis', 'pvacd'] if false_agencies: @@ -332,6 +339,7 @@ def weave( @add_options(DEBUG_OPTIONS) def wells(bbox, county, output, + output_dir, no_bernco, no_bor, no_cabq, @@ -359,7 +367,7 @@ def wells(bbox, county, setattr(config, f"use_source_{agency}", lcs.get(f'no_{agency}', False)) config.sites_only = True - + config.output_dir = output_dir config.finalize() # setup logging here so that the path can be set to config.output_path setup_logging(path=config.output_path) @@ -414,5 +422,4 @@ def setup_config(tag, bbox, county, site_limit, dry): return config - # ============= EOF ============================================= From 60eadafee1b8b39c54060b60c6e6dda9896500b4 Mon Sep 17 00:00:00 2001 From: jross Date: Mon, 31 Mar 2025 15:06:52 -0600 Subject: [PATCH 10/26] sites only cli --- frontend/cli.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/frontend/cli.py b/frontend/cli.py index 580d153..3f04aab 100644 --- a/frontend/cli.py +++ b/frontend/cli.py @@ -182,11 +182,14 @@ def cli(): required=True, help="Output summary file, single unified timeseries file, or separated timeseries files", ), - click.option( + +] +PERSISTER_OPTIONS = [ + click.option(click.option( "--output-dir", default=".", help="Output root directory. Default is current directory", - ), + )) ] @@ -206,6 +209,7 @@ def _add_options(func): required=True, ) @add_options(OUTPUT_OPTIONS) +@add_options(PERSISTER_OPTIONS) @add_options(DT_OPTIONS) @add_options(SPATIAL_OPTIONS) @add_options(ALL_SOURCE_OPTIONS) @@ -335,6 +339,7 @@ def weave( @cli.command() @add_options(SPATIAL_OPTIONS) @add_options(OUTPUT_OPTIONS) +@add_options(PERSISTER_OPTIONS) @add_options(ALL_SOURCE_OPTIONS) @add_options(DEBUG_OPTIONS) def wells(bbox, county, From 5ddf81441d0b56187e3d9d90db65a3743342fd89 Mon Sep 17 00:00:00 2001 From: jross Date: Mon, 31 Mar 2025 15:08:01 -0600 Subject: [PATCH 11/26] sites only cli --- frontend/cli.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/frontend/cli.py b/frontend/cli.py index 3f04aab..73c1a61 100644 --- a/frontend/cli.py +++ b/frontend/cli.py @@ -338,12 +338,10 @@ def weave( @cli.command() @add_options(SPATIAL_OPTIONS) -@add_options(OUTPUT_OPTIONS) @add_options(PERSISTER_OPTIONS) @add_options(ALL_SOURCE_OPTIONS) @add_options(DEBUG_OPTIONS) def wells(bbox, county, - output, output_dir, no_bernco, no_bor, From 9ad78f85add651621e3cc169b59a3d80c7974797 Mon Sep 17 00:00:00 2001 From: jross Date: Mon, 31 Mar 2025 15:08:40 -0600 Subject: [PATCH 12/26] sites only cli --- frontend/cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/cli.py b/frontend/cli.py index 73c1a61..8bb9811 100644 --- a/frontend/cli.py +++ b/frontend/cli.py @@ -185,11 +185,11 @@ def cli(): ] PERSISTER_OPTIONS = [ - click.option(click.option( + click.option( "--output-dir", default=".", help="Output root directory. Default is current directory", - )) + ) ] From e1754477d711770b113f55f56a3b1cfd40fd20d7 Mon Sep 17 00:00:00 2001 From: jross Date: Mon, 31 Mar 2025 15:10:24 -0600 Subject: [PATCH 13/26] sites only cli --- backend/config.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/backend/config.py b/backend/config.py index 0396a73..03595fb 100644 --- a/backend/config.py +++ b/backend/config.py @@ -192,6 +192,7 @@ def finalize(self): self._update_output_units() self.update_output_name() self.make_output_path() + self.make_output_directory() def all_site_sources(self): sources = self.water_level_sources() @@ -395,6 +396,12 @@ def _validate_county(self): return bool(get_county_polygon(self.county)) return True + def make_output_directory(self): + """ + Create the output directory if it doesn't exist. + """ + if not os.path.exists(self.output_dir): + os.mkdir(self.output_dir) def update_output_name(self): """ From deae8d327003bd1f5d23467e7fff673ebad69ac7 Mon Sep 17 00:00:00 2001 From: jross Date: Mon, 31 Mar 2025 15:11:01 -0600 Subject: [PATCH 14/26] sites only cli --- backend/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/config.py b/backend/config.py index 03595fb..b9dea0c 100644 --- a/backend/config.py +++ b/backend/config.py @@ -190,9 +190,9 @@ def __init__(self, model=None, payload=None): def finalize(self): self._update_output_units() + self.make_output_directory() self.update_output_name() self.make_output_path() - self.make_output_directory() def all_site_sources(self): sources = self.water_level_sources() From 12e6d076655bfc685eff8b466300d7fb377f9a5f Mon Sep 17 00:00:00 2001 From: jross Date: Mon, 31 Mar 2025 15:22:40 -0600 Subject: [PATCH 15/26] sites only cli --- backend/config.py | 59 ++++++++++++++++++----------------------------- 1 file changed, 22 insertions(+), 37 deletions(-) diff --git a/backend/config.py b/backend/config.py index b9dea0c..4639987 100644 --- a/backend/config.py +++ b/backend/config.py @@ -61,46 +61,31 @@ from .connectors.usgs.source import NWISSiteSource, NWISWaterLevelSource from .connectors.wqp.source import WQPSiteSource, WQPAnalyteSource, WQPWaterLevelSource -SOURCE_KEYS = ( - "bernco", - "bor", - "cabq", - "ebid", - "nmbgmr_amp", - "nmed_dwb", - "nmose_isc_seven_rivers", - "nmose_roswell", - "nwis", - "pvacd", - "wqp", -) +SOURCE_DICT = { + "bernco": BernCoSiteSource, + "bor": BORSiteSource, + "cabq": CABQSiteSource, + "ebid": EBIDSiteSource, + "nmbgmr_amp": NMBGMRSiteSource, + "nmed_dwb": DWBSiteSource, + "nmose_isc_seven_rivers": ISCSevenRiversSiteSource, + "nmose_roswell": NMOSERoswellSiteSource, + "nwis": NWISSiteSource, + "pvacd": PVACDSiteSource, + "wqp": WQPSiteSource, +} + +SOURCE_KEYS = list(SOURCE_DICT.keys()) def get_source(source): - if source == "bernco": - return BernCoSiteSource() - elif source == "bor": - return BORSiteSource() - elif source == "cabq": - return CABQSiteSource() - elif source == "ebid": - return EBIDSiteSource() - elif source == "nmbgmr_amp": - return NMBGMRSiteSource() - elif source == "nmed_dwb": - return DWBSiteSource() - elif source == "nmose_isc_seven_rivers": - return ISCSevenRiversSiteSource() - elif source == "nmose_roswell": - return NMOSERoswellSiteSource() - elif source == "nwis": - return NWISSiteSource() - elif source == "pvacd": - return PVACDSiteSource() - elif source == "wqp": - return WQPSiteSource() - - return None + try: + klass = SOURCE_DICT[source] + except KeyError: + raise ValueError(f"Unknown source {source}") + + if klass: + return klass() class Config(Loggable): From 358aebbc81f4d039c9e3937c5a6617756b7a4ca9 Mon Sep 17 00:00:00 2001 From: jross Date: Mon, 31 Mar 2025 17:09:06 -0600 Subject: [PATCH 16/26] added wqp to config_agency when parameter = 'waterlevels' --- frontend/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/cli.py b/frontend/cli.py index 8bb9811..cf038ac 100644 --- a/frontend/cli.py +++ b/frontend/cli.py @@ -276,7 +276,7 @@ def weave( # sources if parameter == "waterlevels": config_agencies = ["bernco", "cabq", "ebid", "nmbgmr_amp", "nmed_dwb", - "nmose_isc_seven_rivers", "nmose_roswell", "nwis", "pvacd"] + "nmose_isc_seven_rivers", "nmose_roswell", "nwis", "pvacd", "wqp"] false_agencies = ['bor', 'nmed_dwb'] From a04dea3d9ba16fb238720c955eeb96e023043477 Mon Sep 17 00:00:00 2001 From: jross Date: Tue, 1 Apr 2025 10:08:47 -0600 Subject: [PATCH 17/26] fixed all_site_sources --- backend/config.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/backend/config.py b/backend/config.py index 4639987..26afe88 100644 --- a/backend/config.py +++ b/backend/config.py @@ -180,8 +180,11 @@ def finalize(self): self.make_output_path() def all_site_sources(self): - sources = self.water_level_sources() - sources.extend(self.analyte_sources()) + sources =[] + for s in SOURCE_KEYS: + if getattr(self, f"use_source_{s}"): + sources.append((get_source(s), None)) + return sources def analyte_sources(self): From 11e6a77d2930420bef6c69fb5cc724b20e1d8c63 Mon Sep 17 00:00:00 2001 From: jross Date: Tue, 1 Apr 2025 10:11:23 -0600 Subject: [PATCH 18/26] fixed all_site_sources --- backend/config.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/config.py b/backend/config.py index 26afe88..b7fb2e8 100644 --- a/backend/config.py +++ b/backend/config.py @@ -183,7 +183,9 @@ def all_site_sources(self): sources =[] for s in SOURCE_KEYS: if getattr(self, f"use_source_{s}"): - sources.append((get_source(s), None)) + source = get_source(s) + source.set_config(self) + sources.append((source, None)) return sources From 2cbf406c311db93ca1253e1286a4cca0e319eeed Mon Sep 17 00:00:00 2001 From: jross Date: Tue, 1 Apr 2025 11:51:42 -0600 Subject: [PATCH 19/26] added pods --- backend/config.py | 6 +++ backend/connectors/nmose/source.py | 57 +++++++++++++++++++++++++ backend/connectors/nmose/transformer.py | 35 +++++++++++++++ backend/transformer.py | 1 + backend/unifier.py | 3 +- 5 files changed, 101 insertions(+), 1 deletion(-) diff --git a/backend/config.py b/backend/config.py index b7fb2e8..45885a1 100644 --- a/backend/config.py +++ b/backend/config.py @@ -39,6 +39,7 @@ OSERoswellWaterLevelSource, ) from .connectors.nmenv.source import DWBSiteSource, DWBAnalyteSource +from .connectors.nmose.source import NMOSEPODSiteSource from .constants import MILLIGRAMS_PER_LITER, WGS84, FEET from .connectors.isc_seven_rivers.source import ( ISCSevenRiversSiteSource, @@ -74,6 +75,7 @@ "nwis": NWISSiteSource, "pvacd": PVACDSiteSource, "wqp": WQPSiteSource, + "nmose_pod": NMOSEPODSiteSource, } SOURCE_KEYS = list(SOURCE_DICT.keys()) @@ -115,6 +117,7 @@ class Config(Loggable): use_source_nwis: bool = True use_source_pvacd: bool = True use_source_wqp: bool = True + use_source_nmose_pod: bool = False # parameter parameter: str = "" @@ -187,6 +190,9 @@ def all_site_sources(self): source.set_config(self) sources.append((source, None)) + # pods = NMOSEPODSiteSource() + # pods.set_config(self) + # sources.append((pods, None)) return sources def analyte_sources(self): diff --git a/backend/connectors/nmose/source.py b/backend/connectors/nmose/source.py index 5cb7a3e..9b848b0 100644 --- a/backend/connectors/nmose/source.py +++ b/backend/connectors/nmose/source.py @@ -1,2 +1,59 @@ import os + +from shapely import wkt +from shapely.geometry.polygon import Polygon + +from backend.connectors import NM_STATE_BOUNDING_POLYGON +from backend.connectors.nmose.transformer import NMOSEPODSiteTransformer from backend.source import BaseSiteSource + + +def wkt_to_arcgis_json(polygon_wkt): + obj = wkt.loads(polygon_wkt) + coords = [[coord[0], coord[1]] for coord in obj.exterior.coords] + return { + 'rings': [coords], + 'spatialReference': { + 'wkid': 4326 + } + } + +class NMOSEPODSiteSource(BaseSiteSource): + """ + NMOSEPODSiteSource is a class that inherits from BaseSiteSource. + It is used to fetch site data from the NMOSEPOD API. + """ + + transformer_klass = NMOSEPODSiteTransformer + chunk_size = 1000 + bounding_polygon = NM_STATE_BOUNDING_POLYGON + + def get_records(self, *args, **kw) -> dict: + config = self.config + params = {} + # if config.has_bounds(): + # bbox = config.bbox_bounding_points() + # params["bBox"] = ",".join([str(b) for b in bbox]) + # else: + # params["stateCd"] = "NM" + # + # if config.start_date: + # params["startDt"] = config.start_dt.date().isoformat() + # if config.end_date: + # params["endDt"] = config.end_dt.date().isoformat() + + params['where'] = "pod_status = 'ACT' AND pod_basin IN ('A','B','C','CC','CD','CL','CP','CR','CT','E','FS','G','GSF','H', 'HA','HC','HS','HU','J','L','LA','LRG','LV','M','MR','NH','P','PL','PN','RA','RG','S','SB','SJ','SS','T','TU','UP','VV')" + params["outFields"] = "OBJECTID,pod_basin,pod_status,easting,northing,datum,utm_accura,status,county,pod_name,pod_nbr,pod_suffix,pod_file" + params["outSR"] = 4326 + params["f"] = "json" + if config.has_bounds(): + wkt = config.bounding_wkt() + else: + wkt = NM_STATE_BOUNDING_POLYGON + + params["geometry"] = wkt_to_arcgis_json(wkt) + params["geometryType"] = "esriGeometryPolygon" + url = "https://services2.arcgis.com/qXZbWTdPDbTjl7Dy/arcgis/rest/services/OSE_PODs/FeatureServer/0/query" + obj = self._execute_json_request(url, params, tag='features') + + return obj \ No newline at end of file diff --git a/backend/connectors/nmose/transformer.py b/backend/connectors/nmose/transformer.py index e69de29..2ed99e6 100644 --- a/backend/connectors/nmose/transformer.py +++ b/backend/connectors/nmose/transformer.py @@ -0,0 +1,35 @@ +from backend.transformer import BaseTransformer, SiteTransformer + + +class NMOSEPODSiteTransformer(SiteTransformer): + def _transform(self, record) -> dict: + """ + Transform the record into a dictionary format. + + Args: + record (dict): The record to transform. + + Returns: + dict: The transformed record. + """ + + properties = record['attributes'] + geometry = record['geometry'] + + # print(properties.keys()) + # print(geometry.keys()) + rec = { + "source": "NMOSEPOD", + "id": properties["pod_file"], + # "name": record["station_nm"], + "latitude": geometry["y"], + "longitude": geometry["x"], + # "elevation": elevation, + # "elevation_units": "ft", + # "horizontal_datum": datum, + # "vertical_datum": record["alt_datum_cd"], + # "aquifer": record["nat_aqfr_cd"], + # "well_depth": record["well_depth_va"], + # "well_depth_units": "ft", + } + return rec \ No newline at end of file diff --git a/backend/transformer.py b/backend/transformer.py index 232bd16..3356320 100644 --- a/backend/transformer.py +++ b/backend/transformer.py @@ -394,6 +394,7 @@ def do_transform( if not record: return + print(record) # ensure that a site or summary record is contained within the boundaing polygon if "longitude" in record and "latitude" in record: if not self.contained(record["longitude"], record["latitude"]): diff --git a/backend/unifier.py b/backend/unifier.py index 3e46008..41fddac 100644 --- a/backend/unifier.py +++ b/backend/unifier.py @@ -335,8 +335,9 @@ def site_unification_test(): cfg.use_source_nwis = False cfg.use_source_pvacd = False cfg.use_source_wqp = False + cfg.use_source_nmose_pod = True - cfg.use_source_nmed_dwb = True + cfg.use_source_nmed_dwb = False From d9bdd29c38c9e8210d62b8063920664c5f2dbcce Mon Sep 17 00:00:00 2001 From: jross Date: Tue, 1 Apr 2025 11:53:01 -0600 Subject: [PATCH 20/26] added pods --- backend/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/config.py b/backend/config.py index 45885a1..306a82c 100644 --- a/backend/config.py +++ b/backend/config.py @@ -117,7 +117,7 @@ class Config(Loggable): use_source_nwis: bool = True use_source_pvacd: bool = True use_source_wqp: bool = True - use_source_nmose_pod: bool = False + use_source_nmose_pod: bool = True # parameter parameter: str = "" From c237a60b7c6f3bc8c0c714f27c78c951f64e4243 Mon Sep 17 00:00:00 2001 From: jross Date: Tue, 1 Apr 2025 11:53:42 -0600 Subject: [PATCH 21/26] added pods --- backend/transformer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/transformer.py b/backend/transformer.py index 3356320..232bd16 100644 --- a/backend/transformer.py +++ b/backend/transformer.py @@ -394,7 +394,6 @@ def do_transform( if not record: return - print(record) # ensure that a site or summary record is contained within the boundaing polygon if "longitude" in record and "latitude" in record: if not self.contained(record["longitude"], record["latitude"]): From 27491c271b7b4b45a6423545ff33392662c62abc Mon Sep 17 00:00:00 2001 From: jross Date: Tue, 1 Apr 2025 11:55:29 -0600 Subject: [PATCH 22/26] added pods --- backend/connectors/nmose/source.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/backend/connectors/nmose/source.py b/backend/connectors/nmose/source.py index 9b848b0..5060577 100644 --- a/backend/connectors/nmose/source.py +++ b/backend/connectors/nmose/source.py @@ -8,8 +8,9 @@ from backend.source import BaseSiteSource -def wkt_to_arcgis_json(polygon_wkt): - obj = wkt.loads(polygon_wkt) +def wkt_to_arcgis_json(obj): + if isinstance(obj, str): + obj = wkt.loads(obj) coords = [[coord[0], coord[1]] for coord in obj.exterior.coords] return { 'rings': [coords], From ef08db75c9151ac9db96376bad6e2a38d718645b Mon Sep 17 00:00:00 2001 From: jross Date: Tue, 1 Apr 2025 12:18:59 -0600 Subject: [PATCH 23/26] added pods --- backend/bounding_polygons.py | 11 +++++++++++ backend/connectors/nmose/source.py | 28 ++++++++++++++++++++-------- backend/source.py | 1 + backend/unifier.py | 2 +- 4 files changed, 33 insertions(+), 9 deletions(-) diff --git a/backend/bounding_polygons.py b/backend/bounding_polygons.py index 32cbbe3..a591ffa 100644 --- a/backend/bounding_polygons.py +++ b/backend/bounding_polygons.py @@ -119,6 +119,17 @@ def get_huc_polygon(huc, as_wkt=True): return _make_shape(obj, as_wkt) +def get_county_names(state="NM"): + state, statefp = _get_statefp(state) + obj = _get_cached_object( + f"{state}.counties", + f"{state} counties", + f"https://reference.geoconnex.us/collections/counties/items?statefp={statefp}&f=json", + ) + + return [f['properties']['name'] for f in obj['features']] + + def get_county_polygon(name, as_wkt=True): if ":" in name: state, county = name.split(":") diff --git a/backend/connectors/nmose/source.py b/backend/connectors/nmose/source.py index 5060577..28ac9e2 100644 --- a/backend/connectors/nmose/source.py +++ b/backend/connectors/nmose/source.py @@ -3,6 +3,7 @@ from shapely import wkt from shapely.geometry.polygon import Polygon +from backend.bounding_polygons import get_county_polygon, get_county_names from backend.connectors import NM_STATE_BOUNDING_POLYGON from backend.connectors.nmose.transformer import NMOSEPODSiteTransformer from backend.source import BaseSiteSource @@ -26,7 +27,7 @@ class NMOSEPODSiteSource(BaseSiteSource): """ transformer_klass = NMOSEPODSiteTransformer - chunk_size = 1000 + chunk_size = 5000 bounding_polygon = NM_STATE_BOUNDING_POLYGON def get_records(self, *args, **kw) -> dict: @@ -43,18 +44,29 @@ def get_records(self, *args, **kw) -> dict: # if config.end_date: # params["endDt"] = config.end_dt.date().isoformat() + url = "https://services2.arcgis.com/qXZbWTdPDbTjl7Dy/arcgis/rest/services/OSE_PODs/FeatureServer/0/query" + params['where'] = "pod_status = 'ACT' AND pod_basin IN ('A','B','C','CC','CD','CL','CP','CR','CT','E','FS','G','GSF','H', 'HA','HC','HS','HU','J','L','LA','LRG','LV','M','MR','NH','P','PL','PN','RA','RG','S','SB','SJ','SS','T','TU','UP','VV')" params["outFields"] = "OBJECTID,pod_basin,pod_status,easting,northing,datum,utm_accura,status,county,pod_name,pod_nbr,pod_suffix,pod_file" params["outSR"] = 4326 params["f"] = "json" + params["resultRecordCount"] = self.chunk_size + params['resultOffset'] = 0 + if config.has_bounds(): wkt = config.bounding_wkt() - else: - wkt = NM_STATE_BOUNDING_POLYGON + params["geometry"] = wkt_to_arcgis_json(wkt) + params["geometryType"] = "esriGeometryPolygon" - params["geometry"] = wkt_to_arcgis_json(wkt) - params["geometryType"] = "esriGeometryPolygon" - url = "https://services2.arcgis.com/qXZbWTdPDbTjl7Dy/arcgis/rest/services/OSE_PODs/FeatureServer/0/query" - obj = self._execute_json_request(url, params, tag='features') + records = [] + i=1 + while 1: + rs = self._execute_json_request(url, params, tag='features') + records.extend(rs) + params['resultOffset'] += self.chunk_size + print((i, len(rs))) + if len(rs) < self.chunk_size: + break + i+=1 - return obj \ No newline at end of file + return records \ No newline at end of file diff --git a/backend/source.py b/backend/source.py index 457006b..0ba7e91 100644 --- a/backend/source.py +++ b/backend/source.py @@ -300,6 +300,7 @@ def _execute_json_request( self.warn(f"service responded but with no data. \n{resp.text}") return [] else: + print('ffasdfsafasdfasdf', resp.url) self.warn(f"service responded with status {resp.status_code}") self.warn(f"service responded with text {resp.text}") return [] diff --git a/backend/unifier.py b/backend/unifier.py index 41fddac..f31fa99 100644 --- a/backend/unifier.py +++ b/backend/unifier.py @@ -315,7 +315,7 @@ def waterlevel_unification_test(): def site_unification_test(): cfg = Config() - cfg.county = "chaves" + # cfg.county = "chaves" cfg.output_summary = False From 104ae3c3fba105546cebdee16d5b47033b296ba5 Mon Sep 17 00:00:00 2001 From: jross Date: Tue, 1 Apr 2025 15:43:59 -0600 Subject: [PATCH 24/26] added pods --- backend/connectors/nmose/source.py | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/connectors/nmose/source.py b/backend/connectors/nmose/source.py index 28ac9e2..c0a1eb9 100644 --- a/backend/connectors/nmose/source.py +++ b/backend/connectors/nmose/source.py @@ -64,7 +64,6 @@ def get_records(self, *args, **kw) -> dict: rs = self._execute_json_request(url, params, tag='features') records.extend(rs) params['resultOffset'] += self.chunk_size - print((i, len(rs))) if len(rs) < self.chunk_size: break i+=1 From 88fb0b3a8ff4ee83abb7085f70b95a35713f8fb0 Mon Sep 17 00:00:00 2001 From: jross Date: Tue, 1 Apr 2025 15:48:04 -0600 Subject: [PATCH 25/26] added pods --- backend/bounding_polygons.py | 11 ----------- backend/connectors/nmose/source.py | 3 --- backend/source.py | 2 +- 3 files changed, 1 insertion(+), 15 deletions(-) diff --git a/backend/bounding_polygons.py b/backend/bounding_polygons.py index a591ffa..32cbbe3 100644 --- a/backend/bounding_polygons.py +++ b/backend/bounding_polygons.py @@ -119,17 +119,6 @@ def get_huc_polygon(huc, as_wkt=True): return _make_shape(obj, as_wkt) -def get_county_names(state="NM"): - state, statefp = _get_statefp(state) - obj = _get_cached_object( - f"{state}.counties", - f"{state} counties", - f"https://reference.geoconnex.us/collections/counties/items?statefp={statefp}&f=json", - ) - - return [f['properties']['name'] for f in obj['features']] - - def get_county_polygon(name, as_wkt=True): if ":" in name: state, county = name.split(":") diff --git a/backend/connectors/nmose/source.py b/backend/connectors/nmose/source.py index c0a1eb9..53fe355 100644 --- a/backend/connectors/nmose/source.py +++ b/backend/connectors/nmose/source.py @@ -1,9 +1,6 @@ import os from shapely import wkt -from shapely.geometry.polygon import Polygon - -from backend.bounding_polygons import get_county_polygon, get_county_names from backend.connectors import NM_STATE_BOUNDING_POLYGON from backend.connectors.nmose.transformer import NMOSEPODSiteTransformer from backend.source import BaseSiteSource diff --git a/backend/source.py b/backend/source.py index 0ba7e91..44e705f 100644 --- a/backend/source.py +++ b/backend/source.py @@ -300,9 +300,9 @@ def _execute_json_request( self.warn(f"service responded but with no data. \n{resp.text}") return [] else: - print('ffasdfsafasdfasdf', resp.url) self.warn(f"service responded with status {resp.status_code}") self.warn(f"service responded with text {resp.text}") + self.warn(f"service at url: {resp.url}") return [] # ========================================================================== From bcfdaf7d8b67fde9fd80de7aa3452426196513e5 Mon Sep 17 00:00:00 2001 From: jross Date: Wed, 9 Apr 2025 13:19:08 -0600 Subject: [PATCH 26/26] added additional fields to pods site export --- backend/connectors/nmose/source.py | 8 ++++++-- backend/connectors/nmose/transformer.py | 10 +++++----- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/backend/connectors/nmose/source.py b/backend/connectors/nmose/source.py index 53fe355..4fc03e4 100644 --- a/backend/connectors/nmose/source.py +++ b/backend/connectors/nmose/source.py @@ -43,8 +43,12 @@ def get_records(self, *args, **kw) -> dict: url = "https://services2.arcgis.com/qXZbWTdPDbTjl7Dy/arcgis/rest/services/OSE_PODs/FeatureServer/0/query" - params['where'] = "pod_status = 'ACT' AND pod_basin IN ('A','B','C','CC','CD','CL','CP','CR','CT','E','FS','G','GSF','H', 'HA','HC','HS','HU','J','L','LA','LRG','LV','M','MR','NH','P','PL','PN','RA','RG','S','SB','SJ','SS','T','TU','UP','VV')" - params["outFields"] = "OBJECTID,pod_basin,pod_status,easting,northing,datum,utm_accura,status,county,pod_name,pod_nbr,pod_suffix,pod_file" + # params['where'] = "pod_status = 'ACT' AND pod_basin IN ('A','B','C','CC','CD','CL','CP','CR','CT','E','FS', + # 'G','GSF','H', 'HA','HC','HS','HU','J','L','LA','LRG','LV','M','MR','NH','P','PL','PN','RA','RG','S','SB','SJ','SS','T','TU','UP','VV')" + #pods = 157127 + params['where'] = "pod_status = 'ACT' AND pod_basin NOT IN ('SP', 'SD', 'LWD')" + params["outFields"] = ("OBJECTID,pod_basin,pod_status,easting,northing,datum,utm_accura,status," + "pod_name,pod_nbr,pod_suffix,pod_file,depth_well,aquifer,elevation") params["outSR"] = 4326 params["f"] = "json" params["resultRecordCount"] = self.chunk_size diff --git a/backend/connectors/nmose/transformer.py b/backend/connectors/nmose/transformer.py index 2ed99e6..7f6997f 100644 --- a/backend/connectors/nmose/transformer.py +++ b/backend/connectors/nmose/transformer.py @@ -24,12 +24,12 @@ def _transform(self, record) -> dict: # "name": record["station_nm"], "latitude": geometry["y"], "longitude": geometry["x"], - # "elevation": elevation, - # "elevation_units": "ft", + "elevation": properties['elevation'], + "elevation_units": "ft", # "horizontal_datum": datum, # "vertical_datum": record["alt_datum_cd"], - # "aquifer": record["nat_aqfr_cd"], - # "well_depth": record["well_depth_va"], - # "well_depth_units": "ft", + "aquifer": properties["aquifer"], + "well_depth": properties["depth_well"], + "well_depth_units": "ft", } return rec \ No newline at end of file