diff --git a/backend/bounding_polygons.py b/backend/bounding_polygons.py index d9cd100..32cbbe3 100644 --- a/backend/bounding_polygons.py +++ b/backend/bounding_polygons.py @@ -174,6 +174,7 @@ def get_state_polygon(state): # private helpers ============================ def _make_shape(obj, as_wkt): poly = shape(obj["geometry"]) + poly = poly.simplify(0.1) if as_wkt: return poly.wkt return poly diff --git a/backend/config.py b/backend/config.py index ec9be97..306a82c 100644 --- a/backend/config.py +++ b/backend/config.py @@ -39,6 +39,7 @@ OSERoswellWaterLevelSource, ) from .connectors.nmenv.source import DWBSiteSource, DWBAnalyteSource +from .connectors.nmose.source import NMOSEPODSiteSource from .constants import MILLIGRAMS_PER_LITER, WGS84, FEET from .connectors.isc_seven_rivers.source import ( ISCSevenRiversSiteSource, @@ -61,46 +62,32 @@ from .connectors.usgs.source import NWISSiteSource, NWISWaterLevelSource from .connectors.wqp.source import WQPSiteSource, WQPAnalyteSource, WQPWaterLevelSource -SOURCE_KEYS = ( - "bernco", - "bor", - "cabq", - "ebid", - "nmbgmr_amp", - "nmed_dwb", - "nmose_isc_seven_rivers", - "nmose_roswell", - "nwis", - "pvacd", - "wqp", -) +SOURCE_DICT = { + "bernco": BernCoSiteSource, + "bor": BORSiteSource, + "cabq": CABQSiteSource, + "ebid": EBIDSiteSource, + "nmbgmr_amp": NMBGMRSiteSource, + "nmed_dwb": DWBSiteSource, + "nmose_isc_seven_rivers": ISCSevenRiversSiteSource, + "nmose_roswell": NMOSERoswellSiteSource, + "nwis": NWISSiteSource, + "pvacd": PVACDSiteSource, + "wqp": WQPSiteSource, + "nmose_pod": NMOSEPODSiteSource, +} + +SOURCE_KEYS = list(SOURCE_DICT.keys()) def get_source(source): - if source == "bernco": - return BernCoSiteSource() - elif source == "bor": - return BORSiteSource() - elif source == "cabq": - return CABQSiteSource() - elif source == "ebid": - return EBIDSiteSource() - elif source == "nmbgmr_amp": - return NMBGMRSiteSource() - elif source == "nmed_dwb": - return DWBSiteSource() - elif source == "nmose_isc_seven_rivers": - return ISCSevenRiversSiteSource() - elif source == "nmose_roswell": - return NMOSERoswellSiteSource() - elif source == "nwis": - return NWISSiteSource() - elif source == "pvacd": - return PVACDSiteSource() - elif source == "wqp": - return WQPSiteSource() - - return None + try: + klass = SOURCE_DICT[source] + except KeyError: + raise ValueError(f"Unknown source {source}") + + if klass: + return klass() class Config(Loggable): @@ -116,6 +103,8 @@ class Config(Loggable): county: str = "" wkt: str = "" + sites_only = False + # sources use_source_bernco: bool = True use_source_bor: bool = True @@ -128,6 +117,7 @@ class Config(Loggable): use_source_nwis: bool = True use_source_pvacd: bool = True use_source_wqp: bool = True + use_source_nmose_pod: bool = True # parameter parameter: str = "" @@ -186,6 +176,25 @@ def __init__(self, model=None, payload=None): for s in SOURCE_KEYS: setattr(self, f"use_source_{s}", s in payload.get("sources", [])) + def finalize(self): + self._update_output_units() + self.make_output_directory() + self.update_output_name() + self.make_output_path() + + def all_site_sources(self): + sources =[] + for s in SOURCE_KEYS: + if getattr(self, f"use_source_{s}"): + source = get_source(s) + source.set_config(self) + sources.append((source, None)) + + # pods = NMOSEPODSiteSource() + # pods.set_config(self) + # sources.append((pods, None)) + return sources + def analyte_sources(self): sources = [] @@ -383,8 +392,14 @@ def _validate_county(self): return bool(get_county_polygon(self.county)) return True + def make_output_directory(self): + """ + Create the output directory if it doesn't exist. + """ + if not os.path.exists(self.output_dir): + os.mkdir(self.output_dir) - def _update_output_name(self): + def update_output_name(self): """ Generate a unique output name based on existing directories in the output directory. @@ -419,7 +434,7 @@ def _update_output_name(self): self.output_name = output_name - def _make_output_path(self): + def make_output_path(self): if not os.path.exists(self.output_path): os.mkdir(self.output_path) diff --git a/backend/connectors/nmbgmr/source.py b/backend/connectors/nmbgmr/source.py index d75adae..a91e29e 100644 --- a/backend/connectors/nmbgmr/source.py +++ b/backend/connectors/nmbgmr/source.py @@ -73,30 +73,33 @@ def get_records(self): if config.site_limit: params["limit"] = config.site_limit - if config.parameter.lower() != "waterlevels": - params["parameter"] = get_analyte_search_param( - config.parameter, NMBGMR_ANALYTE_MAPPING - ) - else: - params["parameter"] = "Manual groundwater levels" + if not config.sites_only: + + if config.parameter.lower() != "waterlevels": + params["parameter"] = get_analyte_search_param( + config.parameter, NMBGMR_ANALYTE_MAPPING + ) + else: + params["parameter"] = "Manual groundwater levels" # tags="features" because the response object is a GeoJSON sites = self._execute_json_request( _make_url("locations"), params, tag="features", timeout=30 ) - for site in sites: - print(f"Obtaining well data for {site['properties']['point_id']}") - well_data = self._execute_json_request( - _make_url("wells"), - params={"pointid": site["properties"]["point_id"]}, - tag="", - ) - site["properties"]["formation"] = well_data["formation"] - site["properties"]["well_depth"] = well_data["well_depth_ftbgs"] - site["properties"]["well_depth_units"] = FEET - # site["properties"]["formation"] = None - # site["properties"]["well_depth"] = None - # site["properties"]["well_depth_units"] = FEET + if not config.sites_only: + for site in sites: + print(f"Obtaining well data for {site['properties']['point_id']}") + well_data = self._execute_json_request( + _make_url("wells"), + params={"pointid": site["properties"]["point_id"]}, + tag="", + ) + site["properties"]["formation"] = well_data["formation"] + site["properties"]["well_depth"] = well_data["well_depth_ftbgs"] + site["properties"]["well_depth_units"] = FEET + # site["properties"]["formation"] = None + # site["properties"]["well_depth"] = None + # site["properties"]["well_depth_units"] = FEET return sites diff --git a/backend/connectors/nmbgmr/transformer.py b/backend/connectors/nmbgmr/transformer.py index dd1163e..420c7f6 100644 --- a/backend/connectors/nmbgmr/transformer.py +++ b/backend/connectors/nmbgmr/transformer.py @@ -38,9 +38,9 @@ def _transform(self, record): "vertical_datum": props["altitude_datum"], "usgs_site_id": props["site_id"], "alternate_site_id": props["alternate_site_id"], - "formation": props["formation"], - "well_depth": props["well_depth"], - "well_depth_units": props["well_depth_units"], + "formation": props.get("formation", ""), + "well_depth": props.get("well_depth", ""), + "well_depth_units": props.get("well_depth_units", ""), } return rec diff --git a/backend/connectors/nmenv/source.py b/backend/connectors/nmenv/source.py index 335fd73..3646073 100644 --- a/backend/connectors/nmenv/source.py +++ b/backend/connectors/nmenv/source.py @@ -47,29 +47,43 @@ def health(self): return self.get_records(top=10, analyte="TDS") def get_records(self, *args, **kw): + analyte = None if "analyte" in kw: analyte = kw["analyte"] elif self.config: analyte = self.config.parameter - analyte = get_analyte_search_param(analyte, DWB_ANALYTE_MAPPING) - if analyte is None: - return [] - service = self.get_service() - ds = service.datastreams() - q = ds.query() - fs = [f"ObservedProperty/id eq {analyte}"] - if self.config: + if self.config.sites_only: + ds = service.things() + q = ds.query() + fs = [] if self.config.has_bounds(): fs.append( - f"st_within(Thing/Location/location, geography'{self.config.bounding_wkt()}')" + f"st_within(Locations/location, geography'{self.config.bounding_wkt()}')" ) - - q = q.filter(" and ".join(fs)) - q = q.expand("Thing/Locations") - return [ds.thing.locations.entities[0] for ds in q.list()] + q = q.expand("Locations") + if fs: + q = q.filter(" and ".join(fs)) + return [thing.locations.entities[0] for thing in q.list()] + else: + analyte = get_analyte_search_param(analyte, DWB_ANALYTE_MAPPING) + if analyte is None: + return [] + + ds = service.datastreams() + q = ds.query() + fs = [f"ObservedProperty/id eq {analyte}"] + if self.config: + if self.config.has_bounds(): + fs.append( + f"st_within(Thing/Location/location, geography'{self.config.bounding_wkt()}')" + ) + + q = q.filter(" and ".join(fs)) + q = q.expand("Thing/Locations") + return [di.thing.locations.entities[0] for di in q.list()] class DWBAnalyteSource(STAnalyteSource): diff --git a/backend/connectors/nmose/source.py b/backend/connectors/nmose/source.py index 5cb7a3e..4fc03e4 100644 --- a/backend/connectors/nmose/source.py +++ b/backend/connectors/nmose/source.py @@ -1,2 +1,72 @@ import os + +from shapely import wkt +from backend.connectors import NM_STATE_BOUNDING_POLYGON +from backend.connectors.nmose.transformer import NMOSEPODSiteTransformer from backend.source import BaseSiteSource + + +def wkt_to_arcgis_json(obj): + if isinstance(obj, str): + obj = wkt.loads(obj) + coords = [[coord[0], coord[1]] for coord in obj.exterior.coords] + return { + 'rings': [coords], + 'spatialReference': { + 'wkid': 4326 + } + } + +class NMOSEPODSiteSource(BaseSiteSource): + """ + NMOSEPODSiteSource is a class that inherits from BaseSiteSource. + It is used to fetch site data from the NMOSEPOD API. + """ + + transformer_klass = NMOSEPODSiteTransformer + chunk_size = 5000 + bounding_polygon = NM_STATE_BOUNDING_POLYGON + + def get_records(self, *args, **kw) -> dict: + config = self.config + params = {} + # if config.has_bounds(): + # bbox = config.bbox_bounding_points() + # params["bBox"] = ",".join([str(b) for b in bbox]) + # else: + # params["stateCd"] = "NM" + # + # if config.start_date: + # params["startDt"] = config.start_dt.date().isoformat() + # if config.end_date: + # params["endDt"] = config.end_dt.date().isoformat() + + url = "https://services2.arcgis.com/qXZbWTdPDbTjl7Dy/arcgis/rest/services/OSE_PODs/FeatureServer/0/query" + + # params['where'] = "pod_status = 'ACT' AND pod_basin IN ('A','B','C','CC','CD','CL','CP','CR','CT','E','FS', + # 'G','GSF','H', 'HA','HC','HS','HU','J','L','LA','LRG','LV','M','MR','NH','P','PL','PN','RA','RG','S','SB','SJ','SS','T','TU','UP','VV')" + #pods = 157127 + params['where'] = "pod_status = 'ACT' AND pod_basin NOT IN ('SP', 'SD', 'LWD')" + params["outFields"] = ("OBJECTID,pod_basin,pod_status,easting,northing,datum,utm_accura,status," + "pod_name,pod_nbr,pod_suffix,pod_file,depth_well,aquifer,elevation") + params["outSR"] = 4326 + params["f"] = "json" + params["resultRecordCount"] = self.chunk_size + params['resultOffset'] = 0 + + if config.has_bounds(): + wkt = config.bounding_wkt() + params["geometry"] = wkt_to_arcgis_json(wkt) + params["geometryType"] = "esriGeometryPolygon" + + records = [] + i=1 + while 1: + rs = self._execute_json_request(url, params, tag='features') + records.extend(rs) + params['resultOffset'] += self.chunk_size + if len(rs) < self.chunk_size: + break + i+=1 + + return records \ No newline at end of file diff --git a/backend/connectors/nmose/transformer.py b/backend/connectors/nmose/transformer.py index e69de29..7f6997f 100644 --- a/backend/connectors/nmose/transformer.py +++ b/backend/connectors/nmose/transformer.py @@ -0,0 +1,35 @@ +from backend.transformer import BaseTransformer, SiteTransformer + + +class NMOSEPODSiteTransformer(SiteTransformer): + def _transform(self, record) -> dict: + """ + Transform the record into a dictionary format. + + Args: + record (dict): The record to transform. + + Returns: + dict: The transformed record. + """ + + properties = record['attributes'] + geometry = record['geometry'] + + # print(properties.keys()) + # print(geometry.keys()) + rec = { + "source": "NMOSEPOD", + "id": properties["pod_file"], + # "name": record["station_nm"], + "latitude": geometry["y"], + "longitude": geometry["x"], + "elevation": properties['elevation'], + "elevation_units": "ft", + # "horizontal_datum": datum, + # "vertical_datum": record["alt_datum_cd"], + "aquifer": properties["aquifer"], + "well_depth": properties["depth_well"], + "well_depth_units": "ft", + } + return rec \ No newline at end of file diff --git a/backend/connectors/wqp/source.py b/backend/connectors/wqp/source.py index 4987fee..996b3aa 100644 --- a/backend/connectors/wqp/source.py +++ b/backend/connectors/wqp/source.py @@ -87,15 +87,15 @@ def get_records(self): } if config.has_bounds(): params["bBox"] = ",".join([str(b) for b in config.bbox_bounding_points()]) - - if config.parameter.lower() != "waterlevels": - params["characteristicName"] = get_analyte_search_param( - config.parameter, WQP_ANALYTE_MAPPING - ) - else: - # every record with pCode 30210 (depth in m) has a corresponding - # record with pCode 72019 (depth in ft) but not vice versa - params["pCode"] = "30210" + if not config.sites_only: + if config.parameter.lower() != "waterlevels": + params["characteristicName"] = get_analyte_search_param( + config.parameter, WQP_ANALYTE_MAPPING + ) + else: + # every record with pCode 30210 (depth in m) has a corresponding + # record with pCode 72019 (depth in ft) but not vice versa + params["pCode"] = "30210" params.update(get_date_range(config)) diff --git a/backend/source.py b/backend/source.py index 457006b..44e705f 100644 --- a/backend/source.py +++ b/backend/source.py @@ -302,6 +302,7 @@ def _execute_json_request( else: self.warn(f"service responded with status {resp.status_code}") self.warn(f"service responded with text {resp.text}") + self.warn(f"service at url: {resp.url}") return [] # ========================================================================== diff --git a/backend/unifier.py b/backend/unifier.py index 9523da9..f31fa99 100644 --- a/backend/unifier.py +++ b/backend/unifier.py @@ -40,17 +40,6 @@ def health_check(source: BaseSiteSource) -> bool: return bool(source.health()) -def unify_sites(config): - print("Unifying sites\n") - - # def func(config, persister): - # for source in config.site_sources(): - # s = source() - # persister.load(s.read(config)) - - # _unify_wrapper(config, func) - - def unify_analytes(config): print("Unifying analytes\n") # config.report() -- report is done in cli.py, no need to do it twice @@ -73,6 +62,16 @@ def unify_waterlevels(config): return True +def unify_sites(config): + print("Unifying sites only\n") + + # config.report() -- report is done in cli.py, no need to do it twice + config.validate() + + if not config.dry: + _unify_parameter(config, config.all_site_sources()) + + return True def _perister_factory(config): """ @@ -136,39 +135,44 @@ def _site_wrapper(site_source, parameter_source, persister, config): start_ind = 1 end_ind = 0 first_flag = True - for sites in site_source.chunks(sites): - if site_limit and sites_with_records_count == site_limit: - break - - if type(sites) == list: - if first_flag: - end_ind += len(sites) - first_flag = False + + if config.sites_only: + persister.sites.extend(sites) + else: + for sites in site_source.chunks(sites): + if site_limit and sites_with_records_count == site_limit: + break + + if type(sites) == list: + n = len(sites) + if first_flag: + first_flag = False + else: + start_ind = end_ind + 1 + + end_ind += n + + if use_summarize: + summary_records = parameter_source.read( + sites, use_summarize, start_ind, end_ind + ) + if summary_records: + persister.records.extend(summary_records) else: - start_ind = end_ind + 1 - end_ind += len(sites) - - if use_summarize: - summary_records = parameter_source.read( - sites, use_summarize, start_ind, end_ind - ) - if summary_records: - persister.records.extend(summary_records) - else: - results = parameter_source.read( - sites, use_summarize, start_ind, end_ind - ) - # no records are returned if there is no site record for parameter - # or if the record isn't clean (doesn't have the correct fields) - # don't count these sites to apply to site_limit - if results is None or len(results) == 0: - continue - - for site, records in results: - persister.timeseries.append((site, records)) - persister.sites.append(site) - - sites_with_records_count += 1 + results = parameter_source.read( + sites, use_summarize, start_ind, end_ind + ) + # no records are returned if there is no site record for parameter + # or if the record isn't clean (doesn't have the correct fields) + # don't count these sites to apply to site_limit + if results is None or len(results) == 0: + continue + + for site, records in results: + persister.timeseries.append((site, records)) + persister.sites.append(site) + + sites_with_records_count += 1 except BaseException: import traceback @@ -191,6 +195,8 @@ def _unify_parameter( elif config.output_timeseries_unified: persister.dump_timeseries_unified(config.output_path) persister.dump_sites(config.output_path) + elif config.sites_only: + persister.dump_sites(config.output_path) else: # config.output_timeseries_separated persister.dump_timeseries_separated(config.output_path) persister.dump_sites(config.output_path) @@ -297,13 +303,48 @@ def waterlevel_unification_test(): cfg.use_source_nwis = False cfg.use_source_nmbgmr = False cfg.use_source_iscsevenrivers = False - # cfg.use_source_pvacd = False - cfg.use_source_oseroswell = False + cfg.use_source_pvacd = False + # cfg.use_source_oseroswell = False cfg.use_source_bernco = False + cfg.use_source_iscsevenrivers = False + cfg.use_source_nmose_isc_seven_rivers = False + cfg.use_source_ebid = False # cfg.site_limit = 10 unify_waterlevels(cfg) +def site_unification_test(): + cfg = Config() + # cfg.county = "chaves" + + + cfg.output_summary = False + cfg.output_name = "sitesonly" + cfg.sites_only = True + # cfg.output_summary = True + # cfg.output_single_timeseries = True + + cfg.use_source_bernco = False + cfg.use_source_bor = False + cfg.use_source_cabq = False + cfg.use_source_ebid = False + cfg.use_source_nmbgmr_amp = False + cfg.use_source_nmed_dwb = False + cfg.use_source_nmose_isc_seven_rivers = False + cfg.use_source_nmose_roswell = False + cfg.use_source_nwis = False + cfg.use_source_pvacd = False + cfg.use_source_wqp = False + cfg.use_source_nmose_pod = True + + cfg.use_source_nmed_dwb = False + + + + cfg.finalize() + + unify_sites(cfg) + def get_datastream(siteid): import httpx @@ -329,7 +370,8 @@ def get_datastreams(): # shandler = logging.StreamHandler() # get_sources(Config()) setup_logging() - waterlevel_unification_test() + site_unification_test() + # waterlevel_unification_test() # analyte_unification_test() # print(health_check("nwis")) # generate_site_bounds() diff --git a/frontend/cli.py b/frontend/cli.py index e03ac0b..cf038ac 100644 --- a/frontend/cli.py +++ b/frontend/cli.py @@ -23,6 +23,7 @@ from backend.logging import setup_logging + # setup_logging() @@ -136,6 +137,12 @@ def cli(): default=False, help="Dry run. Do not execute unifier. Used by unit tests", ), + click.option( + "--yes", + is_flag=True, + default=False, + help="Do not ask for confirmation before running", + ), ] DT_OPTIONS = [ @@ -174,6 +181,14 @@ def cli(): type=click.Choice(["summary", "timeseries_unified", "timeseries_separated"]), required=True, help="Output summary file, single unified timeseries file, or separated timeseries files", + ), + +] +PERSISTER_OPTIONS = [ + click.option( + "--output-dir", + default=".", + help="Output root directory. Default is current directory", ) ] @@ -194,30 +209,32 @@ def _add_options(func): required=True, ) @add_options(OUTPUT_OPTIONS) +@add_options(PERSISTER_OPTIONS) @add_options(DT_OPTIONS) @add_options(SPATIAL_OPTIONS) @add_options(ALL_SOURCE_OPTIONS) @add_options(DEBUG_OPTIONS) def weave( - weave, - output, - start_date, - end_date, - bbox, - county, - no_bernco, - no_bor, - no_cabq, - no_ebid, - no_nmbgmr_amp, - no_nmed_dwb, - no_nmose_isc_seven_rivers, - no_nmose_roswell, - no_nwis, - no_pvacd, - no_wqp, - site_limit, - dry, + weave, + output, + output_dir, + start_date, + end_date, + bbox, + county, + no_bernco, + no_bor, + no_cabq, + no_ebid, + no_nmbgmr_amp, + no_nmed_dwb, + no_nmose_isc_seven_rivers, + no_nmose_roswell, + no_nwis, + no_pvacd, + no_wqp, + site_limit, + dry, ): """ Get parameter timeseries or summary data @@ -227,14 +244,11 @@ def weave( config = setup_config(f"{parameter}", bbox, county, site_limit, dry) config.parameter = parameter - # make sure config.output_name is properly set - config._update_output_name() - - # make output_path now so that die.log can be written to it live - config._make_output_path() - - # setup logging here so that the path can be set to config.output_path - setup_logging(path=config.output_path) + # # make sure config.output_name is properly set + # config.update_output_name() + # + # # make output_path now so that die.log can be written to it live + # config.make_output_path() # output type if output == "summary": @@ -249,53 +263,33 @@ def weave( summary = False timeseries_unified = False timeseries_separated = True + else: + click.echo(f"Invalid output type: {output}") + return config.output_summary = summary config.output_timeseries_unified = timeseries_unified config.output_timeseries_separated = timeseries_separated + false_agencies = [] + config_agencies = [] # sources if parameter == "waterlevels": - config.use_source_bernco = no_bernco - config.use_source_cabq = no_cabq - config.use_source_ebid = no_ebid - config.use_source_nmbgmr_amp = no_nmbgmr_amp - config.use_source_nmose_isc_seven_rivers = no_nmose_isc_seven_rivers - config.use_source_nmose_roswell = no_nmose_roswell - config.use_source_nwis = no_nwis - config.use_source_pvacd = no_pvacd - config.use_source_wqp = no_wqp - - config.use_source_bor = False - config.use_source_nmed_dwb = False + config_agencies = ["bernco", "cabq", "ebid", "nmbgmr_amp", "nmed_dwb", + "nmose_isc_seven_rivers", "nmose_roswell", "nwis", "pvacd", "wqp"] + + false_agencies = ['bor', 'nmed_dwb'] elif parameter == "carbonate": - config.use_source_nmbgmr_amp = no_nmbgmr_amp - config.use_source_wqp = no_wqp - - config.use_source_bor = False - config.use_source_bernco = False - config.use_source_cabq = False - config.use_source_ebid = False - config.use_source_nmed_dwb = False - config.use_source_nmose_isc_seven_rivers = False - config.use_source_nmose_roswell = False - config.use_source_nwis = False - config.use_source_pvacd = False + config_agencies = ['nmbgmr_amp', 'wqp'] + false_agencies = ['bor', 'bernco', 'cabq', 'ebid', 'nmed_dwb', + 'nmose_isc_seven_rivers', 'nmose_roswell', 'nwis', 'pvacd'] elif parameter in ["arsenic", "uranium"]: - config.use_source_bor = no_bor - config.use_source_nmbgmr_amp = no_nmbgmr_amp - config.use_source_nmed_dwb = no_nmed_dwb - config.use_source_wqp = no_wqp - - config.use_source_bernco = False - config.use_source_cabq = False - config.use_source_ebid = False - config.use_source_nmose_isc_seven_rivers = False - config.use_source_nmose_roswell = False - config.use_source_nwis = False - config.use_source_pvacd = False + config_agencies = ['bor', 'nmbgmr_amp', 'nmed_dwb', 'wqp'] + false_agencies = ['bernco', 'cabq', 'ebid', 'nmose_isc_seven_rivers', + 'nmose_roswell', 'nwis', 'pvacd'] + elif parameter in [ "bicarbonate", @@ -311,31 +305,31 @@ def weave( "sulfate", "tds", ]: - config.use_source_bor = no_bor - config.use_source_nmbgmr_amp = no_nmbgmr_amp - config.use_source_nmed_dwb = no_nmed_dwb - config.use_source_nmose_isc_seven_rivers = no_nmose_isc_seven_rivers - config.use_source_wqp = no_wqp - - config.use_source_bernco = False - config.use_source_cabq = False - config.use_source_ebid = False - config.use_source_nmose_roswell = False - config.use_source_nwis = False - config.use_source_pvacd = False + config_agencies = ['bor', 'nmbgmr_amp', 'nmed_dwb', 'nmose_isc_seven_rivers', 'wqp'] + false_agencies = ['bernco', 'cabq', 'ebid', 'nmose_roswell', 'nwis', 'pvacd'] + if false_agencies: + for agency in false_agencies: + setattr(config, f"use_source_{agency}", False) + + lcs = locals() + if config_agencies: + for agency in config_agencies: + setattr(config, f"use_source_{agency}", lcs.get(f'no_{agency}', False)) # dates config.start_date = start_date config.end_date = end_date + config.finalize() + # setup logging here so that the path can be set to config.output_path + setup_logging(path=config.output_path) + if not dry: config.report() # prompt user to continue if not click.confirm("Do you want to continue?", default=True): return - config._update_output_units() - if parameter.lower() == "waterlevels": unify_waterlevels(config) else: @@ -344,11 +338,49 @@ def weave( @cli.command() @add_options(SPATIAL_OPTIONS) -def wells(bbox, county): +@add_options(PERSISTER_OPTIONS) +@add_options(ALL_SOURCE_OPTIONS) +@add_options(DEBUG_OPTIONS) +def wells(bbox, county, + output_dir, + no_bernco, + no_bor, + no_cabq, + no_ebid, + no_nmbgmr_amp, + no_nmed_dwb, + no_nmose_isc_seven_rivers, + no_nmose_roswell, + no_nwis, + no_pvacd, + no_wqp, + site_limit, + dry, + yes): """ Get locations """ - config = setup_config("sites", bbox, county) + + config = setup_config("sites", bbox, county, site_limit, dry) + config_agencies = ["bernco", "bor", "cabq", "ebid", "nmbgmr_amp", "nmed_dwb", + "nmose_isc_seven_rivers", "nmose_roswell", "nwis", "pvacd", + "wqp"] + lcs = locals() + for agency in config_agencies: + setattr(config, f"use_source_{agency}", lcs.get(f'no_{agency}', False)) + + config.sites_only = True + config.output_dir = output_dir + config.finalize() + # setup logging here so that the path can be set to config.output_path + setup_logging(path=config.output_path) + + config.report() + if not yes: + # prompt user to continue + if not click.confirm("Do you want to continue?", default=True): + return + unify_sites(config) @@ -393,5 +425,4 @@ def setup_config(tag, bbox, county, site_limit, dry): return config - # ============= EOF ============================================= diff --git a/frontend/cronjob_worker.sh b/frontend/cronjob_worker.sh new file mode 100644 index 0000000..4a3925f --- /dev/null +++ b/frontend/cronjob_worker.sh @@ -0,0 +1,3 @@ + + +die weave \ No newline at end of file