Work on fixing chunk sizes

Author: jacob-a-brown

backend/connectors/nmbgmr/source.py

@@ -72,9 +72,6 @@ def get_records(self):
         if config.has_bounds():
             params["wkt"] = config.bounding_wkt()
 
-        if config.site_limit:
-            params["limit"] = config.site_limit
-
         if not config.sites_only:
 
             if config.parameter.lower() != "waterlevels":
@@ -90,18 +87,18 @@ def get_records(self):
         )
         if not config.sites_only:
             for site in sites:
-                print(f"Obtaining well data for {site['properties']['point_id']}")
-                well_data = self._execute_json_request(
-                    _make_url("wells"),
-                    params={"pointid": site["properties"]["point_id"]},
-                    tag="",
-                )
-                site["properties"]["formation"] = well_data["formation"]
-                site["properties"]["well_depth"] = well_data["well_depth_ftbgs"]
-                site["properties"]["well_depth_units"] = FEET
-                # site["properties"]["formation"] = None
-                # site["properties"]["well_depth"] = None
+                # print(f"Obtaining well data for {site['properties']['point_id']}")
+                # well_data = self._execute_json_request(
+                #     _make_url("wells"),
+                #     params={"pointid": site["properties"]["point_id"]},
+                #     tag="",
+                # )
+                # site["properties"]["formation"] = well_data["formation"]
+                # site["properties"]["well_depth"] = well_data["well_depth_ftbgs"]
                 # site["properties"]["well_depth_units"] = FEET
+                site["properties"]["formation"] = None
+                site["properties"]["well_depth"] = None
+                site["properties"]["well_depth_units"] = FEET
 
         return sites
 

backend/unifier.py

@@ -156,11 +156,18 @@ def _site_wrapper(site_source, parameter_source, persister, config):
                     end_ind += n
 
                 if use_summarize:
+                    print("summarize")
+                    print(sites_with_records_count, site_limit)
                     summary_records = parameter_source.read(
                         sites, use_summarize, start_ind, end_ind
                     )
                     if summary_records:
+                        print("here", len(summary_records))
                         persister.records.extend(summary_records)
+                        sites_with_records_count += len(summary_records)
+                    else:
+                        print("there")
+                        continue
                 else:
                     results = parameter_source.read(
                         sites, use_summarize, start_ind, end_ind
@@ -175,7 +182,8 @@ def _site_wrapper(site_source, parameter_source, persister, config):
                         persister.timeseries.append((site, records))
                         persister.sites.append(site)
 
-                sites_with_records_count += 1
+                    print("incrementing sites_with_records_count")
+                    sites_with_records_count += 1
 
     except BaseException:
         import traceback