updating mosaic metadata branch

intertidal/composites.py

@@ -198,6 +198,9 @@ def tidal_composites(
     log.info(f"{run_id}: Loading red band to identify nodata pixels")
     nodata = satellite_ds.nbart_red.nodata
     nodata_array = (satellite_ds.nbart_red != nodata).compute()
+
+    # Calculate the total clear pixel count for each pixel
+    qa_count_clear_total = nodata_array.sum(dim="time").astype("int16")
 
     # Mask tides to make nodata match satellite data array
     tides_highres = tides_highres.where(nodata_array)
@@ -262,6 +265,9 @@ def tidal_composites(
         (ds_low_masked.nbart_red != nodata).sum(dim="time").astype("int16")
     )
 
+    # Add the total count clear (Only add once)
+    ds_lowtide["qa_count_clear_total"] = qa_count_clear_total
+
     # Add low and high tide thresholds to the output datasets
     ds_lowtide["qa_low_threshold"] = low_threshold
     ds_hightide["qa_high_threshold"] = high_threshold
@@ -484,8 +490,12 @@ def tidal_composites_cli(
     # Record params in logs
     log.info(f"{run_id}: Using parameters {input_params}")
 
-    # Configure S3
+    # Configure S3, adding additional GDAL env vars to reduce
+    # timeout issues when accessing data on S3
+    # TODO: pass these directly into `configure_s3_access`
     configure_s3_access(cloud_defaults=True, aws_unsigned=aws_unsigned)
+    os.environ["GDAL_HTTP_TIMEOUT"] = "300"    # default is 30s
+    os.environ["GDAL_HTTP_RETRY_DELAY"] = "5"  # seconds between retries
 
     if process_tile:
         try:
@@ -593,6 +603,7 @@ def tidal_composites_cli(
                 "qa_low_threshold": (np.float32, np.nan),
                 "qa_high_threshold": (np.float32, np.nan),
                 "qa_count_clear": (np.int16, -999),
+                "qa_count_clear_total": (np.int16, -999),
             }
 
             # Sets correct dtypes and nodata

tests/README.md

@@ -10,7 +10,7 @@ Integration tests
 
 This directory contains tests that are run to verify that DEA Intertidal code runs correctly. The ``test_intertidal.py`` file runs a small-scale full workflow analysis over an intertidal flat in the Gulf of Carpentaria using the DEA Intertidal [Command Line Interface (CLI) tools](../notebooks/Intertidal_CLI.ipynb), and compares these results against a LiDAR validation DEM to produce some simple accuracy metrics.
 
-The latest integration test completed at **2026-03-23 17:31**. Compared to the previous run, it had an:
+The latest integration test completed at **2026-03-26 14:06**. Compared to the previous run, it had an:
 - RMSE accuracy of **0.14 m ( :heavy_minus_sign: no change)**
 - MAE accuracy of **0.12 m ( :heavy_minus_sign: no change)**
 - Bias of **0.12 m ( :heavy_minus_sign: no change)**

tests/validation.csv

@@ -143,3 +143,5 @@ time,Correlation,RMSE,MAE,R-squared,Bias,Regression slope
 2026-03-19 02:11:55.124933+00:00,0.975,0.145,0.122,0.95,0.116,1.121
 2026-03-19 03:33:01.026986+00:00,0.975,0.145,0.122,0.95,0.116,1.121
 2026-03-23 06:31:50.184064+00:00,0.975,0.145,0.122,0.95,0.116,1.121
+2026-03-26 02:07:06.035294+00:00,0.975,0.145,0.122,0.95,0.116,1.121
+2026-03-26 03:06:30.485318+00:00,0.975,0.145,0.122,0.95,0.116,1.121