diff --git a/intertidal/composites.py b/intertidal/composites.py index f23eaca..197e03b 100644 --- a/intertidal/composites.py +++ b/intertidal/composites.py @@ -198,6 +198,9 @@ def tidal_composites( log.info(f"{run_id}: Loading red band to identify nodata pixels") nodata = satellite_ds.nbart_red.nodata nodata_array = (satellite_ds.nbart_red != nodata).compute() + + # Calculate the total clear pixel count for each pixel + qa_count_clear_total = nodata_array.sum(dim="time").astype("int16") # Mask tides to make nodata match satellite data array tides_highres = tides_highres.where(nodata_array) @@ -262,6 +265,9 @@ def tidal_composites( (ds_low_masked.nbart_red != nodata).sum(dim="time").astype("int16") ) + # Add the total count clear (Only add once) + ds_lowtide["qa_count_clear_total"] = qa_count_clear_total + # Add low and high tide thresholds to the output datasets ds_lowtide["qa_low_threshold"] = low_threshold ds_hightide["qa_high_threshold"] = high_threshold @@ -484,8 +490,12 @@ def tidal_composites_cli( # Record params in logs log.info(f"{run_id}: Using parameters {input_params}") - # Configure S3 + # Configure S3, adding additional GDAL env vars to reduce + # timeout issues when accessing data on S3 + # TODO: pass these directly into `configure_s3_access` configure_s3_access(cloud_defaults=True, aws_unsigned=aws_unsigned) + os.environ["GDAL_HTTP_TIMEOUT"] = "300" # default is 30s + os.environ["GDAL_HTTP_RETRY_DELAY"] = "5" # seconds between retries if process_tile: try: @@ -593,6 +603,7 @@ def tidal_composites_cli( "qa_low_threshold": (np.float32, np.nan), "qa_high_threshold": (np.float32, np.nan), "qa_count_clear": (np.int16, -999), + "qa_count_clear_total": (np.int16, -999), } # Sets correct dtypes and nodata diff --git a/tests/README.md b/tests/README.md index e6bddab..da1e03a 100644 --- a/tests/README.md +++ b/tests/README.md @@ -10,7 +10,7 @@ Integration tests This directory contains tests that are run to verify that DEA Intertidal code runs correctly. The ``test_intertidal.py`` file runs a small-scale full workflow analysis over an intertidal flat in the Gulf of Carpentaria using the DEA Intertidal [Command Line Interface (CLI) tools](../notebooks/Intertidal_CLI.ipynb), and compares these results against a LiDAR validation DEM to produce some simple accuracy metrics. -The latest integration test completed at **2026-03-23 17:31**. Compared to the previous run, it had an: +The latest integration test completed at **2026-03-26 14:06**. Compared to the previous run, it had an: - RMSE accuracy of **0.14 m ( :heavy_minus_sign: no change)** - MAE accuracy of **0.12 m ( :heavy_minus_sign: no change)** - Bias of **0.12 m ( :heavy_minus_sign: no change)** diff --git a/tests/validation.csv b/tests/validation.csv index 366e0c5..6d52324 100644 --- a/tests/validation.csv +++ b/tests/validation.csv @@ -143,3 +143,5 @@ time,Correlation,RMSE,MAE,R-squared,Bias,Regression slope 2026-03-19 02:11:55.124933+00:00,0.975,0.145,0.122,0.95,0.116,1.121 2026-03-19 03:33:01.026986+00:00,0.975,0.145,0.122,0.95,0.116,1.121 2026-03-23 06:31:50.184064+00:00,0.975,0.145,0.122,0.95,0.116,1.121 +2026-03-26 02:07:06.035294+00:00,0.975,0.145,0.122,0.95,0.116,1.121 +2026-03-26 03:06:30.485318+00:00,0.975,0.145,0.122,0.95,0.116,1.121 diff --git a/tests/validation.jpg b/tests/validation.jpg index 3025a5e..af9bc5e 100644 Binary files a/tests/validation.jpg and b/tests/validation.jpg differ