feat(ilamb): update ilamb3, remove ohc-NOAA, add evspsbl-pr diagnostic

changelog/573.feature.md

@@ -0,0 +1 @@
+Update `ilamb3`, remove the `ohc-noaa` diagnostic and add the `evspsbl-pr` diagnostic.

packages/climate-ref-core/src/climate_ref_core/data/cmip6_cmip7_variable_map.json

@@ -230,6 +230,21 @@
    "region": "30S-90S",
    "frequency": "mon"
   },
+  "Amon.evspsbl": {
+   "table_id": "Amon",
+   "variable_id": "evspsbl",
+   "cmip6_compound_name": "Amon.evspsbl",
+   "cmip7_compound_name": "atmos.evspsbl.tavg-u-hxy-u.mon.glb",
+   "branded_variable": "evspsbl_tavg-u-hxy-u",
+   "branding_suffix": "tavg-u-hxy-u",
+   "temporal_label": "tavg",
+   "vertical_label": "u",
+   "horizontal_label": "hxy",
+   "area_label": "u",
+   "realm": "atmos land",
+   "region": "glb",
+   "frequency": "mon"
+  },
   "Amon.fco2antt": {
    "table_id": "Amon",
    "variable_id": "fco2antt",
@@ -1985,6 +2000,51 @@
    "region": "glb",
    "frequency": "fx"
   },
+  "Omon.evs": {
+   "table_id": "Omon",
+   "variable_id": "evspsbl",
+   "cmip6_compound_name": "Omon.evs",
+   "cmip7_compound_name": "ocean.evspsbl.tavg-u-hxy-ifs.mon.glb",
+   "branded_variable": "evspsbl_tavg-u-hxy-ifs",
+   "branding_suffix": "tavg-u-hxy-ifs",
+   "temporal_label": "tavg",
+   "vertical_label": "u",
+   "horizontal_label": "hxy",
+   "area_label": "ifs",
+   "realm": "ocean",
+   "region": "glb",
+   "frequency": "mon"
+  },
+  "Omon.msftmrho": {
+   "table_id": "Omon",
+   "variable_id": "msftm",
+   "cmip6_compound_name": "Omon.msftmrho",
+   "cmip7_compound_name": "ocean.msftm.tavg-rho-hyb-sea.mon.glb",
+   "branded_variable": "msftm_tavg-rho-hyb-sea",
+   "branding_suffix": "tavg-rho-hyb-sea",
+   "temporal_label": "tavg",
+   "vertical_label": "rho",
+   "horizontal_label": "hyb",
+   "area_label": "sea",
+   "realm": "ocean",
+   "region": "glb",
+   "frequency": "mon"
+  },
+  "Omon.msftmz": {
+   "table_id": "Omon",
+   "variable_id": "msftm",
+   "cmip6_compound_name": "Omon.msftmz",
+   "cmip7_compound_name": "ocean.msftm.tavg-ol-hyb-sea.mon.glb",
+   "branded_variable": "msftm_tavg-ol-hyb-sea",
+   "branding_suffix": "tavg-ol-hyb-sea",
+   "temporal_label": "tavg",
+   "vertical_label": "ol",
+   "horizontal_label": "hyb",
+   "area_label": "sea",
+   "realm": "ocean",
+   "region": "glb",
+   "frequency": "mon"
+  },
   "Omon.so": {
    "table_id": "Omon",
    "variable_id": "so",
@@ -2210,6 +2270,21 @@
    "region": "30S-90S",
    "frequency": "mon"
   },
+  "SImon.sidmassevapsubl": {
+   "table_id": "SImon",
+   "variable_id": "evspsbl",
+   "cmip6_compound_name": "SImon.sidmassevapsubl",
+   "cmip7_compound_name": "seaIce.evspsbl.tavg-u-hxy-si.mon.glb",
+   "branded_variable": "evspsbl_tavg-u-hxy-si",
+   "branding_suffix": "tavg-u-hxy-si",
+   "temporal_label": "tavg",
+   "vertical_label": "u",
+   "horizontal_label": "hxy",
+   "area_label": "si",
+   "realm": "seaIce",
+   "region": "glb",
+   "frequency": "mon"
+  },
   "SImon.sifllwdtop": {
    "table_id": "SImon",
    "variable_id": "rlds",

packages/climate-ref-core/src/climate_ref_core/datasets.py

@@ -3,7 +3,7 @@
 """
 
 import hashlib
-from collections.abc import Collection, Iterable, Iterator
+from collections.abc import Collection, Iterable, Iterator, Mapping
 from typing import Any
 
 import pandas as pd
@@ -15,7 +15,7 @@
 __all__ = ["Selector", "SourceDatasetType"]
 
 
-def _clean_facets(raw_values: dict[str, str | Collection[str]]) -> dict[str, tuple[str, ...]]:
+def _clean_facets(raw_values: Mapping[str, str | Collection[str]]) -> dict[str, tuple[str, ...]]:
     """
     Clean the value of a facet filter to a tuple of strings
     """

packages/climate-ref-core/src/climate_ref_core/esgf/cmip7.py

@@ -95,7 +95,14 @@ def _convert_file_to_cmip7(cmip6_path: Path, cmip7_facets: dict[str, Any]) -> Pa
         try:
             logger.info(f"Writing translated CMIP7 file: {output_file}")
             suppress_bounds_coordinates(ds_cmip7)
-            ds_cmip7.to_netcdf(output_file)
+
+            # Apply lossy compression - these are converted files used for
+            # verification only, so precision loss is acceptable.
+            encoding = {
+                var: {"zlib": True, "complevel": 5, "least_significant_digit": 3}
+                for var in ds_cmip7.data_vars
+            }
+            ds_cmip7.to_netcdf(output_file, encoding=encoding)
         except PermissionError:
             # If we can't write but file exists (race condition or permission issue), use it
             if output_file.exists():
@@ -224,7 +231,7 @@ def _convert_to_cmip7_metadata(self, cmip6_row: dict[str, Any]) -> dict[str, Any
                 cmip7_row["branding_suffix"] = entry.branding_suffix
                 cmip7_row["branded_variable"] = entry.branded_variable
             except KeyError:
-                logger.debug(
+                logger.error(
                     f"No DReq entry for {table_id}.{variable_id}, region/branding_suffix will not be set"
                 )
 
@@ -273,7 +280,7 @@ def fetch_datasets(self) -> pd.DataFrame:
                         cmip7_path = _convert_file_to_cmip7(cmip6_path, cmip7_row)
                         converted_files.append(str(cmip7_path))
                     except Exception as e:
-                        logger.warning(f"Failed to convert {cmip6_path.name}: {e}")
+                        logger.exception(f"Failed to convert {cmip6_path.name}: {e}")
                         continue
                 else:
                     logger.warning(f"CMIP6 file not found: {file_path}")

packages/climate-ref-ilamb/pyproject.toml

@@ -24,7 +24,7 @@ classifiers = [
 ]
 dependencies = [
   "climate-ref-core",
-  "ilamb3>=2025.9.9,<2025.12",  # to be relaxed in #548
+  "ilamb3>=2026.2.19",
   "scipy<1.16",        # https://github.com/statsmodels/statsmodels/issues/9584
   "xarray<2025.11",    # ilamb3 incompatibility with integrate_space units handling
 ]

packages/climate-ref-ilamb/src/climate_ref_ilamb/configure/ilamb.yaml

@@ -2,10 +2,12 @@ registry: ilamb
 
 gpp-WECANN:
   sources:
-    # TODO: Update to use the obs4REF equiv
+      # TODO: Update to use the obs4REF equiv
     gpp: ilamb/gpp/WECANN/gpp.nc
+    # gpp: obs4REF/ColumbiaU/WECANN-1-0/mon/gpp/gn/20250516/gpp_mon_WECANN-1-0_REF_gn_200701-201512.nc
   relationships:
     pr: ilamb/pr/GPCPv2.3/pr.nc
+    # pr: obs4REF/NOAA-NCEI/GPCP-2-3/mon/pr/gn/v20210727/pr_mon_GPCP-2-3_PCMDI_gn_197901-201907.nc
     tas: ilamb/tas/CRU4.02/tas.nc
   variable_cmap: Greens
 
@@ -16,8 +18,11 @@ gpp-FLUXNET2015:
 
 mrro-LORA:
   sources:
-    # TODO: Update to use the obs4REF equiv
+     # TODO: Update to use the obs4REF equiv
     mrro: ilamb/mrro/LORA/LORA.nc
+    # mrro:
+    #   obs_source: obs4ref
+    #   source_id: LORA-1-1
   variable_cmap: Blues
 
 mrsos-WangMao:
@@ -37,8 +42,12 @@ cSoil-HWSD2:
   variable_cmap: viridis
 
 lai-AVH15C1:
+  # TODO: Update to use the obs4REF equiv
   sources:
     lai: ilamb/lai/AVH15C1/lai.nc
+    # lai:
+    #   obs_source: obs4mips
+    #   source_id: NOAA-NCEI-LAI-AVHRR-5-0
   variable_cmap: Greens
 
 nbp-Hoffman:
@@ -47,6 +56,13 @@ nbp-Hoffman:
   sources:
     # TODO: Update to use the obs4REF equiv
     nbp: ilamb/nbp/HOFFMAN/nbp_1850-2010.nc
+    # nbp:
+    #   obs_source: obs4ref
+    #   variable_id: nbp
+    #   source_id: Hoffman
+  transforms:
+  - integrate_space:
+      varname: nbp
 
 snc-ESACCI:
   sources:
@@ -58,3 +74,17 @@ burntFractionAll-GFED:
   alternate_vars:
   - burntFractionAll
   test_source_id: CESM2
+
+emp-GLEAMGPCP2.3:
+  analysis_variable: emp
+  sources:
+    et: ilamb/evspsbl/GLEAMv3.3a/et.nc
+    pr: obs4REF/NOAA-NCEI/GPCP-2-3/mon/pr/gn/v20210727/pr_mon_GPCP-2-3_PCMDI_gn_197901-201907.nc
+  transforms:
+  - select_time:
+      vmin: 2000-01
+      vmax: 2015-01
+  - expression:
+      expr: emp = et - pr
+  alternate_vars:
+  - evspsbl

packages/climate-ref-ilamb/src/climate_ref_ilamb/configure/iomb.yaml

@@ -2,7 +2,7 @@ registry: iomb
 
 thetao-WOA2023-surface:
   sources:
-    # TODO: Update to use the obs4REF equiv
+    # NOTE: obs4REF equiv starts beyond historical, cannot use for CMIP6
     thetao: ilamb/WOA/thetao_mon_WOA_A5B4_gn_200501-201412.nc
   variable_cmap: Reds
   transforms:
@@ -13,7 +13,7 @@ thetao-WOA2023-surface:
 
 so-WOA2023-surface:
   sources:
-    # TODO: Update to use the obs4REF equiv
+    # NOTE: obs4REF equiv starts beyond historical, cannot use for CMIP6
     so: ilamb/WOA/so_mon_WOA_A5B4_gn_200501-201412.nc
   transforms:
   - select_depth:
@@ -30,20 +30,5 @@ amoc-RAPID:
   transforms:
   - msftmz_to_rapid
   sources:
-    # TODO: Update to use the obs4REF equiv
+    # obs4REF equiv does not work
     amoc: ilamb/RAPID/amoc_mon_RAPID_BE_NA_200404-202302.nc
-
-ohc-NOAA:
-  sources:
-    ohc: ilamb/NOAA/ohc_yr_OHC_BE_gm_200506-202406.nc
-  related_vars:
-  - thetao
-  - volcello
-  transforms:
-  - select_depth:
-      vmin: 0
-      vmax: 2000
-  - ocean_heat_content:
-      reference_year: 2005
-  analyses:
-  - accumulate

packages/climate-ref-ilamb/src/climate_ref_ilamb/dataset_registry/ilamb.txt

@@ -11,3 +11,4 @@ ilamb/tas/CRU4.02/tas.nc sha1:2674da18a1a93483b50b1626e7a7ab741bf53d09
 ilamb/nbp/HOFFMAN/nbp_1850-2010.nc sha1:8350af00614d6afc6b70ad314aa499a9ece80ec2
 ilamb/snc/Snow-cci/snc_mon_Snow-cci_BE_gn_198201-201906.nc sha1:c0bfecd2f8b886e9301428d28bb6ff0507601be2
 ilamb/burntFractionAll/GFED/burntArea.nc sha1:cf9d73c6a8bfc594737c9ba6ca4df613df4a28ab
+ilamb/evspsbl/GLEAMv3.3a/et.nc sha1:5aaf73949af8c6b509ef16f684aa8efeccd983e2

packages/climate-ref-ilamb/src/climate_ref_ilamb/dataset_registry/iomb.txt

@@ -1,4 +1,3 @@
 ilamb/WOA/so_mon_WOA_A5B4_gn_200501-201412.nc sha1:831c42c3b2ba443c255150289a2c725d7f3e5838
 ilamb/WOA/thetao_mon_WOA_A5B4_gn_200501-201412.nc sha1:86d9056208291d76233e65b26c658c1fa54c3ea6
 ilamb/RAPID/amoc_mon_RAPID_BE_NA_200404-202302.nc sha1:3efe773e5c2a3c832977791ff7fd9cb9f473fe65
-ilamb/NOAA/ohc_yr_OHC_BE_gm_200506-202406.nc sha1:a918799d8e24e4f0015b9047a74d470ae9f0445c