diff --git a/amocatlas/data_sources/fbc.py b/amocatlas/data_sources/fbc.py index d94c4a7..290471d 100644 --- a/amocatlas/data_sources/fbc.py +++ b/amocatlas/data_sources/fbc.py @@ -19,8 +19,6 @@ from typing import Union import xarray as xr -import datetime -import pandas as pd # Import the modules used from amocatlas import logger, utilities @@ -34,21 +32,19 @@ DATASOURCE_ID = "fbc" # Default list of FBC data files -FBC_DEFAULT_FILES = [ - "FBC_overflow_transport.txt", -] -FBC_TRANSPORT_FILES = ["FBC_overflow_transport.txt"] -FBC_DEFAULT_SOURCE = "https://envofar.fo/var/ftp/Timeseries/" +FBC_DEFAULT_FILES = ["OS_GSR_FBC_D_1995_2024.nc"] +FBC_TRANSPORT_FILES = ["OS_GSR_FBC_D_1995_2024.nc"] +FBC_DEFAULT_SOURCE = "https://zenodo.org/records/19554222/files/" FBC_METADATA = { - "project": "Faroe Bank Channel overflow 1995-2015", - "weblink": "https://envofar.fo/var/ftp/Timeseries/FBC_overflow_transport.txt", + "project": "Faroe Bank Channel overflow 1995-2024", + "weblink": "https://zenodo.org/records/19554222", "comment": "Dataset accessed and processed via http://github.com/AMOCcommunity/amocatlas", } FBC_FILE_METADATA = { - "FBC_overflow_transport.txt": { - "data_product": "Daily averaged kinematic FBC-overflow flux (transport) in Sv", + "OS_GSR_FBC_D_1995_2024.nc": { + "data_product": "Daily average of FBC overflow transport, estimated from an array of moored ADCPs in Sv", }, } @@ -124,7 +120,7 @@ def read_fbc( added_attrs_per_dataset = [] if track_added_attrs else None for file in file_list: - if not (file.lower().endswith(".txt")): + if not (file.lower().endswith(".nc")): log_warning("Skipping unsupported file type : %s", file) continue @@ -141,57 +137,11 @@ def read_fbc( ) # Open dataset - if file.lower().endswith(".txt"): - # file.txt - try: - # column_names, _ = utilities.parse_ascii_header( - # file_path, comment_char="%" - # ) - data_start = utilities.find_data_start(file_path) - - df = pd.read_csv( - file_path, - sep=r"\s+", - encoding="latin-1", - skiprows=data_start, - names=["Decimal year", "Month", "Day", "Flux"], - ) - except Exception as e: - log_error("Failed to parse ASCII file: %s: %s", file_path, e) - raise FileNotFoundError( - f"Failed to parse ASCII file: {file_path}: {e}" - ) from e - - # Time handling - try: - df = df.apply( - lambda col: col.astype(str) - .str.replace(",", "", regex=False) - .astype(float) - ) - # df['Decimal year'] = df['Decimal year'].astype(str).str.replace(',', '',regex=False).astype(float) - df["TIME"] = df["Decimal year"].apply( - lambda x: datetime.datetime(int(x), 1, 1) - + datetime.timedelta( - days=(x - int(x)) - * ( - datetime.datetime(int(x) + 1, 1, 1) - - datetime.datetime(int(x), 1, 1) - ).days - ) - ) - df = df.drop(columns=["Decimal year"]) - ds = df.set_index("TIME").to_xarray() - except Exception as e: - log_error( - "Failed to convert DataFrame to xarray Dataset for %s: %s", - file, - e, - ) - raise ValueError( - f"Failed to convert DataFrame to xarray Dataset for {file}: {e}", - ) from e + if file.lower().endswith(".nc"): + # Use ReaderUtils for consistent dataset loading + + ds = ReaderUtils.safe_load_dataset(file_path) # Attach metadata # Attach metadata with optional tracking @@ -222,6 +172,10 @@ def read_fbc( DATASOURCE_ID, track_added_attrs=False, ) + else: + raise ValueError( + f"Unsupported file type for {file}. Only .nc files are supported." + ) datasets.append(ds) diff --git a/amocatlas/metadata/fbc.yml b/amocatlas/metadata/fbc.yml index 0c679d5..f0fd4d0 100644 --- a/amocatlas/metadata/fbc.yml +++ b/amocatlas/metadata/fbc.yml @@ -1,8 +1,8 @@ metadata: program: "FBC" description: "FBC Overflow transport time series" - project: "B. Hansen et al.: A stable Faroe Bank Channel overflow" - weblink: https://envofar.fo/data/index.php?dir=Timeseries&sort=N&order=A + project: "B. Hansen, K. M. Larsen: NACLIM - Fluxes: Faroe Bank Channel overflow transport" + weblink: https://zenodo.org/records/19554222 comment: Dataset accessed and processed via http://github.com/AMOCcommunity/amocatlas acknowledgment: > Funding for the in situ Faroe Bank Channel measurements is from the Environmental Research Programme of the Nordic Council of Ministers (NMR) 1993–1998, from national Nordic research councils, from the Danish DANCEA programme, and from the European Framework Programs, lately under grant agreement no. GA212643 (THOR) and under grant agreement no. 308299 (NACLIM). @@ -11,29 +11,18 @@ metadata: license: featureType: timeSeries time_coverage_start: '1995-11-13' - time_coverage_end: '2023-05-19' + time_coverage_end: '2024-05-18' contributing_institutions: "Faroe Marine Research Institute (FAMRI)" files: - FBC_overflow_transport.txt: - source_url: https://envofar.fo/var/ftp/Timeseries/FBC_overflow_transport.txt + OS_GSR_FBC_D_1995_2024.nc: + source_url: https://zenodo.org/records/19554222/files/ data_product: "Daily averaged kinematic FBC-overflow flux (transport) in Sv" - variables_to_remove: [Day, Month] variable_mapping: - "Flux": TRANS_FBC + "FBC_tr": TRANS_FBC original_variable_metadata: - Flux: + FBC_tr: long_name: "FBC Overflow" description: "FBC Overflow transport time series" units: Sverdrup - standard_name: ocean_volume_transport_across_line - Month: - long_name: "Month" - description: "Month" - units: - standard_name: - Day: - long_name: "Day" - description: "Day" - units: - standard_name: + standard_name: ocean_volume_transport_across_line \ No newline at end of file diff --git a/data/OS_GSR_FBC_D_1995_2024.nc b/data/OS_GSR_FBC_D_1995_2024.nc new file mode 100644 index 0000000..56a822c Binary files /dev/null and b/data/OS_GSR_FBC_D_1995_2024.nc differ diff --git a/tests/test_fbc.py b/tests/test_fbc.py index 0615507..9c56171 100644 --- a/tests/test_fbc.py +++ b/tests/test_fbc.py @@ -29,15 +29,15 @@ def test_default_files_configuration(self): """Test default files configuration is reasonable.""" files = fbc.FBC_DEFAULT_FILES assert len(files) > 0 - assert "FBC_overflow_transport.txt" in files + assert "OS_GSR_FBC_D_1995_2024.nc" in files transport_files = fbc.FBC_TRANSPORT_FILES assert len(transport_files) > 0 - assert "FBC_overflow_transport.txt" in transport_files + assert "OS_GSR_FBC_D_1995_2024.nc" in transport_files - # Files should be text format for FBC data + # Files should be NetCDF format for FBC data for file in files: - assert file.endswith(".txt") + assert file.endswith(".nc") def test_function_exists_and_callable(self): """Test that main function exists and is callable.""" @@ -62,4 +62,3 @@ def test_module_imports_successfully(self): assert fbc is not None # Check required dependencies are accessible assert hasattr(fbc, "xr") # xarray - assert hasattr(fbc, "pd") # pandas