From 4725a439f7ec17f788e86dceab8e02b26f91f0f3 Mon Sep 17 00:00:00 2001 From: FlxPo Date: Mon, 30 Mar 2026 15:30:29 +0200 Subject: [PATCH 1/3] replace data.cquest.org downloads with carte.gouv.fr downloads --- mobility/parsers/admin_boundaries.py | 2 +- mobility/parsers/local_admin_units.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mobility/parsers/admin_boundaries.py b/mobility/parsers/admin_boundaries.py index d6e41ac2..df2efcdc 100644 --- a/mobility/parsers/admin_boundaries.py +++ b/mobility/parsers/admin_boundaries.py @@ -12,7 +12,7 @@ def prepare_french_admin_boundaries(): logging.info("Preparing french city limits...") - url = "https://data.cquest.org/ign/adminexpress/ADMIN-EXPRESS-COG-CARTO_3-2__SHP_LAMB93_FXX_2023-05-03.7z" + url = "https://data.geopf.fr/telechargement/download/ADMIN-EXPRESS-COG-CARTO/ADMIN-EXPRESS-COG-CARTO_3-2__SHP_LAMB93_FXX_2024-02-22/ADMIN-EXPRESS-COG-CARTO_3-2__SHP_LAMB93_FXX_2024-02-22.7z" path = pathlib.Path(os.environ["MOBILITY_PACKAGE_DATA_FOLDER"]) / "ign/admin-express/ADMIN-EXPRESS-COG-CARTO_3-2__SHP_LAMB93_FXX_2023-05-03.7z" download_file(url, path) diff --git a/mobility/parsers/local_admin_units.py b/mobility/parsers/local_admin_units.py index c4d97cf0..3dc690e8 100644 --- a/mobility/parsers/local_admin_units.py +++ b/mobility/parsers/local_admin_units.py @@ -16,7 +16,7 @@ class LocalAdminUnits(FileAsset): Use .get() method to get its content (under Parquet format). - In France, uses adminexpress base from IGN, stored on cquest.org. For Paris, Lyon and Marseille, each 'arrondissement' is considered a distinct admin unit. + In France, uses adminexpress base from IGN, stored on https://cartes.gouv.fr/. For Paris, Lyon and Marseille, each 'arrondissement' is considered a distinct admin unit. In Switzerland, uses swisstopo data stored on geo.admin.ch @@ -64,7 +64,7 @@ def prepare_french_local_admin_units(self): logging.info("Preparing french city limits...") - url = "https://data.cquest.org/ign/adminexpress/ADMIN-EXPRESS-COG-CARTO_3-2__SHP_LAMB93_FXX_2023-05-03.7z" + url = "https://data.geopf.fr/telechargement/download/ADMIN-EXPRESS-COG-CARTO/ADMIN-EXPRESS-COG-CARTO_3-2__SHP_LAMB93_FXX_2024-02-22/ADMIN-EXPRESS-COG-CARTO_3-2__SHP_LAMB93_FXX_2024-02-22.7z" path = pathlib.Path(os.environ["MOBILITY_PACKAGE_DATA_FOLDER"]) / "ign/admin-express/ADMIN-EXPRESS-COG-CARTO_3-2__SHP_LAMB93_FXX_2023-05-03.7z" download_file(url, path) From 021ac03dbce4a60db53d7f447251b7d14ece71bb Mon Sep 17 00:00:00 2001 From: FlxPo Date: Mon, 30 Mar 2026 15:49:59 +0200 Subject: [PATCH 2/3] fix folder structure --- mobility/parsers/admin_boundaries.py | 2 +- mobility/parsers/local_admin_units.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mobility/parsers/admin_boundaries.py b/mobility/parsers/admin_boundaries.py index df2efcdc..708c490e 100644 --- a/mobility/parsers/admin_boundaries.py +++ b/mobility/parsers/admin_boundaries.py @@ -21,7 +21,7 @@ def prepare_french_admin_boundaries(): # Convert to geoparquet path = path.parent / "ADMIN-EXPRESS-COG-CARTO_3-2__SHP_LAMB93_FXX_2023-05-03" / \ - "ADMIN-EXPRESS-COG-CARTO" / "1_DONNEES_LIVRAISON_2023-05-03" / "ADECOGC_3-2_SHP_LAMB93_FXX" + "ADMIN-EXPRESS-COG-CARTO" / "1_DONNEES_LIVRAISON_2023-05-03" / "ADECOGC_3-2_SHP_LAMB93_FXX-ED2024-02-22" for shp_file in ["ARRONDISSEMENT_MUNICIPAL.shp", "COMMUNE.shp", "EPCI.shp", "REGION.shp"]: diff --git a/mobility/parsers/local_admin_units.py b/mobility/parsers/local_admin_units.py index 3dc690e8..e8ee1bef 100644 --- a/mobility/parsers/local_admin_units.py +++ b/mobility/parsers/local_admin_units.py @@ -73,7 +73,7 @@ def prepare_french_local_admin_units(self): # Convert to geoparquet path = path.parent / "ADMIN-EXPRESS-COG-CARTO_3-2__SHP_LAMB93_FXX_2023-05-03" / \ - "ADMIN-EXPRESS-COG-CARTO" / "1_DONNEES_LIVRAISON_2023-05-03" / "ADECOGC_3-2_SHP_LAMB93_FXX" + "ADMIN-EXPRESS-COG-CARTO" / "1_DONNEES_LIVRAISON_2023-05-03" / "ADECOGC_3-2_SHP_LAMB93_FXX-ED2024-02-22" # Replace Paris / Lyon / Marseille cities with their constituting arrondissements arrond = gpd.read_file(path / "ARRONDISSEMENT_MUNICIPAL.shp") From 2f5ff0d5ea06bb9debfa88122ba174f36750b090 Mon Sep 17 00:00:00 2001 From: FlxPo Date: Mon, 30 Mar 2026 16:12:47 +0200 Subject: [PATCH 3/3] fix folder structure 2 --- mobility/parsers/admin_boundaries.py | 11 ++- mobility/parsers/local_admin_units.py | 9 +- mobility/spatial/local_admin_units.py | 133 ++++++++++++++++++++++++++ 3 files changed, 148 insertions(+), 5 deletions(-) create mode 100644 mobility/spatial/local_admin_units.py diff --git a/mobility/parsers/admin_boundaries.py b/mobility/parsers/admin_boundaries.py index 708c490e..12ccc914 100644 --- a/mobility/parsers/admin_boundaries.py +++ b/mobility/parsers/admin_boundaries.py @@ -20,9 +20,14 @@ def prepare_french_admin_boundaries(): z.extractall(path.parent) # Convert to geoparquet - path = path.parent / "ADMIN-EXPRESS-COG-CARTO_3-2__SHP_LAMB93_FXX_2023-05-03" / \ - "ADMIN-EXPRESS-COG-CARTO" / "1_DONNEES_LIVRAISON_2023-05-03" / "ADECOGC_3-2_SHP_LAMB93_FXX-ED2024-02-22" - + path = ( + path.parent + / "ADMIN-EXPRESS-COG-CARTO_3-2__SHP_LAMB93_FXX_2024-02-22" + / "ADMIN-EXPRESS-COG-CARTO" + / "1_DONNEES_LIVRAISON_2024-03-00169" + / "ADECOGC_3-2_SHP_LAMB93_FXX-ED2024-02-22" + ) + for shp_file in ["ARRONDISSEMENT_MUNICIPAL.shp", "COMMUNE.shp", "EPCI.shp", "REGION.shp"]: df = gpd.read_file(path / shp_file) diff --git a/mobility/parsers/local_admin_units.py b/mobility/parsers/local_admin_units.py index e8ee1bef..d3d73551 100644 --- a/mobility/parsers/local_admin_units.py +++ b/mobility/parsers/local_admin_units.py @@ -72,8 +72,13 @@ def prepare_french_local_admin_units(self): z.extractall(path.parent) # Convert to geoparquet - path = path.parent / "ADMIN-EXPRESS-COG-CARTO_3-2__SHP_LAMB93_FXX_2023-05-03" / \ - "ADMIN-EXPRESS-COG-CARTO" / "1_DONNEES_LIVRAISON_2023-05-03" / "ADECOGC_3-2_SHP_LAMB93_FXX-ED2024-02-22" + path = ( + path.parent + / "ADMIN-EXPRESS-COG-CARTO_3-2__SHP_LAMB93_FXX_2024-02-22" + / "ADMIN-EXPRESS-COG-CARTO" + / "1_DONNEES_LIVRAISON_2024-03-00169" + / "ADECOGC_3-2_SHP_LAMB93_FXX-ED2024-02-22" + ) # Replace Paris / Lyon / Marseille cities with their constituting arrondissements arrond = gpd.read_file(path / "ARRONDISSEMENT_MUNICIPAL.shp") diff --git a/mobility/spatial/local_admin_units.py b/mobility/spatial/local_admin_units.py new file mode 100644 index 00000000..9700ed83 --- /dev/null +++ b/mobility/spatial/local_admin_units.py @@ -0,0 +1,133 @@ +import os +import pathlib +import logging +import zipfile +import py7zr +import shapely +import geopandas as gpd +import pandas as pd + +from mobility.runtime.assets.file_asset import FileAsset +from mobility.runtime.io.download_file import download_file +from mobility.spatial.local_admin_units_categories import LocalAdminUnitsCategories + +class LocalAdminUnits(FileAsset): + """FileAsset class preparing local admin units in France and Switzerland. + + Use .get() method to get its content (under Parquet format). + + In France, uses adminexpress base from IGN, stored on cartes.gouv.fr. For Paris, Lyon and Marseille, each 'arrondissement' is considered a distinct admin unit. + + In Switzerland, uses swisstopo data stored on geo.admin.ch + + Data from both countries is merged and stored in Parquet format under coordinates system EPSG:3035. + """ + + def __init__(self): + + inputs = { + "categories": LocalAdminUnitsCategories() + } + + cache_path = pathlib.Path(os.environ["MOBILITY_PACKAGE_DATA_FOLDER"]) / "local_admin_units.parquet" + super().__init__(inputs, cache_path) + + def get_cached_asset(self) -> pd.DataFrame: + + logging.info("Local administrative units already prepared. Reusing the file : " + str(self.cache_path)) + local_admin_units = gpd.read_parquet(self.cache_path) + + return local_admin_units + + + def create_and_get_asset(self) -> pd.DataFrame: + + logging.info("Preparing local administrative units.") + + local_admin_units_fr = self.prepare_french_local_admin_units() + local_admin_units_ch = self.prepare_swiss_local_admin_units() + + local_admin_units = pd.concat([local_admin_units_fr, local_admin_units_ch]) + + local_admin_units = pd.merge( + local_admin_units, + self.inputs["categories"].get(), + on="local_admin_unit_id" + ) + + local_admin_units.to_parquet(self.cache_path) + + return local_admin_units + + + def prepare_french_local_admin_units(self): + + logging.info("Preparing french city limits...") + + url = "https://data.geopf.fr/telechargement/download/ADMIN-EXPRESS-COG-CARTO/ADMIN-EXPRESS-COG-CARTO_3-2__SHP_LAMB93_FXX_2024-02-22/ADMIN-EXPRESS-COG-CARTO_3-2__SHP_LAMB93_FXX_2024-02-22.7z" + path = pathlib.Path(os.environ["MOBILITY_PACKAGE_DATA_FOLDER"]) / "ign/admin-express/ADMIN-EXPRESS-COG-CARTO_3-2__SHP_LAMB93_FXX_2023-05-03.7z" + download_file(url, path) + + with py7zr.SevenZipFile(path, "r") as z: + z.extractall(path.parent) + + # Convert to geoparquet + path = path.parent / "ADMIN-EXPRESS-COG-CARTO_3-2__SHP_LAMB93_FXX_2023-05-03" / \ + "ADMIN-EXPRESS-COG-CARTO" / "1_DONNEES_LIVRAISON_2024-03-00169" / "ADECOGC_3-2_SHP_LAMB93_FXX" + + # Replace Paris / Lyon / Marseille cities with their constituting arrondissements + arrond = gpd.read_file(path / "ARRONDISSEMENT_MUNICIPAL.shp") + cities = gpd.read_file(path / "COMMUNE.shp") + + cities = cities[["INSEE_COM", "NOM", "geometry"]] + arrond = arrond[["INSEE_COM", "INSEE_ARM", "NOM", "geometry"]] + + cities_with_arrond = arrond["INSEE_COM"].unique() + + arrond["INSEE_COM"] = arrond["INSEE_ARM"] + arrond = arrond[["INSEE_COM", "NOM", "geometry"]] + + cities = cities[~cities["INSEE_COM"].isin(cities_with_arrond)] + + cities = pd.concat([ + cities, + arrond + ]) + + cities.columns = ["local_admin_unit_id", "local_admin_unit_name", "geometry"] + + cities["local_admin_unit_id"] = "fr-" + cities["local_admin_unit_id"] + cities["country"] = "fr" + + cities = cities.to_crs(3035) + + return cities + + + def prepare_swiss_local_admin_units(self): + + url = "https://data.geo.admin.ch/ch.swisstopo.swissboundaries3d/swissboundaries3d_2024-01/swissboundaries3d_2024-01_2056_5728.gpkg.zip" + + data_folder = pathlib.Path(os.environ["MOBILITY_PACKAGE_DATA_FOLDER"]) / "swisstopo" + zip_path = data_folder / "swissboundaries3d_2024-01_2056_5728.gpkg.zip" + file_path = data_folder / "swissBOUNDARIES3D_1_5_LV95_LN02.gpkg" + + download_file(url, zip_path) + + # Unzip the content + with zipfile.ZipFile(zip_path, "r") as zip_ref: + zip_ref.extractall(data_folder) + + + cities = gpd.read_file(file_path, layer="tlm_hoheitsgebiet") + + cities = cities[["bfs_nummer", "name", "geometry"]].copy() + cities.columns = ["local_admin_unit_id", "local_admin_unit_name", "geometry"] + + cities["local_admin_unit_id"] = "ch-" + cities["local_admin_unit_id"].astype(str) + cities["country"] = "ch" + + cities["geometry"] = shapely.wkb.loads(shapely.wkb.dumps(cities["geometry"], output_dimension=2)) + cities = cities.to_crs(3035) + + return cities