Replace data.cquest.org downloads with carte.gouv.fr downloads

mobility/parsers/admin_boundaries.py

@@ -12,17 +12,22 @@ def prepare_french_admin_boundaries():
 
     logging.info("Preparing french city limits...")
 
-    url = "https://data.cquest.org/ign/adminexpress/ADMIN-EXPRESS-COG-CARTO_3-2__SHP_LAMB93_FXX_2023-05-03.7z"
+    url = "https://data.geopf.fr/telechargement/download/ADMIN-EXPRESS-COG-CARTO/ADMIN-EXPRESS-COG-CARTO_3-2__SHP_LAMB93_FXX_2024-02-22/ADMIN-EXPRESS-COG-CARTO_3-2__SHP_LAMB93_FXX_2024-02-22.7z"
     path = pathlib.Path(os.environ["MOBILITY_PACKAGE_DATA_FOLDER"]) / "ign/admin-express/ADMIN-EXPRESS-COG-CARTO_3-2__SHP_LAMB93_FXX_2023-05-03.7z"
     download_file(url, path)
 
     with py7zr.SevenZipFile(path, "r") as z:
         z.extractall(path.parent)
 
     # Convert to geoparquet
-    path = path.parent / "ADMIN-EXPRESS-COG-CARTO_3-2__SHP_LAMB93_FXX_2023-05-03" / \
-     "ADMIN-EXPRESS-COG-CARTO" / "1_DONNEES_LIVRAISON_2023-05-03" / "ADECOGC_3-2_SHP_LAMB93_FXX"
-
+    path = ( 
+        path.parent 
+        / "ADMIN-EXPRESS-COG-CARTO_3-2__SHP_LAMB93_FXX_2024-02-22" 
+        / "ADMIN-EXPRESS-COG-CARTO" 
+        / "1_DONNEES_LIVRAISON_2024-03-00169" 
+        / "ADECOGC_3-2_SHP_LAMB93_FXX-ED2024-02-22"
+    )
+
     for shp_file in ["ARRONDISSEMENT_MUNICIPAL.shp", "COMMUNE.shp", "EPCI.shp", "REGION.shp"]:
 
          df = gpd.read_file(path / shp_file)

mobility/parsers/local_admin_units.py

@@ -16,7 +16,7 @@ class LocalAdminUnits(FileAsset):
 
     Use .get() method to get its content (under Parquet format).
 
-    In France, uses adminexpress base from IGN, stored on cquest.org. For Paris, Lyon and Marseille, each 'arrondissement' is considered a distinct admin unit.
+    In France, uses adminexpress base from IGN, stored on https://cartes.gouv.fr/. For Paris, Lyon and Marseille, each 'arrondissement' is considered a distinct admin unit.
 
     In Switzerland, uses swisstopo data stored on geo.admin.ch
 
@@ -64,16 +64,21 @@ def prepare_french_local_admin_units(self):
 
         logging.info("Preparing french city limits...")
 
-        url = "https://data.cquest.org/ign/adminexpress/ADMIN-EXPRESS-COG-CARTO_3-2__SHP_LAMB93_FXX_2023-05-03.7z"
+        url = "https://data.geopf.fr/telechargement/download/ADMIN-EXPRESS-COG-CARTO/ADMIN-EXPRESS-COG-CARTO_3-2__SHP_LAMB93_FXX_2024-02-22/ADMIN-EXPRESS-COG-CARTO_3-2__SHP_LAMB93_FXX_2024-02-22.7z"
         path = pathlib.Path(os.environ["MOBILITY_PACKAGE_DATA_FOLDER"]) / "ign/admin-express/ADMIN-EXPRESS-COG-CARTO_3-2__SHP_LAMB93_FXX_2023-05-03.7z"
         download_file(url, path)
 
         with py7zr.SevenZipFile(path, "r") as z:
             z.extractall(path.parent)
 
         # Convert to geoparquet
-        path = path.parent / "ADMIN-EXPRESS-COG-CARTO_3-2__SHP_LAMB93_FXX_2023-05-03" / \
-         "ADMIN-EXPRESS-COG-CARTO" / "1_DONNEES_LIVRAISON_2023-05-03" / "ADECOGC_3-2_SHP_LAMB93_FXX"
+        path = ( 
+            path.parent 
+            / "ADMIN-EXPRESS-COG-CARTO_3-2__SHP_LAMB93_FXX_2024-02-22" 
+            / "ADMIN-EXPRESS-COG-CARTO" 
+            / "1_DONNEES_LIVRAISON_2024-03-00169" 
+            / "ADECOGC_3-2_SHP_LAMB93_FXX-ED2024-02-22"
+        )
 
         # Replace Paris / Lyon / Marseille cities with their constituting arrondissements
         arrond = gpd.read_file(path / "ARRONDISSEMENT_MUNICIPAL.shp")

mobility/spatial/local_admin_units.py

@@ -0,0 +1,133 @@
+import os
+import pathlib
+import logging
+import zipfile
+import py7zr
+import shapely
+import geopandas as gpd
+import pandas as pd
+
+from mobility.runtime.assets.file_asset import FileAsset
+from mobility.runtime.io.download_file import download_file
+from mobility.spatial.local_admin_units_categories import LocalAdminUnitsCategories
+
+class LocalAdminUnits(FileAsset):
+    """FileAsset class preparing local admin units in France and Switzerland.
+
+    Use .get() method to get its content (under Parquet format).
+
+    In France, uses adminexpress base from IGN, stored on cartes.gouv.fr. For Paris, Lyon and Marseille, each 'arrondissement' is considered a distinct admin unit.
+
+    In Switzerland, uses swisstopo data stored on geo.admin.ch
+
+    Data from both countries is merged and stored in Parquet format under coordinates system EPSG:3035.
+    """
+
+    def __init__(self):
+
+        inputs = {
+            "categories": LocalAdminUnitsCategories()
+        }
+
+        cache_path = pathlib.Path(os.environ["MOBILITY_PACKAGE_DATA_FOLDER"]) / "local_admin_units.parquet"
+        super().__init__(inputs, cache_path)
+
+    def get_cached_asset(self) -> pd.DataFrame:
+
+        logging.info("Local administrative units already prepared. Reusing the file : " + str(self.cache_path))
+        local_admin_units = gpd.read_parquet(self.cache_path)
+
+        return local_admin_units
+
+
+    def create_and_get_asset(self) -> pd.DataFrame:
+
+        logging.info("Preparing local administrative units.")
+
+        local_admin_units_fr = self.prepare_french_local_admin_units()
+        local_admin_units_ch = self.prepare_swiss_local_admin_units()
+
+        local_admin_units = pd.concat([local_admin_units_fr, local_admin_units_ch])
+
+        local_admin_units = pd.merge(
+            local_admin_units,
+            self.inputs["categories"].get(),
+            on="local_admin_unit_id"
+        )
+
+        local_admin_units.to_parquet(self.cache_path)
+
+        return local_admin_units
+
+
+    def prepare_french_local_admin_units(self):
+
+        logging.info("Preparing french city limits...")
+
+        url = "https://data.geopf.fr/telechargement/download/ADMIN-EXPRESS-COG-CARTO/ADMIN-EXPRESS-COG-CARTO_3-2__SHP_LAMB93_FXX_2024-02-22/ADMIN-EXPRESS-COG-CARTO_3-2__SHP_LAMB93_FXX_2024-02-22.7z"
+        path = pathlib.Path(os.environ["MOBILITY_PACKAGE_DATA_FOLDER"]) / "ign/admin-express/ADMIN-EXPRESS-COG-CARTO_3-2__SHP_LAMB93_FXX_2023-05-03.7z"
+        download_file(url, path)
+
+        with py7zr.SevenZipFile(path, "r") as z:
+            z.extractall(path.parent)
+
+        # Convert to geoparquet
+        path = path.parent / "ADMIN-EXPRESS-COG-CARTO_3-2__SHP_LAMB93_FXX_2023-05-03" / \
+         "ADMIN-EXPRESS-COG-CARTO" / "1_DONNEES_LIVRAISON_2024-03-00169" / "ADECOGC_3-2_SHP_LAMB93_FXX"
+
+        # Replace Paris / Lyon / Marseille cities with their constituting arrondissements
+        arrond = gpd.read_file(path / "ARRONDISSEMENT_MUNICIPAL.shp")
+        cities = gpd.read_file(path / "COMMUNE.shp")
+
+        cities = cities[["INSEE_COM", "NOM", "geometry"]]
+        arrond = arrond[["INSEE_COM", "INSEE_ARM", "NOM", "geometry"]]
+
+        cities_with_arrond = arrond["INSEE_COM"].unique()
+
+        arrond["INSEE_COM"] = arrond["INSEE_ARM"]
+        arrond = arrond[["INSEE_COM", "NOM", "geometry"]]
+
+        cities = cities[~cities["INSEE_COM"].isin(cities_with_arrond)]
+
+        cities = pd.concat([
+            cities,
+            arrond
+        ])
+
+        cities.columns = ["local_admin_unit_id", "local_admin_unit_name", "geometry"]
+
+        cities["local_admin_unit_id"] = "fr-" + cities["local_admin_unit_id"]
+        cities["country"] = "fr"
+
+        cities = cities.to_crs(3035)
+
+        return cities
+
+
+    def prepare_swiss_local_admin_units(self):
+
+        url = "https://data.geo.admin.ch/ch.swisstopo.swissboundaries3d/swissboundaries3d_2024-01/swissboundaries3d_2024-01_2056_5728.gpkg.zip"
+
+        data_folder = pathlib.Path(os.environ["MOBILITY_PACKAGE_DATA_FOLDER"]) / "swisstopo"
+        zip_path = data_folder / "swissboundaries3d_2024-01_2056_5728.gpkg.zip"
+        file_path = data_folder / "swissBOUNDARIES3D_1_5_LV95_LN02.gpkg"
+
+        download_file(url, zip_path)
+
+        # Unzip the content
+        with zipfile.ZipFile(zip_path, "r") as zip_ref:
+            zip_ref.extractall(data_folder)
+
+
+        cities = gpd.read_file(file_path, layer="tlm_hoheitsgebiet")
+
+        cities = cities[["bfs_nummer", "name", "geometry"]].copy()
+        cities.columns = ["local_admin_unit_id", "local_admin_unit_name", "geometry"]
+
+        cities["local_admin_unit_id"] = "ch-" + cities["local_admin_unit_id"].astype(str)
+        cities["country"] = "ch"
+
+        cities["geometry"] = shapely.wkb.loads(shapely.wkb.dumps(cities["geometry"], output_dimension=2))
+        cities = cities.to_crs(3035)
+
+        return cities