From 4d92f4e7d592592e949ffafcd554f31afe44cf9f Mon Sep 17 00:00:00 2001 From: Michael Lindner Date: Thu, 5 Mar 2026 10:21:28 +0100 Subject: [PATCH 01/13] rename plot_scenario_comparison script --- Snakefile | 23 ++++++++++++++----- ...parison.py => plot_scenario_comparison.py} | 15 ++++++------ 2 files changed, 25 insertions(+), 13 deletions(-) rename scripts/pypsa-de/{plot_ariadne_scenario_comparison.py => plot_scenario_comparison.py} (79%) diff --git a/Snakefile b/Snakefile index 9f648a3d0..e78e04453 100644 --- a/Snakefile +++ b/Snakefile @@ -936,6 +936,20 @@ rule plot_ariadne_variables: "scripts/pypsa-de/plot_ariadne_variables.py" +rule plot_scenario_comparison: + input: + exported_variables=expand( + RESULTS + "ariadne/exported_variables_full.xlsx", + run=config_provider("run", "name"), + ), + output: + price_carbon="results/" + + config["run"]["prefix"] + + "/scenario_comparison/Price-Carbon.png", + script: + "scripts/pypsa-de/plot_scenario_comparison.py" + + rule ariadne_all: input: expand(RESULTS + "graphs/costs.pdf", run=config_provider("run", "name")), @@ -950,12 +964,9 @@ rule ariadne_all: **config["scenario"], allow_missing=True, ), - exported_variables=expand( - RESULTS + "ariadne/exported_variables_full.xlsx", - run=config_provider("run", "name"), - ), - script: - "scripts/pypsa-de/plot_ariadne_scenario_comparison.py" + price_carbon="results/" + + config["run"]["prefix"] + + "/scenario_comparison/Price-Carbon.png", rule build_scenarios: diff --git a/scripts/pypsa-de/plot_ariadne_scenario_comparison.py b/scripts/pypsa-de/plot_scenario_comparison.py similarity index 79% rename from scripts/pypsa-de/plot_ariadne_scenario_comparison.py rename to scripts/pypsa-de/plot_scenario_comparison.py index 68f3f5c59..825f32259 100644 --- a/scripts/pypsa-de/plot_ariadne_scenario_comparison.py +++ b/scripts/pypsa-de/plot_scenario_comparison.py @@ -14,22 +14,21 @@ ) -def scenario_plot(df, var): +def scenario_plot(df, var, output_dir): unit = df._get_label_or_level_values("Unit")[0] if var.startswith("Investment"): unit = "billion EUR2020/yr" df = df.droplevel("Unit") ax = df.T.plot(xlabel="years", ylabel=str(unit), title=str(var)) - prefix = snakemake.config["run"]["prefix"] var = var.replace("|", "-").replace("\\", "-").replace(" ", "-").replace("/", "-") - ax.figure.savefig(f"results/{prefix}/ariadne_comparison/{var}", bbox_inches="tight") + ax.figure.savefig(f"{output_dir}/{var}.png", bbox_inches="tight", dpi=100) plt.close(ax.figure) if __name__ == "__main__": if "snakemake" not in globals(): snakemake = mock_snakemake( - "ariadne_all", + "plot_scenario_comparison", # simpl="", # clusters=22, # opts="", @@ -53,8 +52,10 @@ def scenario_plot(df, var): df = pd.concat(dfs, axis=0) prefix = snakemake.config["run"]["prefix"] - if not os.path.exists(f"results/{prefix}/ariadne_comparison/"): - os.mkdir(f"results/{prefix}/ariadne_comparison/") + + output_dir = f"results/{prefix}/scenario_comparison/" + if not os.path.exists(output_dir): + os.makedirs(output_dir) for var in df._get_label_or_level_values("Variable"): - scenario_plot(df.xs(var, level="Variable"), var) + scenario_plot(df.xs(var, level="Variable"), var, output_dir) From 5fc91a574cd05d4bf6b6cabb65dd9ffd9dec0e46 Mon Sep 17 00:00:00 2001 From: Michael Lindner Date: Thu, 5 Mar 2026 10:22:00 +0100 Subject: [PATCH 02/13] change prefix --- config/config.de.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/config.de.yaml b/config/config.de.yaml index 35d4a786a..d6ba41865 100644 --- a/config/config.de.yaml +++ b/config/config.de.yaml @@ -4,7 +4,7 @@ # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#run run: - prefix: 20260227_compat_pypsa1.1 + prefix: 20260305_housekeeping name: # - ExPol - KN2045_Mix From 0ce0e7fd573e141f71cba10e602666d03bd18e82 Mon Sep 17 00:00:00 2001 From: Michael Lindner Date: Thu, 5 Mar 2026 16:01:50 +0100 Subject: [PATCH 03/13] add rule to compare scenarios --- Snakefile | 52 +++++++++++++++++++++++++++++++++------------------- 1 file changed, 33 insertions(+), 19 deletions(-) diff --git a/Snakefile b/Snakefile index e78e04453..0e0169c2c 100644 --- a/Snakefile +++ b/Snakefile @@ -950,25 +950,6 @@ rule plot_scenario_comparison: "scripts/pypsa-de/plot_scenario_comparison.py" -rule ariadne_all: - input: - expand(RESULTS + "graphs/costs.pdf", run=config_provider("run", "name")), - # expand( - # RESULTS + "ariadne/capacity_detailed.png", - # run=config_provider("run", "name"), - # ), - expand( - RESULTS - + "maps/base_s_{clusters}_{opts}_{sector_opts}-h2_network_incl_kernnetz_{planning_horizons}.pdf", - run=config_provider("run", "name"), - **config["scenario"], - allow_missing=True, - ), - price_carbon="results/" - + config["run"]["prefix"] - + "/scenario_comparison/Price-Carbon.png", - - rule build_scenarios: params: scenarios=config["run"]["name"], @@ -1066,3 +1047,36 @@ rule ariadne_report_only: RESULTS + "ariadne/report/elec_price_duration_curve.pdf", run=config_provider("run", "name"), ), + + +rule compare_scenarios: + input: + price_carbon="results/" + + config["run"]["prefix"] + + "/scenario_comparison/Price-Carbon.png", + # expand( + # RESULTS + "ariadne/capacity_detailed.png", + # run=config_provider("run", "name"), + # ), + + +rule ariadne_all: + input: + expand( + RESULTS + "ariadne/report/elec_price_duration_curve.pdf", + run=config_provider("run", "name"), + ), + # expand( + # RESULTS + "ariadne/capacity_detailed.png", + # run=config_provider("run", "name"), + # ), + expand( + RESULTS + + "maps/base_s_{clusters}_{opts}_{sector_opts}-h2_network_incl_kernnetz_{planning_horizons}.pdf", + run=config_provider("run", "name"), + **config["scenario"], + allow_missing=True, + ), + price_carbon="results/" + + config["run"]["prefix"] + + "/scenario_comparison/Price-Carbon.png", From 2f6e241cdcb3e5ec61393d38d753e89bbdccd0f3 Mon Sep 17 00:00:00 2001 From: Michael Lindner Date: Thu, 5 Mar 2026 16:02:09 +0100 Subject: [PATCH 04/13] update readme --- README.md | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index c4832758b..9907d8d85 100644 --- a/README.md +++ b/README.md @@ -12,37 +12,43 @@ This repository contains the entire scientific project, including data sources a ## Getting ready -You need `conda` or `mamba` to run the analysis. Using conda, you can create an environment from within which you can run the analysis: - +First of all, clone the PyPSA-DE repository using the version control system git in the command line. ``` -conda env create -f envs/{os}.lock.yaml +git clone https://github.com/PyPSA/pypsa-de.git ``` -Where `{os}` should be replaced with your operating system, e.g. for linux the command would be: - +PyPSA-DE relies on a set of other Python packages to function. We manage these using [`pixi`](https://pixi.prefix.dev/latest/). Once pixi is installed, you can activate the project environment for your operating system and have access to all the PyPSA-DE dependencies from the command line: ``` -conda env create -f envs/linux-64.lock.yaml +pixi shell ``` +Tip: You can also set up automatic shell activation in several popular editors (e.g. in VSCode or Zed). Refer to the pixi documentation for the most up-to-date options. + +If you cannot access pixi on your machine, you can also install using `conda`. For more details see the corresponding section in the [documentation](https://pypsa-eur.readthedocs.io/en/latest/installation.html#legacy-method-conda) of PyPSA-Eur + ## Run the analysis Before running any analysis with scenarios, the rule `build_scenarios` must be executed. This will create the file `config/scenarios.automated.yaml` which includes input data and CO2 targets from the IIASA Ariadne database as well as the specifications from the manual scenario file. [This file is specified in the config.de.yaml via they key `run:scenarios:manual_file` and located at `config/scenarios.manual.yaml` by default]. - snakemake build_scenarios -f + snakemake -c1 build_scenarios Note that the hierarchy of scenario files is the following: `scenarios.automated.yaml` > (any `explicitly specified --configfiles`) > `config.de.yaml `> `config.default.yaml `Changes in the file `scenarios.manual.yaml `are only taken into account if the rule `build_scenarios` is executed. -To run the analysis use +To run the analysis use, either + + snakemake -call # The flag -cN specifies the number N of CPU cores available to snakemake + +to generate the solved networks, or - snakemake ariadne_all + snakemake -c1 ariadne_all -This will run all analysis steps to reproduce results. If computational resources on your local machine are limited you may decrease the number of cores by adding, e.g. `-c4` to the call to get only 4 cores. For more option please refer to the [snakemake](https://snakemake.readthedocs.io/en/stable/) documentation. +which will generators additional outputs related to the Ariadne project. If computational resources on your local machine are limited you may decrease the number of cores by adding, e.g. `-c4` to the call to get only 4 cores. For more option please refer to the [snakemake](https://snakemake.readthedocs.io/en/stable/) documentation. ## Repo structure -* `config`: configuration files -* `data/pypsa-de`: Germany specific data from the Ariadne project -* `scripts`: contains the Python scripts for the workflow, the Germany specific code needed to run this repo is contained in `scripts/pypsa-de` +* `config`: configuration files, most importantly `config.de.yaml` and `scenarios.manual.yaml` +* `data/pypsa-de`: Germany specific data +* `scripts`: contains the Python scripts for the workflow, the pypsa-de specific code needed to run this repo is contained in `scripts/pypsa-de` * `cutouts`: very large weather data cutouts supplied by atlite library (does not exist initially) * `data`: place for raw data (does not exist initially) * `resources`: place for intermediate/processing data for the workflow (does not exist initially) From 42318a6f3ba12510ac05717e8a5899a2d467dda2 Mon Sep 17 00:00:00 2001 From: Michael Lindner Date: Thu, 5 Mar 2026 16:02:27 +0100 Subject: [PATCH 05/13] minor comments --- config/config.de.yaml | 1 - scripts/pypsa-de/modify_existing_heating.py | 9 ++++++--- scripts/pypsa-de/modify_industry_production.py | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/config/config.de.yaml b/config/config.de.yaml index d6ba41865..b139cf7d9 100644 --- a/config/config.de.yaml +++ b/config/config.de.yaml @@ -33,7 +33,6 @@ run: pypsa-de: leitmodelle: # Model data downloaded from public IIASA database 'ariadne2' general: REMIND-EU v1.1 - buildings: REMod v1.0 transport: Aladin v1 industry: FORECAST v1.0 reference_scenario: KN2045_Mix diff --git a/scripts/pypsa-de/modify_existing_heating.py b/scripts/pypsa-de/modify_existing_heating.py index 096fbab8d..4f034f8e5 100644 --- a/scripts/pypsa-de/modify_existing_heating.py +++ b/scripts/pypsa-de/modify_existing_heating.py @@ -35,11 +35,14 @@ "Adjusting heating stock towards hard coded values from a\nprevious REMod run. This is only a hotfix." ) # Because REMod is not consistent and a better solution takes too long. - new_values["gas boiler"] = 11.44 # million - new_values["oil boiler"] = 5.99 + new_values["gas boiler"] = ( + 11.44 # million # Schornsteinfeger: 7.78 + 0.725 + 0.866 + 0.442 + 4.130 + ) + # Schornsteinfeger Kohle: 0.08 + new_values["oil boiler"] = 5.99 # Schornsteinfeger: 3.86 + 0.965 new_values["air heat pump"] = 0.38 new_values["ground heat pump"] = 0.38 - new_values["biomass boiler"] = 2.8 + new_values["biomass boiler"] = 2.8 # Schornsteinfeger: 1.16 - 0.08 (Zentral) + total_stock = new_values.sum() existing_factor = existing_heating.loc["Germany"].sum() / total_stock diff --git a/scripts/pypsa-de/modify_industry_production.py b/scripts/pypsa-de/modify_industry_production.py index 24a2822e1..e243ad9c7 100644 --- a/scripts/pypsa-de/modify_industry_production.py +++ b/scripts/pypsa-de/modify_industry_production.py @@ -66,7 +66,7 @@ index=[year], ).T.multiply(1000) else: - # leitmodell for industry demand + # leitmodell for industry production leitmodell = "FORECAST v1.0" ariadne = ( pd.read_csv( From a37161f39b0e5ab715261d1e230ed5937ca5f4d5 Mon Sep 17 00:00:00 2001 From: Michael Lindner Date: Thu, 5 Mar 2026 17:01:20 +0100 Subject: [PATCH 06/13] modularize Snakefile --- Snakefile | 699 +----------------------- rules/pypsa-de/modifications.smk | 375 +++++++++++++ rules/pypsa-de/reporting.smk | 234 ++++++++ rules/pypsa-de/retrieve.smk | 94 ++++ scripts/pypsa-de/plot_ariadne_report.py | 12 +- 5 files changed, 714 insertions(+), 700 deletions(-) create mode 100644 rules/pypsa-de/modifications.smk create mode 100644 rules/pypsa-de/reporting.smk create mode 100644 rules/pypsa-de/retrieve.smk diff --git a/Snakefile b/Snakefile index 0e0169c2c..b4227fd56 100644 --- a/Snakefile +++ b/Snakefile @@ -91,6 +91,11 @@ if config["foresight"] == "perfect": include: "rules/solve_perfect.smk" +include: "rules/pypsa-de/retrieve.smk" +include: "rules/pypsa-de/modifications.smk" +include: "rules/pypsa-de/reporting.smk" + + rule all: input: expand(RESULTS + "graphs/costs.pdf", run=config["run"]["name"]), @@ -386,697 +391,3 @@ rule clean: shutil.rmtree("resources") shutil.rmtree("results") print("Data downloaded to data/ has not been cleaned.") - - -if (ARIADNE_DATABASE := dataset_version("ariadne_database"))["source"] in ["primary"]: - - rule retrieve_ariadne_database: - params: - source="primary", - output: - data="data/ariadne_database.csv", - log: - "logs/retrieve_ariadne_database_primary.log", - resources: - mem_mb=1000, - script: - "scripts/pypsa-de/retrieve_ariadne_database.py" - - -if (ARIADNE_DATABASE := dataset_version("ariadne_database"))["source"] in ["archive"]: - - rule retrieve_ariadne_database: - params: - source="archive", - input: - raw_xlsx=storage(ARIADNE_DATABASE["url"]), - output: - data="data/ariadne_database.csv", - log: - "logs/retrieve_ariadne_database_archive.log", - resources: - mem_mb=1000, - script: - "scripts/pypsa-de/retrieve_ariadne_database.py" - - -if (ARIADNE_TEMPLATE := dataset_version("ariadne_template"))["source"] in [ - "primary", - "archive", -]: - - rule retrieve_ariadne_template: - input: - storage(ARIADNE_TEMPLATE["url"]), - output: - "data/template_ariadne_database.xlsx", - run: - move(input[0], output[0]) - - -if (OPEN_MASTR := dataset_version("open_mastr"))["source"] in ["primary", "archive"]: - - rule retrieve_open_mastr: - input: - storage(OPEN_MASTR["url"]), - params: - "data/mastr", - output: - "data/mastr/bnetza_open_mastr_2023-08-08_B_biomass.csv", - "data/mastr/bnetza_open_mastr_2023-08-08_B_combustion.csv", - run: - unpack_archive(input[0], params[0]) - - -if (EGON := dataset_version("egon"))["source"] in ["build"]: - - rule retrieve_egon_data: - params: - url=EGON["url"], - folder=EGON["folder"], - output: - spatial=f"{EGON['folder']}/demandregio_spatial_2018.json", - mapping=f"{EGON['folder']}/mapping_technologies.json", - shell: - """ - mkdir -p {params.folder} - curl -o {output.spatial} "{params.url}?id_spatial=5&year=2018" - curl -o {output.mapping} "{params.url}_description?id_spatial=5" - """ - - -if (EGON := dataset_version("egon"))["source"] in ["archive"]: - - rule retrieve_egon_data: - input: - spatial=storage(f"{EGON['url']}/demandregio_spatial_2018.json"), - mapping=storage(f"{EGON['url']}/mapping_technologies.json"), - output: - spatial=f"{EGON['folder']}/demandregio_spatial_2018.json", - mapping=f"{EGON['folder']}/mapping_technologies.json", - run: - copy2(input["spatial"], output["spatial"]) - copy2(input["mapping"], output["mapping"]) - - -rule build_exogenous_mobility_data: - params: - reference_scenario=config_provider("pypsa-de", "reference_scenario"), - planning_horizons=config_provider("scenario", "planning_horizons"), - leitmodelle=config_provider("pypsa-de", "leitmodelle"), - ageb_for_mobility=config_provider("pypsa-de", "ageb_for_mobility"), - uba_for_mobility=config_provider("pypsa-de", "uba_for_mobility"), - shipping_oil_share=config_provider("sector", "shipping_oil_share"), - aviation_demand_factor=config_provider("sector", "aviation_demand_factor"), - energy_totals_year=config_provider("energy", "energy_totals_year"), - input: - ariadne="data/ariadne_database.csv", - energy_totals=resources("energy_totals.csv"), - output: - mobility_data=resources( - "modified_mobility_data_{clusters}_{planning_horizons}.csv" - ), - resources: - mem_mb=1000, - log: - logs("build_exogenous_mobility_data_{clusters}_{planning_horizons}.log"), - script: - "scripts/pypsa-de/build_exogenous_mobility_data.py" - - -rule build_egon_data: - input: - demandregio_spatial=f"{EGON['folder']}/demandregio_spatial_2018.json", - mapping_38_to_4=storage( - "https://ffeopendatastorage.blob.core.windows.net/opendata/mapping_from_4_to_38.json", - keep_local=True, - ), - mapping_technologies=f"{EGON['folder']}/mapping_technologies.json", - nuts3=resources("nuts3_shapes.geojson"), - output: - heating_technologies_nuts3=resources("heating_technologies_nuts3.geojson"), - log: - logs("build_egon_data.log"), - script: - "scripts/pypsa-de/build_egon_data.py" - - -rule prepare_district_heating_subnodes: - params: - district_heating=config_provider("sector", "district_heating"), - baseyear=config_provider("scenario", "planning_horizons", 0), - input: - heating_technologies_nuts3=resources("heating_technologies_nuts3.geojson"), - regions_onshore=resources("regions_onshore_base_s_{clusters}.geojson"), - fernwaermeatlas="data/fernwaermeatlas/fernwaermeatlas.xlsx", - cities="data/fernwaermeatlas/cities_geolocations.geojson", - lau_regions=rules.retrieve_lau_regions.output["zip"], - census=storage( - "https://www.destatis.de/static/DE/zensus/gitterdaten/Zensus2022_Heizungsart.zip", - keep_local=True, - ), - osm_land_cover=storage( - "https://heidata.uni-heidelberg.de/api/access/datafile/23053?format=original&gbrecs=true", - keep_local=True, - ), - natura=ancient("data/bundle/natura/natura.tiff"), - groundwater_depth=storage( - "http://thredds-gfnl.usc.es/thredds/fileServer/GLOBALWTDFTP/annualmeans/EURASIA_WTD_annualmean.nc", - keep_local=True, - ), - output: - district_heating_subnodes=resources( - "district_heating_subnodes_base_s_{clusters}.geojson" - ), - regions_onshore_extended=resources( - "regions_onshore_base-extended_s_{clusters}.geojson" - ), - regions_onshore_restricted=resources( - "regions_onshore_base-restricted_s_{clusters}.geojson" - ), - resources: - mem_mb=20000, - script: - "scripts/pypsa-de/prepare_district_heating_subnodes.py" - - -def baseyear_value(wildcards): - return config_provider("scenario", "planning_horizons", 0)(wildcards) - - -rule add_district_heating_subnodes: - params: - district_heating=config_provider("sector", "district_heating"), - baseyear=config_provider("scenario", "planning_horizons", 0), - sector=config_provider("sector"), - heat_pump_sources=config_provider( - "sector", "heat_pump_sources", "urban central" - ), - heat_utilisation_potentials=config_provider( - "sector", "district_heating", "heat_utilisation_potentials" - ), - direct_utilisation_heat_sources=config_provider( - "sector", "district_heating", "direct_utilisation_heat_sources" - ), - adjustments=config_provider("adjustments", "sector"), - input: - unpack(input_heat_source_power), - network=resources( - "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc" - ), - subnodes=resources("district_heating_subnodes_base_s_{clusters}.geojson"), - nuts3=resources("nuts3_shapes.geojson"), - regions_onshore=resources("regions_onshore_base_s_{clusters}.geojson"), - fernwaermeatlas="data/fernwaermeatlas/fernwaermeatlas.xlsx", - cities="data/fernwaermeatlas/cities_geolocations.geojson", - cop_profiles=resources("cop_profiles_base_s_{clusters}_{planning_horizons}.nc"), - direct_heat_source_utilisation_profiles=resources( - "direct_heat_source_utilisation_profiles_base_s_{clusters}_{planning_horizons}.nc" - ), - existing_heating_distribution=lambda w: resources( - f"existing_heating_distribution_base_s_{{clusters}}_{baseyear_value(w)}.csv" - ), - lau_regions=rules.retrieve_lau_regions.output["zip"], - output: - network=resources( - "networks/base-extended_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc" - ), - district_heating_subnodes=resources( - "district_heating_subnodes_base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.geojson" - ), - existing_heating_distribution_extended=( - resources( - "existing_heating_distribution_base-extended_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.csv" - ) - if baseyear_value != "{planning_horizons}" - else [] - ), - resources: - mem_mb=10000, - script: - "scripts/pypsa-de/add_district_heating_subnodes.py" - - -ruleorder: modify_district_heat_share > build_district_heat_share - - -rule modify_district_heat_share: - params: - district_heating=config_provider("sector", "district_heating"), - input: - heating_technologies_nuts3=resources("heating_technologies_nuts3.geojson"), - regions_onshore=resources("regions_onshore_base_s_{clusters}.geojson"), - district_heat_share=resources( - "district_heat_share_base_s_{clusters}_{planning_horizons}.csv" - ), - output: - district_heat_share=resources( - "district_heat_share_base_s_{clusters}_{planning_horizons}-modified.csv" - ), - resources: - mem_mb=1000, - log: - logs("modify_district_heat_share_{clusters}_{planning_horizons}.log"), - script: - "scripts/pypsa-de/modify_district_heat_share.py" - - -rule modify_prenetwork: - params: - efuel_export_ban=config_provider("solving", "constraints", "efuel_export_ban"), - enable_kernnetz=config_provider("wasserstoff_kernnetz", "enable"), - technology_occurrence=config_provider("first_technology_occurrence"), - fossil_boiler_ban=config_provider("new_decentral_fossil_boiler_ban"), - coal_ban=config_provider("coal_generation_ban"), - nuclear_ban=config_provider("nuclear_generation_ban"), - planning_horizons=config_provider("scenario", "planning_horizons"), - H2_transmission_efficiency=config_provider( - "sector", "transmission_efficiency", "H2 pipeline" - ), - H2_retrofit=config_provider("sector", "H2_retrofit"), - H2_retrofit_capacity_per_CH4=config_provider( - "sector", "H2_retrofit_capacity_per_CH4" - ), - transmission_costs=config_provider("costs", "transmission"), - must_run=config_provider("must_run"), - clustering=config_provider("clustering", "temporal", "resolution_sector"), - H2_plants=config_provider("electricity", "H2_plants"), - onshore_nep_force=config_provider("onshore_nep_force"), - offshore_nep_force=config_provider("offshore_nep_force"), - shipping_methanol_efficiency=config_provider( - "sector", "shipping_methanol_efficiency" - ), - shipping_oil_efficiency=config_provider("sector", "shipping_oil_efficiency"), - shipping_methanol_share=config_provider("sector", "shipping_methanol_share"), - scale_capacity=config_provider("scale_capacity"), - bev_charge_rate=config_provider("sector", "bev_charge_rate"), - bev_energy=config_provider("sector", "bev_energy"), - bev_dsm_availability=config_provider("sector", "bev_dsm_availability"), - uba_for_industry=config_provider("pypsa-de", "uba_for_industry", "enable"), - scale_industry_non_energy=config_provider( - "pypsa-de", "uba_for_industry", "scale_industry_non_energy" - ), - limit_cross_border_flows_ac=config_provider( - "pypsa-de", "limit_cross_border_flows_ac" - ), - input: - network=resources( - "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}_brownfield.nc" - ), - wkn=lambda w: ( - resources("wasserstoff_kernnetz_base_s_{clusters}.csv") - if config_provider("wasserstoff_kernnetz", "enable")(w) - else [] - ), - costs=resources("costs_{planning_horizons}_processed.csv"), - modified_mobility_data=resources( - "modified_mobility_data_{clusters}_{planning_horizons}.csv" - ), - biomass_potentials=resources( - "biomass_potentials_s_{clusters}_{planning_horizons}.csv" - ), - industrial_demand=resources( - "industrial_energy_demand_base_s_{clusters}_{planning_horizons}.csv" - ), - industrial_production_per_country_tomorrow=resources( - "industrial_production_per_country_tomorrow_{planning_horizons}-modified.csv" - ), - industry_sector_ratios=resources( - "industry_sector_ratios_{planning_horizons}.csv" - ), - pop_weighted_energy_totals=resources( - "pop_weighted_energy_totals_s_{clusters}.csv" - ), - shipping_demand=resources("shipping_demand_s_{clusters}.csv"), - regions_onshore=resources("regions_onshore_base_s_{clusters}.geojson"), - regions_offshore=resources("regions_offshore_base_s_{clusters}.geojson"), - offshore_connection_points="data/pypsa-de/offshore_connection_points.csv", - new_industrial_energy_demand="data/pypsa-de/UBA_Projektionsbericht2025_Abbildung31_MWMS.csv", - output: - network=resources( - "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}_final.nc" - ), - resources: - mem_mb=4000, - log: - RESULTS - + "logs/modify_prenetwork_base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.log", - script: - "scripts/pypsa-de/modify_prenetwork.py" - - -ruleorder: modify_industry_production > build_industrial_production_per_country_tomorrow - - -rule modify_existing_heating: - input: - ariadne="data/ariadne_database.csv", - existing_heating="data/existing_infrastructure/existing_heating_raw.csv", - output: - existing_heating=resources("existing_heating.csv"), - resources: - mem_mb=1000, - log: - logs("modify_existing_heating.log"), - script: - "scripts/pypsa-de/modify_existing_heating.py" - - -rule build_existing_chp_de: - params: - district_heating_subnodes=config_provider( - "sector", "district_heating", "subnodes" - ), - input: - mastr_biomass="data/mastr/bnetza_open_mastr_2023-08-08_B_biomass.csv", - mastr_combustion="data/mastr/bnetza_open_mastr_2023-08-08_B_combustion.csv", - plz_mapping=storage( - "https://raw.githubusercontent.com/WZBSocialScienceCenter/plz_geocoord/master/plz_geocoord.csv", - keep_local=True, - ), - regions=resources("regions_onshore_base_s_{clusters}.geojson"), - district_heating_subnodes=lambda w: ( - resources("district_heating_subnodes_base_s_{clusters}.geojson") - if config_provider("sector", "district_heating", "subnodes", "enable")(w) - else [] - ), - output: - german_chp=resources("german_chp_base_s_{clusters}.csv"), - resources: - mem_mb=4000, - log: - logs("build_existing_chp_de_{clusters}.log"), - script: - "scripts/pypsa-de/build_existing_chp_de.py" - - -rule modify_industry_production: - params: - reference_scenario=config_provider("pypsa-de", "reference_scenario"), - input: - ariadne="data/ariadne_database.csv", - industrial_production_per_country_tomorrow=resources( - "industrial_production_per_country_tomorrow_{planning_horizons}.csv" - ), - output: - industrial_production_per_country_tomorrow=resources( - "industrial_production_per_country_tomorrow_{planning_horizons}-modified.csv" - ), - resources: - mem_mb=1000, - log: - logs("modify_industry_production_{planning_horizons}.log"), - script: - "scripts/pypsa-de/modify_industry_production.py" - - -rule build_wasserstoff_kernnetz: - params: - kernnetz=config_provider("wasserstoff_kernnetz"), - input: - wasserstoff_kernnetz_1=storage( - "https://fnb-gas.de/wp-content/uploads/2024/07/2024_07_22_Anlage2_Leitungsmeldungen_weiterer_potenzieller_Wasserstoffnetzbetreiber.xlsx", - keep_local=True, - ), - wasserstoff_kernnetz_2=storage( - "https://fnb-gas.de/wp-content/uploads/2024/07/2024_07_22_Anlage3_FNB_Massnahmenliste_Neubau.xlsx", - keep_local=True, - ), - wasserstoff_kernnetz_3=storage( - "https://fnb-gas.de/wp-content/uploads/2024/07/2024_07_22_Anlage4_FNB_Massnahmenliste_Umstellung.xlsx", - keep_local=True, - ), - gadm=storage( - "https://geodata.ucdavis.edu/gadm/gadm4.1/json/gadm41_DEU_1.json.zip", - keep_local=True, - ), - locations="data/pypsa-de/wasserstoff_kernnetz/locations_wasserstoff_kernnetz.csv", - regions_onshore=resources("regions_onshore_base_s.geojson"), - regions_offshore=resources("regions_offshore_base_s.geojson"), - output: - cleaned_wasserstoff_kernnetz=resources("wasserstoff_kernnetz.csv"), - log: - logs("build_wasserstoff_kernnetz.log"), - script: - "scripts/pypsa-de/build_wasserstoff_kernnetz.py" - - -rule cluster_wasserstoff_kernnetz: - params: - kernnetz=config_provider("wasserstoff_kernnetz"), - input: - cleaned_h2_network=resources("wasserstoff_kernnetz.csv"), - regions_onshore=resources("regions_onshore_base_s_{clusters}.geojson"), - regions_offshore=resources("regions_offshore_base_s_{clusters}.geojson"), - output: - clustered_h2_network=resources("wasserstoff_kernnetz_base_s_{clusters}.csv"), - log: - logs("cluster_wasserstoff_kernnetz_{clusters}.log"), - script: - "scripts/pypsa-de/cluster_wasserstoff_kernnetz.py" - - -rule export_ariadne_variables: - params: - planning_horizons=config_provider("scenario", "planning_horizons"), - hours=config_provider("clustering", "temporal", "resolution_sector"), - config_industry=config_provider("industry"), - energy_totals_year=config_provider("energy", "energy_totals_year"), - co2_sequestration_cost=config_provider("sector", "co2_sequestration_cost"), - post_discretization=config_provider("solving", "options", "post_discretization"), - NEP_year=lambda w: config_provider("costs", "custom_cost_fn")(w)[-8:-4], - NEP_transmission=config_provider("costs", "transmission"), - input: - template="data/template_ariadne_database.xlsx", - industry_demands=expand( - resources( - "industrial_energy_demand_base_s_{clusters}_{planning_horizons}.csv" - ), - **config["scenario"], - allow_missing=True, - ), - networks=expand( - RESULTS - + "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc", - **config["scenario"], - allow_missing=True, - ), - costs=expand( - resources("costs_{planning_horizons}_processed.csv"), - **config["scenario"], - allow_missing=True, - ), - industrial_production_per_country_tomorrow=expand( - resources( - "industrial_production_per_country_tomorrow_{planning_horizons}-modified.csv" - ), - **config["scenario"], - allow_missing=True, - ), - industry_sector_ratios=expand( - resources("industry_sector_ratios_{planning_horizons}.csv"), - **config["scenario"], - allow_missing=True, - ), - industrial_production=resources("industrial_production_per_country.csv"), - energy_totals=resources("energy_totals.csv"), - output: - exported_variables=RESULTS + "ariadne/exported_variables.xlsx", - exported_variables_full=RESULTS + "ariadne/exported_variables_full.xlsx", - resources: - mem_mb=16000, - log: - RESULTS + "logs/export_ariadne_variables.log", - script: - "scripts/pypsa-de/export_ariadne_variables.py" - - -rule plot_ariadne_variables: - params: - reference_scenario=config_provider("pypsa-de", "reference_scenario"), - input: - exported_variables_full=RESULTS + "ariadne/exported_variables_full.xlsx", - ariadne_database="data/ariadne_database.csv", - output: - primary_energy=RESULTS + "ariadne/primary_energy.png", - primary_energy_detailed=RESULTS + "ariadne/primary_energy_detailed.png", - secondary_energy=RESULTS + "ariadne/secondary_energy.png", - secondary_energy_detailed=RESULTS + "ariadne/secondary_energy_detailed.png", - final_energy=RESULTS + "ariadne/final_energy.png", - final_energy_detailed=RESULTS + "ariadne/final_energy_detailed.png", - capacity=RESULTS + "ariadne/capacity.png", - capacity_detailed=RESULTS + "ariadne/capacity_detailed.png", - energy_demand_emissions=RESULTS + "ariadne/energy_demand_emissions.png", - energy_supply_emissions=RESULTS + "ariadne/energy_supply_emissions.png", - co2_emissions=RESULTS + "ariadne/co2_emissions.png", - primary_energy_price=RESULTS + "ariadne/primary_energy_price.png", - secondary_energy_price=RESULTS + "ariadne/secondary_energy_price.png", - #final_energy_residential_price = RESULTS + "ariadne/final_energy_residential_price.png", - final_energy_industry_price=RESULTS + "ariadne/final_energy_industry_price.png", - final_energy_transportation_price=RESULTS - + "ariadne/final_energy_transportation_price.png", - final_energy_residential_commercial_price=RESULTS - + "ariadne/final_energy_residential_commercial_price.png", - all_prices=RESULTS + "ariadne/all_prices.png", - policy_carbon=RESULTS + "ariadne/policy_carbon.png", - investment_energy_supply=RESULTS + "ariadne/investment_energy_supply.png", - elec_val_2020=RESULTS + "ariadne/elec_val_2020.png", - trade=RESULTS + "ariadne/trade.png", - NEP_plot=RESULTS + "ariadne/NEP_plot.png", - NEP_Trassen_plot=RESULTS + "ariadne/NEP_Trassen_plot.png", - transmission_investment_csv=RESULTS + "ariadne/transmission_investment.csv", - trassenlaenge_csv=RESULTS + "ariadne/trassenlaenge.csv", - Kernnetz_Investment_plot=RESULTS + "ariadne/Kernnetz_Investment_plot.png", - elec_trade=RESULTS + "ariadne/elec-trade-DE.pdf", - h2_trade=RESULTS + "ariadne/h2-trade-DE.pdf", - trade_balance=RESULTS + "ariadne/trade-balance-DE.pdf", - log: - RESULTS + "logs/plot_ariadne_variables.log", - script: - "scripts/pypsa-de/plot_ariadne_variables.py" - - -rule plot_scenario_comparison: - input: - exported_variables=expand( - RESULTS + "ariadne/exported_variables_full.xlsx", - run=config_provider("run", "name"), - ), - output: - price_carbon="results/" - + config["run"]["prefix"] - + "/scenario_comparison/Price-Carbon.png", - script: - "scripts/pypsa-de/plot_scenario_comparison.py" - - -rule build_scenarios: - params: - scenarios=config["run"]["name"], - leitmodelle=config["pypsa-de"]["leitmodelle"], - input: - ariadne_database="data/ariadne_database.csv", - scenario_yaml=config["run"]["scenarios"]["manual_file"], - output: - scenario_yaml=config["run"]["scenarios"]["file"], - log: - "logs/build_scenarios.log", - script: - "scripts/pypsa-de/build_scenarios.py" - - -rule plot_hydrogen_network_incl_kernnetz: - params: - plotting=config_provider("plotting"), - foresight=config_provider("foresight"), - input: - network=RESULTS - + "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc", - regions=resources("regions_onshore_base_s_{clusters}.geojson"), - output: - map=RESULTS - + "maps/base_s_{clusters}_{opts}_{sector_opts}-h2_network_incl_kernnetz_{planning_horizons}.pdf", - threads: 2 - resources: - mem_mb=10000, - log: - RESULTS - + "logs/plot_hydrogen_network_incl_kernnetz/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.log", - benchmark: - ( - RESULTS - + "benchmarks/plot_hydrogen_network_incl_kernnetz/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}" - ) - script: - "scripts/pypsa-de/plot_hydrogen_network_incl_kernnetz.py" - - -rule plot_ariadne_report: - params: - planning_horizons=config_provider("scenario", "planning_horizons"), - plotting=config_provider("plotting"), - run=config_provider("run", "name"), - foresight=config_provider("foresight"), - post_discretization=config_provider("solving", "options", "post_discretization"), - NEP_year=lambda w: config_provider("costs", "custom_cost_fn")(w)[-8:-4], - hours=config_provider("clustering", "temporal", "resolution_sector"), - NEP_transmission=config_provider("costs", "transmission"), - input: - networks=expand( - RESULTS - + "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc", - **config["scenario"], - allow_missing=True, - ), - regions_onshore_clustered=expand( - resources("regions_onshore_base_s_{clusters}.geojson"), - clusters=config["scenario"]["clusters"], - allow_missing=True, - ), - rc="matplotlibrc", - costs=expand( - resources("costs_{planning_horizons}_processed.csv"), - **config["scenario"], - allow_missing=True, - ), - exported_variables_full=RESULTS + "ariadne/exported_variables_full.xlsx", - output: - elec_price_duration_curve=RESULTS - + "ariadne/report/elec_price_duration_curve.pdf", - elec_price_duration_hist=RESULTS + "ariadne/report/elec_price_duration_hist.pdf", - backup_capacity=RESULTS + "ariadne/report/backup_capacity.pdf", - backup_generation=RESULTS + "ariadne/report/backup_generation.pdf", - results=directory(RESULTS + "ariadne/report"), - elec_transmission=directory(RESULTS + "ariadne/report/elec_transmission"), - h2_transmission=directory(RESULTS + "ariadne/report/h2_transmission"), - co2_transmission=directory(RESULTS + "ariadne/report/co2_transmission"), - elec_balances=directory(RESULTS + "ariadne/report/elec_balance_timeseries"), - heat_balances=directory(RESULTS + "ariadne/report/heat_balance_timeseries"), - nodal_balances=directory(RESULTS + "ariadne/report/balance_timeseries_2045"), - resources: - mem_mb=32000, - log: - RESULTS + "logs/plot_ariadne_report.log", - script: - "scripts/pypsa-de/plot_ariadne_report.py" - - -rule ariadne_report_only: - input: - expand( - RESULTS + "ariadne/report/elec_price_duration_curve.pdf", - run=config_provider("run", "name"), - ), - - -rule compare_scenarios: - input: - price_carbon="results/" - + config["run"]["prefix"] - + "/scenario_comparison/Price-Carbon.png", - # expand( - # RESULTS + "ariadne/capacity_detailed.png", - # run=config_provider("run", "name"), - # ), - - -rule ariadne_all: - input: - expand( - RESULTS + "ariadne/report/elec_price_duration_curve.pdf", - run=config_provider("run", "name"), - ), - # expand( - # RESULTS + "ariadne/capacity_detailed.png", - # run=config_provider("run", "name"), - # ), - expand( - RESULTS - + "maps/base_s_{clusters}_{opts}_{sector_opts}-h2_network_incl_kernnetz_{planning_horizons}.pdf", - run=config_provider("run", "name"), - **config["scenario"], - allow_missing=True, - ), - price_carbon="results/" - + config["run"]["prefix"] - + "/scenario_comparison/Price-Carbon.png", diff --git a/rules/pypsa-de/modifications.smk b/rules/pypsa-de/modifications.smk new file mode 100644 index 000000000..a9f8fdb5d --- /dev/null +++ b/rules/pypsa-de/modifications.smk @@ -0,0 +1,375 @@ +# SPDX-FileCopyrightText: Contributors to PyPSA-DE +# +# SPDX-License-Identifier: CC BY 4.0 + + +rule build_scenarios: + params: + scenarios=config["run"]["name"], + leitmodelle=config["pypsa-de"]["leitmodelle"], + input: + ariadne_database="data/ariadne_database.csv", + scenario_yaml=config["run"]["scenarios"]["manual_file"], + output: + scenario_yaml=config["run"]["scenarios"]["file"], + log: + "logs/build_scenarios.log", + script: + scripts("pypsa-de/build_scenarios.py") + + +rule build_exogenous_mobility_data: + params: + reference_scenario=config_provider("pypsa-de", "reference_scenario"), + planning_horizons=config_provider("scenario", "planning_horizons"), + leitmodelle=config_provider("pypsa-de", "leitmodelle"), + ageb_for_mobility=config_provider("pypsa-de", "ageb_for_mobility"), + uba_for_mobility=config_provider("pypsa-de", "uba_for_mobility"), + shipping_oil_share=config_provider("sector", "shipping_oil_share"), + aviation_demand_factor=config_provider("sector", "aviation_demand_factor"), + energy_totals_year=config_provider("energy", "energy_totals_year"), + input: + ariadne="data/ariadne_database.csv", + energy_totals=resources("energy_totals.csv"), + output: + mobility_data=resources( + "modified_mobility_data_{clusters}_{planning_horizons}.csv" + ), + resources: + mem_mb=1000, + log: + logs("build_exogenous_mobility_data_{clusters}_{planning_horizons}.log"), + script: + scripts("pypsa-de/build_exogenous_mobility_data.py") + + +rule build_egon_data: + input: + demandregio_spatial=f"{EGON['folder']}/demandregio_spatial_2018.json", + mapping_38_to_4=storage( + "https://ffeopendatastorage.blob.core.windows.net/opendata/mapping_from_4_to_38.json", + keep_local=True, + ), + mapping_technologies=f"{EGON['folder']}/mapping_technologies.json", + nuts3=resources("nuts3_shapes.geojson"), + output: + heating_technologies_nuts3=resources("heating_technologies_nuts3.geojson"), + log: + logs("build_egon_data.log"), + script: + scripts("pypsa-de/build_egon_data.py") + + +rule prepare_district_heating_subnodes: + params: + district_heating=config_provider("sector", "district_heating"), + baseyear=config_provider("scenario", "planning_horizons", 0), + input: + heating_technologies_nuts3=resources("heating_technologies_nuts3.geojson"), + regions_onshore=resources("regions_onshore_base_s_{clusters}.geojson"), + fernwaermeatlas="data/fernwaermeatlas/fernwaermeatlas.xlsx", + cities="data/fernwaermeatlas/cities_geolocations.geojson", + lau_regions=rules.retrieve_lau_regions.output["zip"], + census=storage( + "https://www.destatis.de/static/DE/zensus/gitterdaten/Zensus2022_Heizungsart.zip", + keep_local=True, + ), + osm_land_cover=storage( + "https://heidata.uni-heidelberg.de/api/access/datafile/23053?format=original&gbrecs=true", + keep_local=True, + ), + natura=ancient("data/bundle/natura/natura.tiff"), + groundwater_depth=storage( + "http://thredds-gfnl.usc.es/thredds/fileServer/GLOBALWTDFTP/annualmeans/EURASIA_WTD_annualmean.nc", + keep_local=True, + ), + output: + district_heating_subnodes=resources( + "district_heating_subnodes_base_s_{clusters}.geojson" + ), + regions_onshore_extended=resources( + "regions_onshore_base-extended_s_{clusters}.geojson" + ), + regions_onshore_restricted=resources( + "regions_onshore_base-restricted_s_{clusters}.geojson" + ), + resources: + mem_mb=20000, + script: + scripts("pypsa-de/prepare_district_heating_subnodes.py") + + +def baseyear_value(wildcards): + return config_provider("scenario", "planning_horizons", 0)(wildcards) + + +rule add_district_heating_subnodes: + params: + district_heating=config_provider("sector", "district_heating"), + baseyear=config_provider("scenario", "planning_horizons", 0), + sector=config_provider("sector"), + heat_pump_sources=config_provider( + "sector", "heat_pump_sources", "urban central" + ), + heat_utilisation_potentials=config_provider( + "sector", "district_heating", "heat_utilisation_potentials" + ), + direct_utilisation_heat_sources=config_provider( + "sector", "district_heating", "direct_utilisation_heat_sources" + ), + adjustments=config_provider("adjustments", "sector"), + input: + unpack(input_heat_source_power), + network=resources( + "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc" + ), + subnodes=resources("district_heating_subnodes_base_s_{clusters}.geojson"), + nuts3=resources("nuts3_shapes.geojson"), + regions_onshore=resources("regions_onshore_base_s_{clusters}.geojson"), + fernwaermeatlas="data/fernwaermeatlas/fernwaermeatlas.xlsx", + cities="data/fernwaermeatlas/cities_geolocations.geojson", + cop_profiles=resources("cop_profiles_base_s_{clusters}_{planning_horizons}.nc"), + direct_heat_source_utilisation_profiles=resources( + "direct_heat_source_utilisation_profiles_base_s_{clusters}_{planning_horizons}.nc" + ), + existing_heating_distribution=lambda w: resources( + f"existing_heating_distribution_base_s_{{clusters}}_{baseyear_value(w)}.csv" + ), + lau_regions=rules.retrieve_lau_regions.output["zip"], + output: + network=resources( + "networks/base-extended_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc" + ), + district_heating_subnodes=resources( + "district_heating_subnodes_base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.geojson" + ), + existing_heating_distribution_extended=( + resources( + "existing_heating_distribution_base-extended_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.csv" + ) + if baseyear_value != "{planning_horizons}" + else [] + ), + resources: + mem_mb=10000, + script: + scripts("pypsa-de/add_district_heating_subnodes.py") + + +ruleorder: modify_district_heat_share > build_district_heat_share + + +rule modify_district_heat_share: + params: + district_heating=config_provider("sector", "district_heating"), + input: + heating_technologies_nuts3=resources("heating_technologies_nuts3.geojson"), + regions_onshore=resources("regions_onshore_base_s_{clusters}.geojson"), + district_heat_share=resources( + "district_heat_share_base_s_{clusters}_{planning_horizons}.csv" + ), + output: + district_heat_share=resources( + "district_heat_share_base_s_{clusters}_{planning_horizons}-modified.csv" + ), + resources: + mem_mb=1000, + log: + logs("modify_district_heat_share_{clusters}_{planning_horizons}.log"), + script: + scripts("pypsa-de/modify_district_heat_share.py") + + +rule modify_prenetwork: + params: + efuel_export_ban=config_provider("solving", "constraints", "efuel_export_ban"), + enable_kernnetz=config_provider("wasserstoff_kernnetz", "enable"), + technology_occurrence=config_provider("first_technology_occurrence"), + fossil_boiler_ban=config_provider("new_decentral_fossil_boiler_ban"), + coal_ban=config_provider("coal_generation_ban"), + nuclear_ban=config_provider("nuclear_generation_ban"), + planning_horizons=config_provider("scenario", "planning_horizons"), + H2_transmission_efficiency=config_provider( + "sector", "transmission_efficiency", "H2 pipeline" + ), + H2_retrofit=config_provider("sector", "H2_retrofit"), + H2_retrofit_capacity_per_CH4=config_provider( + "sector", "H2_retrofit_capacity_per_CH4" + ), + transmission_costs=config_provider("costs", "transmission"), + must_run=config_provider("must_run"), + clustering=config_provider("clustering", "temporal", "resolution_sector"), + H2_plants=config_provider("electricity", "H2_plants"), + onshore_nep_force=config_provider("onshore_nep_force"), + offshore_nep_force=config_provider("offshore_nep_force"), + shipping_methanol_efficiency=config_provider( + "sector", "shipping_methanol_efficiency" + ), + shipping_oil_efficiency=config_provider("sector", "shipping_oil_efficiency"), + shipping_methanol_share=config_provider("sector", "shipping_methanol_share"), + scale_capacity=config_provider("scale_capacity"), + bev_charge_rate=config_provider("sector", "bev_charge_rate"), + bev_energy=config_provider("sector", "bev_energy"), + bev_dsm_availability=config_provider("sector", "bev_dsm_availability"), + uba_for_industry=config_provider("pypsa-de", "uba_for_industry", "enable"), + scale_industry_non_energy=config_provider( + "pypsa-de", "uba_for_industry", "scale_industry_non_energy" + ), + limit_cross_border_flows_ac=config_provider( + "pypsa-de", "limit_cross_border_flows_ac" + ), + input: + network=resources( + "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}_brownfield.nc" + ), + wkn=lambda w: ( + resources("wasserstoff_kernnetz_base_s_{clusters}.csv") + if config_provider("wasserstoff_kernnetz", "enable")(w) + else [] + ), + costs=resources("costs_{planning_horizons}_processed.csv"), + modified_mobility_data=resources( + "modified_mobility_data_{clusters}_{planning_horizons}.csv" + ), + biomass_potentials=resources( + "biomass_potentials_s_{clusters}_{planning_horizons}.csv" + ), + industrial_demand=resources( + "industrial_energy_demand_base_s_{clusters}_{planning_horizons}.csv" + ), + industrial_production_per_country_tomorrow=resources( + "industrial_production_per_country_tomorrow_{planning_horizons}-modified.csv" + ), + industry_sector_ratios=resources( + "industry_sector_ratios_{planning_horizons}.csv" + ), + pop_weighted_energy_totals=resources( + "pop_weighted_energy_totals_s_{clusters}.csv" + ), + shipping_demand=resources("shipping_demand_s_{clusters}.csv"), + regions_onshore=resources("regions_onshore_base_s_{clusters}.geojson"), + regions_offshore=resources("regions_offshore_base_s_{clusters}.geojson"), + offshore_connection_points="data/pypsa-de/offshore_connection_points.csv", + new_industrial_energy_demand="data/pypsa-de/UBA_Projektionsbericht2025_Abbildung31_MWMS.csv", + output: + network=resources( + "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}_final.nc" + ), + resources: + mem_mb=4000, + log: + RESULTS + + "logs/modify_prenetwork_base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.log", + script: + scripts("pypsa-de/modify_prenetwork.py") + + +ruleorder: modify_industry_production > build_industrial_production_per_country_tomorrow + + +rule modify_existing_heating: + input: + ariadne="data/ariadne_database.csv", + existing_heating="data/existing_infrastructure/existing_heating_raw.csv", + output: + existing_heating=resources("existing_heating.csv"), + resources: + mem_mb=1000, + log: + logs("modify_existing_heating.log"), + script: + scripts("pypsa-de/modify_existing_heating.py") + + +rule build_existing_chp_de: + params: + district_heating_subnodes=config_provider( + "sector", "district_heating", "subnodes" + ), + input: + mastr_biomass="data/mastr/bnetza_open_mastr_2023-08-08_B_biomass.csv", + mastr_combustion="data/mastr/bnetza_open_mastr_2023-08-08_B_combustion.csv", + plz_mapping=storage( + "https://raw.githubusercontent.com/WZBSocialScienceCenter/plz_geocoord/master/plz_geocoord.csv", + keep_local=True, + ), + regions=resources("regions_onshore_base_s_{clusters}.geojson"), + district_heating_subnodes=lambda w: ( + resources("district_heating_subnodes_base_s_{clusters}.geojson") + if config_provider("sector", "district_heating", "subnodes", "enable")(w) + else [] + ), + output: + german_chp=resources("german_chp_base_s_{clusters}.csv"), + resources: + mem_mb=4000, + log: + logs("build_existing_chp_de_{clusters}.log"), + script: + scripts("pypsa-de/build_existing_chp_de.py") + + +rule modify_industry_production: + params: + reference_scenario=config_provider("pypsa-de", "reference_scenario"), + input: + ariadne="data/ariadne_database.csv", + industrial_production_per_country_tomorrow=resources( + "industrial_production_per_country_tomorrow_{planning_horizons}.csv" + ), + output: + industrial_production_per_country_tomorrow=resources( + "industrial_production_per_country_tomorrow_{planning_horizons}-modified.csv" + ), + resources: + mem_mb=1000, + log: + logs("modify_industry_production_{planning_horizons}.log"), + script: + scripts("pypsa-de/modify_industry_production.py") + + +rule build_wasserstoff_kernnetz: + params: + kernnetz=config_provider("wasserstoff_kernnetz"), + input: + wasserstoff_kernnetz_1=storage( + "https://fnb-gas.de/wp-content/uploads/2024/07/2024_07_22_Anlage2_Leitungsmeldungen_weiterer_potenzieller_Wasserstoffnetzbetreiber.xlsx", + keep_local=True, + ), + wasserstoff_kernnetz_2=storage( + "https://fnb-gas.de/wp-content/uploads/2024/07/2024_07_22_Anlage3_FNB_Massnahmenliste_Neubau.xlsx", + keep_local=True, + ), + wasserstoff_kernnetz_3=storage( + "https://fnb-gas.de/wp-content/uploads/2024/07/2024_07_22_Anlage4_FNB_Massnahmenliste_Umstellung.xlsx", + keep_local=True, + ), + gadm=storage( + "https://geodata.ucdavis.edu/gadm/gadm4.1/json/gadm41_DEU_1.json.zip", + keep_local=True, + ), + locations="data/pypsa-de/wasserstoff_kernnetz/locations_wasserstoff_kernnetz.csv", + regions_onshore=resources("regions_onshore_base_s.geojson"), + regions_offshore=resources("regions_offshore_base_s.geojson"), + output: + cleaned_wasserstoff_kernnetz=resources("wasserstoff_kernnetz.csv"), + log: + logs("build_wasserstoff_kernnetz.log"), + script: + scripts("pypsa-de/build_wasserstoff_kernnetz.py") + + +rule cluster_wasserstoff_kernnetz: + params: + kernnetz=config_provider("wasserstoff_kernnetz"), + input: + cleaned_h2_network=resources("wasserstoff_kernnetz.csv"), + regions_onshore=resources("regions_onshore_base_s_{clusters}.geojson"), + regions_offshore=resources("regions_offshore_base_s_{clusters}.geojson"), + output: + clustered_h2_network=resources("wasserstoff_kernnetz_base_s_{clusters}.csv"), + log: + logs("cluster_wasserstoff_kernnetz_{clusters}.log"), + script: + scripts("pypsa-de/cluster_wasserstoff_kernnetz.py") diff --git a/rules/pypsa-de/reporting.smk b/rules/pypsa-de/reporting.smk new file mode 100644 index 000000000..629955e4c --- /dev/null +++ b/rules/pypsa-de/reporting.smk @@ -0,0 +1,234 @@ +# SPDX-FileCopyrightText: Contributors to PyPSA-DE +# +# SPDX-License-Identifier: CC BY 4.0 + + +rule export_ariadne_variables: + params: + planning_horizons=config_provider("scenario", "planning_horizons"), + hours=config_provider("clustering", "temporal", "resolution_sector"), + config_industry=config_provider("industry"), + energy_totals_year=config_provider("energy", "energy_totals_year"), + co2_sequestration_cost=config_provider("sector", "co2_sequestration_cost"), + post_discretization=config_provider("solving", "options", "post_discretization"), + NEP_year=lambda w: config_provider("costs", "custom_cost_fn")(w)[-8:-4], + NEP_transmission=config_provider("costs", "transmission"), + input: + template="data/template_ariadne_database.xlsx", + industry_demands=expand( + resources( + "industrial_energy_demand_base_s_{clusters}_{planning_horizons}.csv" + ), + **config["scenario"], + allow_missing=True, + ), + networks=expand( + RESULTS + + "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc", + **config["scenario"], + allow_missing=True, + ), + costs=expand( + resources("costs_{planning_horizons}_processed.csv"), + **config["scenario"], + allow_missing=True, + ), + industrial_production_per_country_tomorrow=expand( + resources( + "industrial_production_per_country_tomorrow_{planning_horizons}-modified.csv" + ), + **config["scenario"], + allow_missing=True, + ), + industry_sector_ratios=expand( + resources("industry_sector_ratios_{planning_horizons}.csv"), + **config["scenario"], + allow_missing=True, + ), + industrial_production=resources("industrial_production_per_country.csv"), + energy_totals=resources("energy_totals.csv"), + output: + exported_variables=RESULTS + "ariadne/exported_variables.xlsx", + exported_variables_full=RESULTS + "ariadne/exported_variables_full.xlsx", + resources: + mem_mb=16000, + log: + RESULTS + "logs/export_ariadne_variables.log", + script: + scripts("pypsa-de/export_ariadne_variables.py") + + +rule plot_ariadne_variables: + params: + reference_scenario=config_provider("pypsa-de", "reference_scenario"), + input: + exported_variables_full=RESULTS + "ariadne/exported_variables_full.xlsx", + ariadne_database="data/ariadne_database.csv", + output: + primary_energy=RESULTS + "ariadne/primary_energy.png", + primary_energy_detailed=RESULTS + "ariadne/primary_energy_detailed.png", + secondary_energy=RESULTS + "ariadne/secondary_energy.png", + secondary_energy_detailed=RESULTS + "ariadne/secondary_energy_detailed.png", + final_energy=RESULTS + "ariadne/final_energy.png", + final_energy_detailed=RESULTS + "ariadne/final_energy_detailed.png", + capacity=RESULTS + "ariadne/capacity.png", + capacity_detailed=RESULTS + "ariadne/capacity_detailed.png", + energy_demand_emissions=RESULTS + "ariadne/energy_demand_emissions.png", + energy_supply_emissions=RESULTS + "ariadne/energy_supply_emissions.png", + co2_emissions=RESULTS + "ariadne/co2_emissions.png", + primary_energy_price=RESULTS + "ariadne/primary_energy_price.png", + secondary_energy_price=RESULTS + "ariadne/secondary_energy_price.png", + #final_energy_residential_price = RESULTS + "ariadne/final_energy_residential_price.png", + final_energy_industry_price=RESULTS + "ariadne/final_energy_industry_price.png", + final_energy_transportation_price=RESULTS + + "ariadne/final_energy_transportation_price.png", + final_energy_residential_commercial_price=RESULTS + + "ariadne/final_energy_residential_commercial_price.png", + all_prices=RESULTS + "ariadne/all_prices.png", + policy_carbon=RESULTS + "ariadne/policy_carbon.png", + investment_energy_supply=RESULTS + "ariadne/investment_energy_supply.png", + elec_val_2020=RESULTS + "ariadne/elec_val_2020.png", + trade=RESULTS + "ariadne/trade.png", + NEP_plot=RESULTS + "ariadne/NEP_plot.png", + NEP_Trassen_plot=RESULTS + "ariadne/NEP_Trassen_plot.png", + transmission_investment_csv=RESULTS + "ariadne/transmission_investment.csv", + trassenlaenge_csv=RESULTS + "ariadne/trassenlaenge.csv", + Kernnetz_Investment_plot=RESULTS + "ariadne/Kernnetz_Investment_plot.png", + elec_trade=RESULTS + "ariadne/elec-trade-DE.pdf", + h2_trade=RESULTS + "ariadne/h2-trade-DE.pdf", + trade_balance=RESULTS + "ariadne/trade-balance-DE.pdf", + log: + RESULTS + "logs/plot_ariadne_variables.log", + script: + scripts("pypsa-de/plot_ariadne_variables.py") + + +rule plot_hydrogen_network_incl_kernnetz: + params: + plotting=config_provider("plotting"), + foresight=config_provider("foresight"), + input: + network=RESULTS + + "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc", + regions=resources("regions_onshore_base_s_{clusters}.geojson"), + output: + map=RESULTS + + "maps/base_s_{clusters}_{opts}_{sector_opts}-h2_network_incl_kernnetz_{planning_horizons}.pdf", + threads: 2 + resources: + mem_mb=10000, + log: + RESULTS + + "logs/plot_hydrogen_network_incl_kernnetz/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.log", + benchmark: + ( + RESULTS + + "benchmarks/plot_hydrogen_network_incl_kernnetz/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}" + ) + script: + scripts("pypsa-de/plot_hydrogen_network_incl_kernnetz.py") + + +rule plot_ariadne_report: + params: + planning_horizons=config_provider("scenario", "planning_horizons"), + plotting=config_provider("plotting"), + run=config_provider("run", "name"), + foresight=config_provider("foresight"), + post_discretization=config_provider("solving", "options", "post_discretization"), + NEP_year=lambda w: config_provider("costs", "custom_cost_fn")(w)[-8:-4], + hours=config_provider("clustering", "temporal", "resolution_sector"), + NEP_transmission=config_provider("costs", "transmission"), + input: + networks=expand( + RESULTS + + "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc", + **config["scenario"], + allow_missing=True, + ), + regions_onshore_clustered=expand( + resources("regions_onshore_base_s_{clusters}.geojson"), + clusters=config["scenario"]["clusters"], + allow_missing=True, + ), + rc="matplotlibrc", + costs=expand( + resources("costs_{planning_horizons}_processed.csv"), + **config["scenario"], + allow_missing=True, + ), + exported_variables_full=RESULTS + "ariadne/exported_variables_full.xlsx", + output: + elec_price_duration_curve=RESULTS + + "ariadne/report/elec_price_duration_curve.pdf", + elec_price_duration_hist=RESULTS + "ariadne/report/elec_price_duration_hist.pdf", + backup_capacity=RESULTS + "ariadne/report/backup_capacity.pdf", + backup_generation=RESULTS + "ariadne/report/backup_generation.pdf", + results=directory(RESULTS + "ariadne/report"), + elec_transmission=directory(RESULTS + "ariadne/report/elec_transmission"), + h2_transmission=directory(RESULTS + "ariadne/report/h2_transmission"), + co2_transmission=directory(RESULTS + "ariadne/report/co2_transmission"), + elec_balances=directory(RESULTS + "ariadne/report/elec_balance_timeseries"), + heat_balances=directory(RESULTS + "ariadne/report/heat_balance_timeseries"), + nodal_balances=directory(RESULTS + "ariadne/report/balance_timeseries_2045"), + resources: + mem_mb=32000, + log: + RESULTS + "logs/plot_ariadne_report.log", + script: + scripts("pypsa-de/plot_ariadne_report.py") + + +rule ariadne_report_only: + input: + expand( + RESULTS + "ariadne/report/elec_price_duration_curve.pdf", + run=config_provider("run", "name"), + ), + + +rule plot_scenario_comparison: + input: + exported_variables=expand( + RESULTS + "ariadne/exported_variables_full.xlsx", + run=config_provider("run", "name"), + ), + output: + price_carbon="results/" + + config["run"]["prefix"] + + "/scenario_comparison/Price-Carbon.png", + script: + scripts("pypsa-de/plot_scenario_comparison.py") + + +rule compare_scenarios: + input: + price_carbon="results/" + + config["run"]["prefix"] + + "/scenario_comparison/Price-Carbon.png", + # expand( + # RESULTS + "ariadne/capacity_detailed.png", + # run=config_provider("run", "name"), + # ), + + +rule ariadne_all: + input: + expand( + RESULTS + "ariadne/report/elec_price_duration_curve.pdf", + run=config_provider("run", "name"), + ), + # expand( + # RESULTS + "ariadne/capacity_detailed.png", + # run=config_provider("run", "name"), + # ), + expand( + RESULTS + + "maps/base_s_{clusters}_{opts}_{sector_opts}-h2_network_incl_kernnetz_{planning_horizons}.pdf", + run=config_provider("run", "name"), + **config["scenario"], + allow_missing=True, + ), + price_carbon="results/" + + config["run"]["prefix"] + + "/scenario_comparison/Price-Carbon.png", diff --git a/rules/pypsa-de/retrieve.smk b/rules/pypsa-de/retrieve.smk new file mode 100644 index 000000000..c022ef63c --- /dev/null +++ b/rules/pypsa-de/retrieve.smk @@ -0,0 +1,94 @@ +# SPDX-FileCopyrightText: Contributors to PyPSA-DE +# +# SPDX-License-Identifier: CC BY 4.0 + + +if (ARIADNE_DATABASE := dataset_version("ariadne_database"))["source"] in ["primary"]: + + rule retrieve_ariadne_database: + params: + source="primary", + output: + data="data/ariadne_database.csv", + log: + "logs/retrieve_ariadne_database_primary.log", + resources: + mem_mb=1000, + script: + scripts("pypsa-de/retrieve_ariadne_database.py") + + +if (ARIADNE_DATABASE := dataset_version("ariadne_database"))["source"] in ["archive"]: + + rule retrieve_ariadne_database: + params: + source="archive", + input: + raw_xlsx=storage(ARIADNE_DATABASE["url"]), + output: + data="data/ariadne_database.csv", + log: + "logs/retrieve_ariadne_database_archive.log", + resources: + mem_mb=1000, + script: + scripts("pypsa-de/retrieve_ariadne_database.py") + + +if (ARIADNE_TEMPLATE := dataset_version("ariadne_template"))["source"] in [ + "primary", + "archive", +]: + + rule retrieve_ariadne_template: + input: + storage(ARIADNE_TEMPLATE["url"]), + output: + "data/template_ariadne_database.xlsx", + run: + move(input[0], output[0]) + + +if (OPEN_MASTR := dataset_version("open_mastr"))["source"] in ["primary", "archive"]: + + rule retrieve_open_mastr: + input: + storage(OPEN_MASTR["url"]), + params: + "data/mastr", + output: + "data/mastr/bnetza_open_mastr_2023-08-08_B_biomass.csv", + "data/mastr/bnetza_open_mastr_2023-08-08_B_combustion.csv", + run: + unpack_archive(input[0], params[0]) + + +if (EGON := dataset_version("egon"))["source"] in ["build"]: + + rule retrieve_egon_data: + params: + url=EGON["url"], + folder=EGON["folder"], + output: + spatial=f"{EGON['folder']}/demandregio_spatial_2018.json", + mapping=f"{EGON['folder']}/mapping_technologies.json", + shell: + """ + mkdir -p {params.folder} + curl -o {output.spatial} "{params.url}?id_spatial=5&year=2018" + curl -o {output.mapping} "{params.url}_description?id_spatial=5" + """ + + +if (EGON := dataset_version("egon"))["source"] in ["archive"]: + + rule retrieve_egon_data: + input: + spatial=storage(f"{EGON['url']}/demandregio_spatial_2018.json"), + mapping=storage(f"{EGON['url']}/mapping_technologies.json"), + output: + spatial=f"{EGON['folder']}/demandregio_spatial_2018.json", + mapping=f"{EGON['folder']}/mapping_technologies.json", + run: + copy2(input["spatial"], output["spatial"]) + copy2(input["mapping"], output["mapping"]) diff --git a/scripts/pypsa-de/plot_ariadne_report.py b/scripts/pypsa-de/plot_ariadne_report.py index c5e3c8f9a..b514bf05d 100644 --- a/scripts/pypsa-de/plot_ariadne_report.py +++ b/scripts/pypsa-de/plot_ariadne_report.py @@ -1272,7 +1272,7 @@ def plot_backup_capacity( df_all = pd.DataFrame() - for year in np.arange(2020, 2050, 5): + for year in networks.keys(): n = networks[year] electricity_cap = ( @@ -1293,7 +1293,7 @@ def plot_backup_capacity( df_all = pd.concat([df_all, df], axis=1) - df_all.columns = np.arange(2020, 2050, 5) + df_all.columns = list(networks.keys()) tech_colors["coal"] = "black" @@ -1392,7 +1392,7 @@ def plot_backup_generation( df_all = pd.DataFrame() - for year in np.arange(2020, 2050, 5): + for year in networks.keys(): n = networks[year] electricity_supply_de = ( @@ -1412,7 +1412,7 @@ def plot_backup_generation( df = df[df > 0.01] df_all = pd.concat([df_all, df], axis=1) - df_all.columns = np.arange(2020, 2050, 5) + df_all.columns = list(networks.keys()) # Create figure plt.figure(figsize=(18, 5)) @@ -3147,7 +3147,7 @@ def plot_h2_trade( for s in scenarios: plot_elec_map_de( networks[planning_horizons.index(year)], - networks[planning_horizons.index(2020)], + networks[0], tech_colors, regions_de, savepath=f"{snakemake.output.elec_transmission}/elec-transmission-DE-{s}-{year}.pdf", @@ -3156,7 +3156,7 @@ def plot_h2_trade( s = "total-expansion" plot_elec_map_de( networks[planning_horizons.index(year)], - networks[planning_horizons.index(2020)], + networks[0], tech_colors, regions_de, savepath=f"{snakemake.output.elec_transmission}/elec-transmission-DE-{s}-{year}_eng.png", From 07f9231312386d290e3518edd33f3340b6e21e73 Mon Sep 17 00:00:00 2001 From: Michael Lindner Date: Fri, 6 Mar 2026 11:19:04 +0100 Subject: [PATCH 07/13] minor --- scripts/pypsa-de/plot_ariadne_report.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/scripts/pypsa-de/plot_ariadne_report.py b/scripts/pypsa-de/plot_ariadne_report.py index b514bf05d..bf0564350 100644 --- a/scripts/pypsa-de/plot_ariadne_report.py +++ b/scripts/pypsa-de/plot_ariadne_report.py @@ -745,11 +745,6 @@ def plot_nodal_heat_balance( start_date = str(network.generators_t.p.index[0])[:4] + "-" + start_date end_date = str(network.generators_t.p.index[-1])[:4] + "-" + end_date - carriers = carriers - loads = loads - start_date = start_date - end_date = end_date - regions = regions period = network.generators_t.p.index[ (network.generators_t.p.index >= start_date) & (network.generators_t.p.index <= end_date) @@ -849,7 +844,6 @@ def f(c): labels = [nice_names_dict.get(l, l) for l in labels] if german_carriers: - german_carriers labels = [carriers_in_german.get(l, l) for l in labels] # rescale the y-axis From 8024efc40881ce576d7e445db0c31ae01d249ab0 Mon Sep 17 00:00:00 2001 From: Michael Lindner Date: Fri, 6 Mar 2026 11:20:36 +0100 Subject: [PATCH 08/13] restore option for using the internal db --- config/config.de.yaml | 6 +- data/versions.csv | 1 + rules/pypsa-de/retrieve.smk | 83 ++++++++++++------- scripts/pypsa-de/build_scenarios.py | 11 ++- scripts/pypsa-de/retrieve_ariadne_database.py | 10 +++ 5 files changed, 76 insertions(+), 35 deletions(-) diff --git a/config/config.de.yaml b/config/config.de.yaml index b139cf7d9..5715e85dc 100644 --- a/config/config.de.yaml +++ b/config/config.de.yaml @@ -35,7 +35,8 @@ pypsa-de: general: REMIND-EU v1.1 transport: Aladin v1 industry: FORECAST v1.0 - reference_scenario: KN2045_Mix + use_internal_db: false + reference_scenario: KN2045_Mix # KN2045_Bal_v4 region: Deutschland ageb_for_mobility: true # In 2020 use AGEB data for final energy demand and KBA for vehicles uba_for_mobility: # Available for 2025–2035; uses MWMS scenario from UBA Projektionsbericht 2025 @@ -717,6 +718,9 @@ data: ariadne_database: source: archive version: latest + ariadne_database_internal: + source: primary + version: latest ariadne_template: source: archive version: latest diff --git a/data/versions.csv b/data/versions.csv index 22d42df90..e6d1de838 100644 --- a/data/versions.csv +++ b/data/versions.csv @@ -3,6 +3,7 @@ aquifer_data,v1.2,primary,latest supported,2025-12-02,,https://download.bgr.de/b aquifer_data,v1.2,archive,latest supported,2026-01-13,,https://data.pypsa.org/workflows/eur/aquifer_data/v1.2/IHME1500_v12.zip ariadne_database,v1.0,primary,latest supported,2026-01-15,Public facing database of the second Ariadne Szenarienbericht,https://zenodo.org/records/15174592/files/250505_Ariadne2_Data_v1.0.xlsx ariadne_database,v1.0,archive,latest supported,2026-02-23,Public facing database of the second Ariadne Szenarienbericht,https://data.pypsa.org/workflows/de/ariadne_database/v1.0/250505_Ariadne2_Data_v1.0.xlsx +ariadne_database_internal,rolling,primary,latest supported,2026-03-06,Internal database of the second Ariadne Szenarienbericht,https://data.ene.iiasa.ac.at/ariadne2-intern ariadne_template,2025-11-20,primary,latest supported,2026-01-15,,https://github.com/iiasa/ariadne-intern-workflow/raw/7caa948ad6aa71cf8d17e69a64dd29f01306a9ff/attachments/2025-11-20_template_Ariadne.xlsx ariadne_template,2025-11-20,archive,latest supported,2026-02-23,,https://data.pypsa.org/workflows/de/ariadne_template/2025-11-20/2025-11-20_template_Ariadne.xlsx attributed_ports,2020-07-10,primary,latest supported,2025-12-02,,https://datacatalogfiles.worldbank.org/ddh-published/0038118/1/DR0046414/attributed_ports.geojson diff --git a/rules/pypsa-de/retrieve.smk b/rules/pypsa-de/retrieve.smk index c022ef63c..0a523c12e 100644 --- a/rules/pypsa-de/retrieve.smk +++ b/rules/pypsa-de/retrieve.smk @@ -2,37 +2,58 @@ # # SPDX-License-Identifier: CC BY 4.0 - -if (ARIADNE_DATABASE := dataset_version("ariadne_database"))["source"] in ["primary"]: - - rule retrieve_ariadne_database: - params: - source="primary", - output: - data="data/ariadne_database.csv", - log: - "logs/retrieve_ariadne_database_primary.log", - resources: - mem_mb=1000, - script: - scripts("pypsa-de/retrieve_ariadne_database.py") - - -if (ARIADNE_DATABASE := dataset_version("ariadne_database"))["source"] in ["archive"]: - - rule retrieve_ariadne_database: - params: - source="archive", - input: - raw_xlsx=storage(ARIADNE_DATABASE["url"]), - output: - data="data/ariadne_database.csv", - log: - "logs/retrieve_ariadne_database_archive.log", - resources: - mem_mb=1000, - script: - scripts("pypsa-de/retrieve_ariadne_database.py") +if config["pypsa-de"]["use_internal_db"]: + if (ARIADNE_DATABASE_INTERNAL := dataset_version("ariadne_database_internal"))[ + "source" + ] in ["primary"]: + + rule retrieve_ariadne_database_internal: + params: + source="internal", + output: + data="data/ariadne_database.csv", + log: + "logs/retrieve_ariadne_database_internal_primary.log", + resources: + mem_mb=1000, + script: + scripts("pypsa-de/retrieve_ariadne_database.py") + +else: + if (ARIADNE_DATABASE := dataset_version("ariadne_database"))["source"] in [ + "primary" + ]: + + rule retrieve_ariadne_database: + params: + source="primary", + output: + data="data/ariadne_database.csv", + log: + "logs/retrieve_ariadne_database_primary.log", + resources: + mem_mb=1000, + script: + scripts("pypsa-de/retrieve_ariadne_database.py") + + if (ARIADNE_DATABASE := dataset_version("ariadne_database"))["source"] in [ + "archive" + ]: + + rule retrieve_ariadne_database: + params: + source="archive", + version=ARIADNE_DATABASE["version"], + input: + raw_xlsx=storage(ARIADNE_DATABASE["url"]), + output: + data="data/ariadne_database.csv", + log: + "logs/retrieve_ariadne_database_archive.log", + resources: + mem_mb=1000, + script: + scripts("pypsa-de/retrieve_ariadne_database.py") if (ARIADNE_TEMPLATE := dataset_version("ariadne_template"))["source"] in [ diff --git a/scripts/pypsa-de/build_scenarios.py b/scripts/pypsa-de/build_scenarios.py index db6da62de..46bbff03d 100644 --- a/scripts/pypsa-de/build_scenarios.py +++ b/scripts/pypsa-de/build_scenarios.py @@ -15,9 +15,14 @@ def get_transport_growth(df, planning_horizons): - aviation = df.xs( - ("Final Energy|Bunkers|Aviation", "TWh/yr"), level=("variable", "unit") - ).squeeze() + aviation = df.xs("Final Energy|Bunkers|Aviation", level="variable") + unit = aviation.index.get_level_values("unit").item() + aviation = aviation.squeeze() + + if unit == "PJ/yr": + aviation /= 3.6 # convert PJ to TWh + elif unit != "TWh/yr": + raise ValueError("Unexpected unit for aviation energy demand.") aviation[2020] = 111.25 # Ariadne2-internal DB, Aladin model aviation_growth_factor = aviation / aviation[2020] diff --git a/scripts/pypsa-de/retrieve_ariadne_database.py b/scripts/pypsa-de/retrieve_ariadne_database.py index 3975fa150..db5bfa576 100644 --- a/scripts/pypsa-de/retrieve_ariadne_database.py +++ b/scripts/pypsa-de/retrieve_ariadne_database.py @@ -21,6 +21,16 @@ logger.info("Successfully retrieved database.") db.timeseries().to_csv(snakemake.output.data) + elif snakemake.params.get("source") == "internal": + import pyam + + logger.info("Retrieving from IIASA internal database 'ariadne2_intern'.") + + db = pyam.read_iiasa("ariadne2_intern") + + logger.info("Successfully retrieved database.") + db.timeseries().to_csv(snakemake.output.data) + elif snakemake.params.get("source") == "archive": # Read all sheets first; then select the one called "data". sheets = pd.read_excel(snakemake.input.raw_xlsx, sheet_name=None) From 0383a601a9ba4a27ef4a8a42b0ecce122ae5f3c3 Mon Sep 17 00:00:00 2001 From: Michael Lindner Date: Fri, 6 Mar 2026 13:49:09 +0100 Subject: [PATCH 09/13] convert units for internal ariadne DB --- scripts/pypsa-de/build_exogenous_mobility_data.py | 8 +++++++- scripts/pypsa-de/export_ariadne_variables.py | 6 ++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/scripts/pypsa-de/build_exogenous_mobility_data.py b/scripts/pypsa-de/build_exogenous_mobility_data.py index 4d80a0ca4..0f7660df6 100644 --- a/scripts/pypsa-de/build_exogenous_mobility_data.py +++ b/scripts/pypsa-de/build_exogenous_mobility_data.py @@ -120,7 +120,13 @@ def get_mobility_data( for fuel in fuels: for subsector in subsectors: key = f"Final Energy|Transportation|{subsector}|{fuel}" - mobility_data.loc[fuel] += df.get((key, "TWh/yr"), 0.0) + unit = "TWh/yr" + if unit not in df.index.get_level_values("unit").unique(): + unit = "PJ/yr" + increment = df.get((key, unit), 0.0) + if unit == "PJ/yr": + increment /= 3.6 # convert PJ to TWh + mobility_data.loc[fuel] += increment mobility_data = mobility_data.mul(1e6) # convert TWh to MWh mobility_data["million_EVs"] = ( diff --git a/scripts/pypsa-de/export_ariadne_variables.py b/scripts/pypsa-de/export_ariadne_variables.py index d5f43f069..da5fe3573 100644 --- a/scripts/pypsa-de/export_ariadne_variables.py +++ b/scripts/pypsa-de/export_ariadne_variables.py @@ -5743,6 +5743,12 @@ def get_data( } ) + # For export to the Ariadne-internal DB, convert most Wh-based entries to J + ariadne_df.loc[ariadne_df["Unit"] == "TWh/yr", planning_horizons] *= 3.6 + ariadne_df.loc[ariadne_df["Unit"] == "TWh/yr", "Unit"] = "PJ/yr" + ariadne_df.loc[ariadne_df["Unit"] == "EUR2020/GJ", planning_horizons] /= 3.6 + ariadne_df.loc[ariadne_df["Unit"] == "EUR2020/GJ", "Unit"] = "EUR2020/MWh" + with pd.ExcelWriter(snakemake.output.exported_variables) as writer: ariadne_df.round(5).to_excel(writer, sheet_name="data", index=False) meta.to_frame().T.to_excel(writer, sheet_name="meta", index=False) From 59ac1b16d711ece48a1636546cb593b56e1ab404 Mon Sep 17 00:00:00 2001 From: Michael Lindner Date: Mon, 9 Mar 2026 10:26:05 +0100 Subject: [PATCH 10/13] minor --- scripts/pypsa-de/export_ariadne_variables.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/pypsa-de/export_ariadne_variables.py b/scripts/pypsa-de/export_ariadne_variables.py index da5fe3573..7310f45fa 100644 --- a/scripts/pypsa-de/export_ariadne_variables.py +++ b/scripts/pypsa-de/export_ariadne_variables.py @@ -5746,8 +5746,8 @@ def get_data( # For export to the Ariadne-internal DB, convert most Wh-based entries to J ariadne_df.loc[ariadne_df["Unit"] == "TWh/yr", planning_horizons] *= 3.6 ariadne_df.loc[ariadne_df["Unit"] == "TWh/yr", "Unit"] = "PJ/yr" - ariadne_df.loc[ariadne_df["Unit"] == "EUR2020/GJ", planning_horizons] /= 3.6 - ariadne_df.loc[ariadne_df["Unit"] == "EUR2020/GJ", "Unit"] = "EUR2020/MWh" + ariadne_df.loc[ariadne_df["Unit"] == "EUR2020/MWh", planning_horizons] /= 3.6 + ariadne_df.loc[ariadne_df["Unit"] == "EUR2020/MWh", "Unit"] = "EUR2020/GJ" with pd.ExcelWriter(snakemake.output.exported_variables) as writer: ariadne_df.round(5).to_excel(writer, sheet_name="data", index=False) From 222015662750352408bf16db53851a6fd2e697cf Mon Sep 17 00:00:00 2001 From: Michael Lindner Date: Mon, 9 Mar 2026 10:51:30 +0100 Subject: [PATCH 11/13] improve existing heat pump estimate --- scripts/pypsa-de/modify_existing_heating.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/pypsa-de/modify_existing_heating.py b/scripts/pypsa-de/modify_existing_heating.py index 4f034f8e5..fed8a3162 100644 --- a/scripts/pypsa-de/modify_existing_heating.py +++ b/scripts/pypsa-de/modify_existing_heating.py @@ -40,9 +40,11 @@ ) # Schornsteinfeger Kohle: 0.08 new_values["oil boiler"] = 5.99 # Schornsteinfeger: 3.86 + 0.965 - new_values["air heat pump"] = 0.38 - new_values["ground heat pump"] = 0.38 new_values["biomass boiler"] = 2.8 # Schornsteinfeger: 1.16 - 0.08 (Zentral) + + new_values["air heat pump"] = ( + 1.7 # Heat pumps approximated based on https://www.waermepumpe.de/fileadmin/user_upload/Mediengalerie/Zahlen_und_Daten/Absatzzahlen_Marktanteile/Diagramm_Absatz_WP_2006-2025.png + ) + new_values["ground heat pump"] = 0.5 total_stock = new_values.sum() existing_factor = existing_heating.loc["Germany"].sum() / total_stock From 8b0be4fc721c5290d66624f746f5f9a5a4dac370 Mon Sep 17 00:00:00 2001 From: Michael Lindner Date: Mon, 9 Mar 2026 11:00:57 +0100 Subject: [PATCH 12/13] minor --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9907d8d85..bc8970a54 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ to generate the solved networks, or snakemake -c1 ariadne_all -which will generators additional outputs related to the Ariadne project. If computational resources on your local machine are limited you may decrease the number of cores by adding, e.g. `-c4` to the call to get only 4 cores. For more option please refer to the [snakemake](https://snakemake.readthedocs.io/en/stable/) documentation. +which will generators additional outputs related to the Ariadne project. If you would like to use more computational resources you may increase the number of cores by adding, e.g. `-c4` to the call to get only 4 cores, or `-call` to use all cores. For more option please refer to the [snakemake](https://snakemake.readthedocs.io/en/stable/) documentation. ## Repo structure From c2252d2aaca93c7135c3f29702f708f1f9205135 Mon Sep 17 00:00:00 2001 From: Michael Lindner Date: Fri, 13 Mar 2026 13:03:45 +0100 Subject: [PATCH 13/13] increase tolerance of assert --- scripts/pypsa-de/export_ariadne_variables.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/pypsa-de/export_ariadne_variables.py b/scripts/pypsa-de/export_ariadne_variables.py index 7310f45fa..bc09e1960 100644 --- a/scripts/pypsa-de/export_ariadne_variables.py +++ b/scripts/pypsa-de/export_ariadne_variables.py @@ -3200,6 +3200,8 @@ def get_emissions(n, region, _energy_totals, _industry_demand): ], ).sum() + CHP_emissions_E.sum(), + rtol=1e-2, + atol=1e-2, ) var["Emissions|CO2|Energy|Supply|Electricity"] = ( @@ -5573,6 +5575,7 @@ def get_data( ll="vopt", sector_opts="None", run="KN2045_Mix", + configfiles="config/test/config.dach.yaml", ) configure_logging(snakemake) set_scenario_config(snakemake)