statistics post processing for csv creation and plotting

config/config.default.yaml

@@ -1026,6 +1026,22 @@ plotting:
   energy_max: 20000
   energy_min: -20000
   energy_threshold: 50.
+  statistics:
+    countries:
+    - all
+    carriers:
+    - all
+    - electricity
+    - heat
+    - co2
+    carrier_groups:
+      electricity: [AC, low_voltage]
+    metrics:
+    - energy_balance
+    - total_cost
+    comparison_folder: ""
+    scenario_comparison:
+    - ""
 
   nice_names:
     OCGT: "Open-Cycle Gas"

doc/configtables/plotting.csv

@@ -1,13 +1,19 @@
 ,Unit,Values,Description
 map,,,
 -- boundaries,°,"[x1,x2,y1,y2]",Boundaries of the map plots in degrees latitude (y) and longitude (x)
-projection,,,,
--- name,--,"Valid Cartopy projection name","See https://scitools.org.uk/cartopy/docs/latest/reference/projections.html for list of available projections."
+projection,,,
+-- name,--,Valid Cartopy projection name,See https://scitools.org.uk/cartopy/docs/latest/reference/projections.html for list of available projections.
 -- args,--,--,"Other entries under 'projection' are passed as keyword arguments to the projection constructor, e.g. ``central_longitude: 10.``."
 costs_max,bn Euro,float,Upper y-axis limit in cost bar plots.
 costs_threshold,bn Euro,float,Threshold below which technologies will not be shown in cost bar plots.
 energy_max,TWh,float,Upper y-axis limit in energy bar plots.
 energy_min,TWh,float,Lower y-axis limit in energy bar plots.
 energy_threshold,TWh,float,Threshold below which technologies will not be shown in energy bar plots.
-tech_colors,--,carrier -> HEX colour code,Mapping from network ``carrier`` to a colour (`HEX colour code <https://en.wikipedia.org/wiki/Web_colors#Hex_triplet>`_).
+countries,--,[str],List of countries you want to create the statistcs for. Statistics includes csv files and figures. Default is all coutries but also single countries can be defined like ``DE``.
+carriers,--,[str],"Carrier you want to create the statistcs for, i.e. csv files and figures. This carrier must be an available bus carrier in your network or substring of a carrier or a defined carrier group."
+carrier_groups,--,str -> carrier,Mapping from ``carrier_groups`` to individual bus carriers in the network.
+metrics,--,[str],"List of metrics you want to create your statistics for. Default is `energy_balance' and 'total_cost'. However, you can choose other metrics from the list below:",
+comparison_folder, --,str,"Folder name where the scenario comparison statistics are stored. If this is empty, no senario comparison plots will be generated Defaultis an empty string ''."
+scenario_comparison,--,[str],"List of scenarios you want to compare. If the list is empty, all scenarios from ``run:name`` are compared. Only works, when a ``comparison_folder`` is defined. Default is an empty list."
 nice_names,--,str -> str,Mapping from network ``carrier`` to a more readable name.
+tech_colors,--,carrier -> HEX colour code,Mapping from network ``carrier`` to a colour (`HEX colour code <https://en.wikipedia.org/wiki/Web_colors#Hex_triplet>`_).

doc/release_notes.rst

@@ -478,6 +478,9 @@ PyPSA-Eur 0.11.0 (25th May 2024)
 
 **New Features**
 
+* Add plotting routine with statistics where we allow for plotting of individual energy carriers and countries. Besides the plots, we create all necessary csv files for the plotting routine.
+
+
 * Introduced scenario management to support the simultaneous execution of
   multiple scenarios with a single ``snakemake`` call. A ``scenarios.yaml`` file
   allows customizable scenario names with configuration overrides. To enable,
@@ -575,6 +578,16 @@ PyPSA-Eur 0.11.0 (25th May 2024)
   fraction of carbon in plastics that is permanently sequestered in landfills.
   The default assumption is that all carbon in plastics is eventually released
   to the atmosphere. (https://github.com/PyPSA/pypsa-eur/pull/1060)
+* Removed rule ``copy_config``. Instead, a config file is created for each
+  network output of the ``solve_*`` rules, with the same content as ``n.meta``.
+
+* Added new HVDC transmission projects from `TYNDP 2024 draft projects
+  <https://tyndp.entsoe.eu/news/176-pan-european-electricity-transmission-projects-and-33-storage-projects-will-be-assessed-in-tyndp-2024>`__.
+
+* Upgrade to Snakemake v8.5+. This version is the new minimum version required.
+  To upgrade an existing environment, run ``conda install -c bioconda
+  snakemake-minimal">=8.5"`` and ``pip install snakemake-storage-plugin-http``
+  (https://github.com/PyPSA/pypsa-eur/pull/825).
 
 * Added options for building waste-to-energy plants with and without carbon
   capture to consume non-recycled and non-sequestered plastics. Config settings:

rules/collect.smk

@@ -83,3 +83,63 @@ rule validate_elec_networks:
             run=config["run"]["name"],
             kind=["production", "prices", "cross_border"],
         ),
+
+
+rule plot_statistics:
+    input:
+        lambda w: expand(
+            (
+                RESULTS
+                + "statistics/csv/base_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}/country_{country}/.statistics_{carrier}_csv"
+            ),
+            **config["scenario"],
+            run=config["run"]["name"],
+            country=config_provider("plotting", "statistics")(w).get(
+                "countries", "all"
+            ),
+            carrier=config_provider("plotting", "statistics")(w).get(
+                "carriers", "all"
+            ),
+            allow_missing=True,
+        ),
+        expand(
+            RESULTS
+            + "statistics/figures/single/base_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}/country_{country}/.statistics_{carrier}_plots",
+            **config["scenario"],
+            country=config_provider("plotting", "statistics")(run).get(
+                "countries", "all"
+            ),
+            carrier=config_provider("plotting", "statistics")(run).get(
+                "carriers", "all"
+            ),
+            run=config["run"]["name"],
+        ),
+        expand(
+            RESULTS
+            + "statistics/figures/comparison/country_{country}/.statistics_{carrier}_plots",
+            country=config_provider("plotting", "statistics")(run).get(
+                "countries", "all"
+            ),
+            carrier=config_provider("plotting", "statistics")(run).get(
+                "carriers", "all"
+            ),
+            run=config["run"]["name"],
+        ),
+        expand(
+            "results/statistics/"
+            + config_provider("plotting", "statistics")(run).get(
+                "comparison_folder", "''"
+            )
+            + "/"
+            + "figures/country_{country}/.statistics_{carrier}_plots",
+            country=config_provider("plotting", "statistics")(run).get(
+                "countries", "all"
+            ),
+            carrier=config_provider("plotting", "statistics")(run).get(
+                "carriers", "all"
+            ),
+            run=config["run"]["name"],
+        )
+        if config_provider("plotting", "statistics")(run).get("comparison_folder", "")
+        != ""
+        else [],

rules/postprocess.smk

@@ -274,3 +274,159 @@ rule plot_base_statistics:
         + "figures/.statistics_plots_base_s_{clusters}_elec_l{ll}_{opts}",
     script:
         "../scripts/plot_statistics.py"
+
+
+STATISTICS = {
+    "capacity_factor": ("-", "p.u."),
+    "installed_capacity": (1e3, "GW"),
+    "optimal_capacity": (1e3, "GW"),
+    "capex": (1e9, "bn €"),
+    "opex": (1e9, "bn €"),
+    "total_cost": ("1e9", "bn €"),
+    "curtailment": (1e3, "GWh"),
+    "supply": (1e6, "TWh"),
+    "withdrawal": (1e6, "TWh"),
+    "energy_balance": (1e6, "TWh"),
+    "market_value": ("-", "€/MWh"),
+}
+
+
+rule write_statistics:
+    params:
+        statistics=STATISTICS,
+    input:
+        network=RESULTS
+        + "postnetworks/base_s_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
+    output:
+        **{
+            f"{metric}": RESULTS
+            + "statistics/csv/base_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}/country_{country}/{carrier}_"
+            + f"{metric}.csv"
+            for carrier in config_provider("plotting", "statistics")(run).get(
+                "carriers", "all"
+            )
+            for metric in config_provider("plotting", "statistics")(run).get(
+                "metrics", STATISTICS
+            )
+        },
+        csv_touch=RESULTS
+        + "statistics/csv/base_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}/country_{country}/.statistics_{carrier}_csv",
+    log:
+        RESULTS
+        + "logs/write_statistics/base_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_country-{country}_carrier-{carrier}.log",
+    script:
+        "../scripts/write_statistics.py"
+
+
+rule plot_statistics_single:
+    params:
+        plotting=config_provider("plotting"),
+        statistics=STATISTICS,
+    input:
+        **{
+            f"{metric}": RESULTS
+            + "statistics/csv/base_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}/country_{country}/{carrier}_"
+            + f"{metric}.csv"
+            for carrier in config_provider("plotting", "statistics")(run).get(
+                "carriers", "all"
+            )
+            for metric in config_provider("plotting", "statistics")(run).get(
+                "metrics", STATISTICS
+            )
+        },
+    output:
+        **{
+            f"{metric}": RESULTS
+            + "statistics/figures/single/base_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}/country_{country}/{carrier}_"
+            + f"{metric}.pdf"
+            for carrier in config_provider("plotting", "statistics")(run).get(
+                "carriers", "all"
+            )
+            for metric in config_provider("plotting", "statistics")(run).get(
+                "metrics", STATISTICS
+            )
+        },
+        barplots_touch=RESULTS
+        + "statistics/figures/single/base_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}/country_{country}/.statistics_{carrier}_plots",
+    log:
+        RESULTS
+        + "logs/plot_statistics_single/base_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_country-{country}_carrier-{carrier}.log",
+    script:
+        "../scripts/plot_statistics_single.py"
+
+
+rule plot_statistics_comparison:
+    params:
+        plotting=config_provider("plotting"),
+        statistics=STATISTICS,
+    input:
+        expand(
+            RESULTS
+            + "statistics/csv/base_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}/country_{country}/{carrier}_{metric}.csv",
+            **config["scenario"],
+            metric=config_provider("plotting", "statistics")(run).get(
+                "metrics", STATISTICS
+            ),
+            allow_missing=True,
+        ),
+    output:
+        **{
+            f"{metric}": RESULTS
+            + "statistics/figures/comparison/country_{country}/{carrier}_"
+            + f"{metric}.pdf"
+            for carrier in config_provider("plotting", "statistics")(run).get(
+                "carriers", "all"
+            )
+            for metric in config_provider("plotting", "statistics")(run).get(
+                "metrics", STATISTICS
+            )
+        },
+        barplots_touch=RESULTS
+        + "statistics/figures/comparison/country_{country}/.statistics_{carrier}_plots",
+    log:
+        RESULTS
+        + "logs/plot_statistics_comparison/country-{country}_carrier-{carrier}.log",
+    script:
+        "../scripts/plot_statistics_comparison.py"
+
+
+def get_scnario_copmarison_run(w):
+    run = config_provider("plotting", "statistics")(w).get(
+        "scenario_comparison", config["run"]["name"]
+    )
+    if run == [""] or run == "":
+        run = config["run"]["name"]
+    return run
+
+
+rule plot_statistics_scenario_comparison:
+    params:
+        plotting=config_provider("plotting"),
+        statistics=STATISTICS,
+    input:
+        expand(
+            RESULTS
+            + "statistics/csv/base_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}/country_{country}/{carrier}_{metric}.csv",
+            **config["scenario"],
+            metric=config_provider("plotting", "statistics")(run).get(
+                "metrics", STATISTICS
+            ),
+            run=get_scnario_copmarison_run(run),
+            allow_missing=True,
+        ),
+    output:
+        **{
+            f"{metric}": "results/statistics/{comparison_folder}/figures/country_{country}/{carrier}_"
+            + f"{metric}.pdf"
+            for carrier in config_provider("plotting", "statistics")(run).get(
+                "carriers", "all"
+            )
+            for metric in config_provider("plotting", "statistics")(run).get(
+                "metrics", STATISTICS
+            )
+        },
+        barplots_touch="results/statistics/{comparison_folder}/figures/country_{country}/.statistics_{carrier}_plots",
+    log:
+        "results/logs/{comparison_folder}/plot_statistics_scenario_comparison/country-{country}_carrier-{carrier}.log",
+    script:
+        "../scripts/plot_statistics_comparison.py"