From e2bd3926b39396825c2dcc7f10edcf791dfe4ae9 Mon Sep 17 00:00:00 2001 From: Philipp Glaum Date: Fri, 9 Feb 2024 13:47:24 +0100 Subject: [PATCH 01/18] implement statistics post processing for csv creation and plotting. --- config/config.default.yaml | 9 ++- rules/collect.smk | 19 +++++ rules/postprocess.smk | 89 ++++++++++++++++++++- scripts/make_statistics_csv.py | 98 +++++++++++++++++++++++ scripts/plot_statistics_comparison.py | 109 ++++++++++++++++++++++++++ scripts/plot_statistics_single.py | 79 +++++++++++++++++++ 6 files changed, 400 insertions(+), 3 deletions(-) create mode 100644 scripts/make_statistics_csv.py create mode 100644 scripts/plot_statistics_comparison.py create mode 100644 scripts/plot_statistics_single.py diff --git a/config/config.default.yaml b/config/config.default.yaml index 93b0568e3..5101b66e1 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -44,7 +44,7 @@ scenario: opts: - '' sector_opts: - - Co2L0-3H-T-H-B-I-A-dist1 + - Co2L0-3h-T-H-B-I-A-dist1 planning_horizons: # - 2020 # - 2030 @@ -813,6 +813,13 @@ plotting: energy_max: 20000 energy_min: -20000 energy_threshold: 50. + countries: + - all + carriers: + - electricity + - heat + carrier_groups: + electricity: [AC, low_voltage] nice_names: OCGT: "Open-Cycle Gas" diff --git a/rules/collect.smk b/rules/collect.smk index dc0a94cc6..caa46e9bc 100644 --- a/rules/collect.smk +++ b/rules/collect.smk @@ -81,3 +81,22 @@ rule validate_elec_networks: **config["scenario"], kind=["production", "prices", "cross_border"], ), + + +rule plot_statistics: + input: + [ + expand( + RESULTS + + "statistics/figures/comparison/country_{country}/.statistics_{carrier}_plots", + country=config["plotting"].get("countries", "all"), + carrier=config["plotting"].get("carriers", ["all"]), + ), + expand( + RESULTS + + "statistics/figures/single/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}/country_{country}/.statistics_{carrier}_plots", + **config["scenario"], + country=config["plotting"].get("countries", "all"), + carrier=config["plotting"].get("carriers", ["all"]), + ), + ], diff --git a/rules/postprocess.smk b/rules/postprocess.smk index 2e90d2333..ce3141d8e 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -257,14 +257,99 @@ STATISTICS_BARPLOTS = [ "capacity_factor", "installed_capacity", "optimal_capacity", - "capital_expenditure", - "operational_expenditure", + "capex", + "opex", "curtailment", "supply", "withdrawal", "market_value", + "energy_balance", ] +STATISTICS_CSV = [ + "capacity_factor", + "installed_capacity", + "optimal_capacity", + "capex", + "opex", + "curtailment", + "supply", + "withdrawal", + "market_value", +] + + +rule save_statistics_csv: + params: + barplots=STATISTICS_CSV, + input: + network=RESULTS + + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", + output: + **{ + f"{csv}": RESULTS + + "statistics/csv/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}/country_{country}/{carrier}_" + + f"{csv}.csv" + for carrier in config["plotting"].get("carriers", "all") + for csv in STATISTICS_CSV + }, + csv_touch=RESULTS + + "statistics/csv/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}/country_{country}/.statistics_{carrier}_csv", + script: + "../scripts/make_statistics_csv.py" + + +rule plot_statistics_single: + params: + plotting=config["plotting"], + barplots=STATISTICS_BARPLOTS, + input: + **{ + f"{csv}": RESULTS + + "statistics/csv/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}/country_{country}/{carrier}_" + + f"{csv}.csv" + for carrier in config["plotting"].get("carriers", "all") + for csv in STATISTICS_CSV + }, + output: + **{ + f"{plot}": RESULTS + + "statistics/figures/single/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}/country_{country}/{carrier}_" + + f"{plot}.pdf" + for carrier in config["plotting"].get("carriers", "all") + for plot in STATISTICS_BARPLOTS + }, + barplots_touch=RESULTS + + "statistics/figures/single/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}/country_{country}/.statistics_{carrier}_plots", + script: + "../scripts/plot_statistics_single.py" + + +rule plot_statistics_comparison: + params: + plotting=config["plotting"], + barplots=STATISTICS_BARPLOTS, + input: + expand( + RESULTS + + "statistics/csv/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}/country_{country}/{carrier}_{csv}.csv", + **config["scenario"], + csv=STATISTICS_CSV, + allow_missing=True, + ), + output: + **{ + f"{plot}": RESULTS + + "statistics/figures/comparison/country_{country}/{carrier}_" + + f"{plot}.pdf" + for carrier in config["plotting"].get("carriers", "all") + for plot in STATISTICS_BARPLOTS + }, + barplots_touch=RESULTS + + "statistics/figures/comparison/country_{country}/.statistics_{carrier}_plots", + script: + "../scripts/plot_statistics_comparison.py" + rule plot_elec_statistics: params: diff --git a/scripts/make_statistics_csv.py b/scripts/make_statistics_csv.py new file mode 100644 index 000000000..7dfa84044 --- /dev/null +++ b/scripts/make_statistics_csv.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors +# +# SPDX-License-Identifier: MIT + +import matplotlib.pyplot as plt +import pandas as pd +import pypsa +import seaborn as sns +from _helpers import configure_logging +from pypsa.statistics import get_carrier + + +# grouperfunctions = hier schreiben und dann in statistics. +def groupby_country_and_carrier(n, c, nice_names=False): + df = n.df(c) + bus = "bus1" if "bus" not in n.df(c) else "bus" + country = df[bus].map(n.buses.location).map(n.buses.country).rename("country") + carrier = get_carrier(n, c, nice_names) + return [country, carrier] + + +if __name__ == "__main__": + if "snakemake" not in globals(): + from _helpers import mock_snakemake + + snakemake = mock_snakemake( + "save_statistics_csv", + simpl="", + ll="v1.5", + clusters="5", + opts="", + sector_opts="24h-T-H-B-I-A-dist1", + planning_horizons="2040", + country="all", + carrier="electricity", + ) + # configure_logging(snakemake) + config = snakemake.config + + n = pypsa.Network(snakemake.input.network) + + kwargs = {"nice_names": False} + + wildcards = dict(snakemake.wildcards) + carrier = wildcards.pop("carrier") + country = wildcards.pop("country") + + if carrier == "all": + pass + elif carrier in config["plotting"].get("carrier_groups", []): + bus_carrier = config["plotting"]["carrier_groups"][carrier] + kwargs["bus_carrier"] = bus_carrier + elif n.buses.carrier.str.contains(carrier).any(): + if carrier not in n.buses.carrier.unique(): + carrier = [ + bus_carrier + for bus_carrier in n.buses.carrier.unique() + if carrier in bus_carrier + ] + bus_carrier = carrier + kwargs["bus_carrier"] = bus_carrier + else: + raise ValueError( + f"Carrier {carrier} not found in network or carrier group in config." + ) + + if country == "all" or not country: + kwargs["groupby"] = get_carrier + else: + kwargs["groupby"] = groupby_country_and_carrier + + for output in snakemake.output.keys(): + if "touch" in output: + continue + if output != "energy_balance": + func = eval(f"n.statistics.{output}") + try: + ds = func(**kwargs) + if ds.empty: + print( + f"Empty series for {output} with bus carrier {bus_carrier} and country {country}." + ) + pd.Series().to_csv(snakemake.output[output]) + continue + except Exception as e: + print(f"An error occurred: {e}") + pd.Series().to_csv(snakemake.output[output]) + pass + continue + if country and country != "all": + ds = ds.xs(country, level="country") + ds.name = ds.attrs["name"] + ds.dropna().to_csv(snakemake.output[output]) + # touch file + with open(snakemake.output.csv_touch, "a"): + pass diff --git a/scripts/plot_statistics_comparison.py b/scripts/plot_statistics_comparison.py new file mode 100644 index 000000000..73776dd51 --- /dev/null +++ b/scripts/plot_statistics_comparison.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors +# +# SPDX-License-Identifier: MIT + +import re + +import matplotlib.pyplot as plt +import matplotlib.ticker as ticker +import pandas as pd +import seaborn as sns +from _helpers import configure_logging +from plot_summary import rename_techs + +sns.set_theme("paper", style="whitegrid") +STACKED = { + "capacity_factor": False, + "market_value": False, +} + + +def rename_index(df): + # rename index and drop duplicates + specific = df.index.map(lambda x: f"{x[1]}({x[0]})") + generic = df.index.get_level_values("carrier") + duplicated = generic.duplicated(keep=False) + index = specific.where(duplicated, generic) + df = df.set_axis(index) + # rename columns and drop duplicates + columns = df.columns.str.split("_", expand=True) + columns = [ + columns.get_level_values(level).unique() + for level in range(columns.nlevels) + if not columns.get_level_values(level).duplicated(keep=False).all() + ] + columns = pd.MultiIndex.from_arrays(columns) + df.columns = columns.map("\n".join) + return df + + +def plot_static_comparison(df, ax, stacked=False): + df = df[df != 0] + df = df.dropna(axis=0, how="all").fillna(0) + c = tech_colors[df.index.get_level_values("carrier").map(rename_techs)] + df = df.pipe(rename_index).T + df.plot.bar(color=c.values, ax=ax, stacked=stacked, legend=False) + ax.legend( + bbox_to_anchor=(1, 1), + loc="upper left", + ) + ax.grid(axis="x") + + +def read_csv(input, output): + try: + files = list(filter(lambda x: output in x, input)) + pattern = r"elec_.*?(\d{4})" + network_labels = [re.search(pattern, f).group() for f in files] + df = pd.concat( + [pd.read_csv(f).set_index(["component", "carrier"]) for f in files], + axis=1, + keys=network_labels, + ) + # get plot label and drop from index + label = df.columns.get_level_values(1).unique()[0] + df.columns = df.columns.droplevel(1) + except Exception as e: + print(f"Error reading csv file for {output}: {e}") + df = pd.DataFrame() + return df + + +if __name__ == "__main__": + if "snakemake" not in globals(): + from _helpers import mock_snakemake + + snakemake = mock_snakemake( + "plot_statistics_comparison", + country="all", + carrier="electricity", + ) + configure_logging(snakemake) + + tech_colors = pd.Series(snakemake.params.plotting["tech_colors"]) + + for output in snakemake.output.keys(): + if "touch" in output: + with open(snakemake.output.barplots_touch, "a"): + pass + continue + fig, ax = plt.subplots() + if "energy_balance" not in output: + df = read_csv(snakemake.input, output) + else: + supply = read_csv(snakemake.input, "supply") + withdrawal = read_csv(snakemake.input, "withdrawal") + df = ( + pd.concat([supply, withdrawal.mul(-1)]) + .groupby(["component", "carrier"]) + .sum() + ) + if df.empty: + fig.savefig(snakemake.output[output]) + continue + + plot_static_comparison(df, ax, stacked=STACKED.get(output, True)) + + fig.savefig(snakemake.output[output], bbox_inches="tight") diff --git a/scripts/plot_statistics_single.py b/scripts/plot_statistics_single.py new file mode 100644 index 000000000..dfb709509 --- /dev/null +++ b/scripts/plot_statistics_single.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors +# +# SPDX-License-Identifier: MIT + +import matplotlib.pyplot as plt +import pandas as pd +import seaborn as sns +from _helpers import configure_logging +from plot_summary import rename_techs +from pypsa.statistics import get_carrier + +sns.set_theme("paper", style="whitegrid") + + +def rename_index(ds): + specific = ds.index.map(lambda x: f"{x[1]}\n({x[0]})") + generic = ds.index.get_level_values("carrier") + duplicated = generic.duplicated(keep=False) + index = specific.where(duplicated, generic) + return ds.set_axis(index) + + +def plot_static_per_carrier(ds, ax, drop_zero=True): + ds = ds[ds != 0] + ds = ds.dropna() + c = tech_colors[ds.index.get_level_values("carrier").map(rename_techs)] + ds = ds.pipe(rename_index) + ds.T.plot.barh(color=c.values, ax=ax) + ax.grid(axis="x") + + +def read_csv(input): + try: + df = pd.read_csv(input) + df = df.set_index(["component", "carrier"]).squeeze() + except Exception as e: + print(f"An error occurred reading file {input}: {e}") + df = pd.DataFrame() + return df + + +if __name__ == "__main__": + if "snakemake" not in globals(): + from _helpers import mock_snakemake + + snakemake = mock_snakemake( + "plot_statistics_single", + simpl="", + ll="v1.5", + clusters="5", + opts="", + sector_opts="24h-T-H-B-I-A-dist1", + planning_horizons="2040", + country="all", + carrier="heat", + ) + configure_logging(snakemake) + + tech_colors = pd.Series(snakemake.params.plotting["tech_colors"]) + + for output in snakemake.output.keys(): + if "touch" in output: + with open(snakemake.output.barplots_touch, "a"): + pass + continue + fig, ax = plt.subplots() + if output != "energy_balance": + ds = read_csv(snakemake.input[output]) + else: + supply = read_csv(snakemake.input["supply"]) + withdrawal = read_csv(snakemake.input["withdrawal"]) + ds = pd.concat([supply, withdrawal.mul(-1)]) + if ds.empty: + fig.savefig(snakemake.output[output]) + continue + plot_static_per_carrier(ds, ax) + fig.savefig(snakemake.output[output], bbox_inches="tight") From ba11c2bcceb84518f96fa07c9f31ed6a5cfdf4d1 Mon Sep 17 00:00:00 2001 From: Philipp Glaum Date: Mon, 12 Feb 2024 09:42:39 +0100 Subject: [PATCH 02/18] add release note and documentation --- doc/configtables/plotting.csv | 9 ++++++--- doc/release_notes.rst | 1 + scripts/plot_statistics_comparison.py | 1 + scripts/plot_statistics_single.py | 2 +- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/doc/configtables/plotting.csv b/doc/configtables/plotting.csv index 82fc203c0..3124ce306 100644 --- a/doc/configtables/plotting.csv +++ b/doc/configtables/plotting.csv @@ -1,13 +1,16 @@ ,Unit,Values,Description map,,, -- boundaries,°,"[x1,x2,y1,y2]",Boundaries of the map plots in degrees latitude (y) and longitude (x) -projection,,,, --- name,--,"Valid Cartopy projection name","See https://scitools.org.uk/cartopy/docs/latest/reference/projections.html for list of available projections." +projection,,, +-- name,--,Valid Cartopy projection name,See https://scitools.org.uk/cartopy/docs/latest/reference/projections.html for list of available projections. -- args,--,--,"Other entries under 'projection' are passed as keyword arguments to the projection constructor, e.g. ``central_longitude: 10.``." costs_max,bn Euro,float,Upper y-axis limit in cost bar plots. costs_threshold,bn Euro,float,Threshold below which technologies will not be shown in cost bar plots. energy_max,TWh,float,Upper y-axis limit in energy bar plots. energy_min,TWh,float,Lower y-axis limit in energy bar plots. energy_threshold,TWh,float,Threshold below which technologies will not be shown in energy bar plots. -tech_colors,--,carrier -> HEX colour code,Mapping from network ``carrier`` to a colour (`HEX colour code `_). +countries,--,[str],List of countries you want to plot in your statistics figures. Default is all coutries but also single countries can be defined like ``DE``. +carriers,--,[str],Carrier you want to plot from in your statistics figures. This carrier must be an available bus carrier in your network or substring of a carrier or a defined carrier group. +carrier_groups,--,str -> carrier,Mapping from ``carrier_groups`` to individual bus carriers in the network. nice_names,--,str -> str,Mapping from network ``carrier`` to a more readable name. +tech_colors,--,carrier -> HEX colour code,Mapping from network ``carrier`` to a colour (`HEX colour code `_). diff --git a/doc/release_notes.rst b/doc/release_notes.rst index ee7bd64b5..2a3d731c5 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -10,6 +10,7 @@ Release Notes Upcoming Release ================ +* Add plotting routine with statistics where we allow for plotting of individual energy carriers and countries. Besides the plots, we create all necessary csv files for the plotting routine. * Add new default to overdimension heating in individual buildings. This allows them to cover heat demand peaks e.g. 10% higher than those in the data. The disadvantage of manipulating the costs is that the capacity is then not quite diff --git a/scripts/plot_statistics_comparison.py b/scripts/plot_statistics_comparison.py index 73776dd51..63287c100 100644 --- a/scripts/plot_statistics_comparison.py +++ b/scripts/plot_statistics_comparison.py @@ -54,6 +54,7 @@ def plot_static_comparison(df, ax, stacked=False): def read_csv(input, output): try: + # filter required csv to plot the wanted output files = list(filter(lambda x: output in x, input)) pattern = r"elec_.*?(\d{4})" network_labels = [re.search(pattern, f).group() for f in files] diff --git a/scripts/plot_statistics_single.py b/scripts/plot_statistics_single.py index dfb709509..780fa62cb 100644 --- a/scripts/plot_statistics_single.py +++ b/scripts/plot_statistics_single.py @@ -22,7 +22,7 @@ def rename_index(ds): return ds.set_axis(index) -def plot_static_per_carrier(ds, ax, drop_zero=True): +def plot_static_per_carrier(ds, ax): ds = ds[ds != 0] ds = ds.dropna() c = tech_colors[ds.index.get_level_values("carrier").map(rename_techs)] From 5b2cbcd9abad3e32129d39f5d1b50895b200104f Mon Sep 17 00:00:00 2001 From: Philipp Glaum Date: Mon, 12 Feb 2024 09:46:03 +0100 Subject: [PATCH 03/18] change statistics script name --- rules/postprocess.smk | 8 ++++---- scripts/{make_statistics_csv.py => write_statistics.py} | 0 2 files changed, 4 insertions(+), 4 deletions(-) rename scripts/{make_statistics_csv.py => write_statistics.py} (100%) diff --git a/rules/postprocess.smk b/rules/postprocess.smk index 9d19d4632..c7263a52d 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -45,7 +45,7 @@ if config["foresight"] != "perfect": ( LOGS + "plot_power_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log" - ) + ), benchmark: ( BENCHMARKS @@ -74,7 +74,7 @@ if config["foresight"] != "perfect": ( LOGS + "plot_hydrogen_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log" - ) + ), benchmark: ( BENCHMARKS @@ -102,7 +102,7 @@ if config["foresight"] != "perfect": ( LOGS + "plot_gas_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log" - ) + ), benchmark: ( BENCHMARKS @@ -311,7 +311,7 @@ rule save_statistics_csv: csv_touch=RESULTS + "statistics/csv/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}/country_{country}/.statistics_{carrier}_csv", script: - "../scripts/make_statistics_csv.py" + "../scripts/write_statistics.py" rule plot_statistics_single: diff --git a/scripts/make_statistics_csv.py b/scripts/write_statistics.py similarity index 100% rename from scripts/make_statistics_csv.py rename to scripts/write_statistics.py From 4702f6961db2f6ed6c2d49c576f83307d8773d69 Mon Sep 17 00:00:00 2001 From: Philipp Glaum Date: Mon, 12 Feb 2024 12:08:53 +0100 Subject: [PATCH 04/18] simplify statistics plotting --- rules/postprocess.smk | 23 ++++++----- scripts/plot_statistics_comparison.py | 29 +++++++++----- scripts/plot_statistics_single.py | 10 +---- scripts/write_statistics.py | 58 ++++++++++++++++++--------- 4 files changed, 71 insertions(+), 49 deletions(-) diff --git a/rules/postprocess.smk b/rules/postprocess.smk index c7263a52d..1aca7c49a 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -278,10 +278,9 @@ STATISTICS_BARPLOTS = [ "supply", "withdrawal", "market_value", - "energy_balance", ] -STATISTICS_CSV = [ +STATISTICS = [ "capacity_factor", "installed_capacity", "optimal_capacity", @@ -291,12 +290,14 @@ STATISTICS_CSV = [ "supply", "withdrawal", "market_value", + "energy_balance", + "total_cost", ] rule save_statistics_csv: params: - barplots=STATISTICS_CSV, + barplots=STATISTICS, input: network=RESULTS + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", @@ -306,7 +307,7 @@ rule save_statistics_csv: + "statistics/csv/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}/country_{country}/{carrier}_" + f"{csv}.csv" for carrier in config["plotting"].get("carriers", "all") - for csv in STATISTICS_CSV + for csv in STATISTICS }, csv_touch=RESULTS + "statistics/csv/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}/country_{country}/.statistics_{carrier}_csv", @@ -317,14 +318,14 @@ rule save_statistics_csv: rule plot_statistics_single: params: plotting=config["plotting"], - barplots=STATISTICS_BARPLOTS, + barplots=STATISTICS, input: **{ f"{csv}": RESULTS + "statistics/csv/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}/country_{country}/{carrier}_" + f"{csv}.csv" for carrier in config["plotting"].get("carriers", "all") - for csv in STATISTICS_CSV + for csv in STATISTICS }, output: **{ @@ -332,7 +333,7 @@ rule plot_statistics_single: + "statistics/figures/single/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}/country_{country}/{carrier}_" + f"{plot}.pdf" for carrier in config["plotting"].get("carriers", "all") - for plot in STATISTICS_BARPLOTS + for plot in STATISTICS }, barplots_touch=RESULTS + "statistics/figures/single/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}/country_{country}/.statistics_{carrier}_plots", @@ -343,13 +344,13 @@ rule plot_statistics_single: rule plot_statistics_comparison: params: plotting=config["plotting"], - barplots=STATISTICS_BARPLOTS, + barplots=STATISTICS, input: expand( RESULTS + "statistics/csv/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}/country_{country}/{carrier}_{csv}.csv", **config["scenario"], - csv=STATISTICS_CSV, + csv=STATISTICS, allow_missing=True, ), output: @@ -358,7 +359,7 @@ rule plot_statistics_comparison: + "statistics/figures/comparison/country_{country}/{carrier}_" + f"{plot}.pdf" for carrier in config["plotting"].get("carriers", "all") - for plot in STATISTICS_BARPLOTS + for plot in STATISTICS }, barplots_touch=RESULTS + "statistics/figures/comparison/country_{country}/.statistics_{carrier}_plots", @@ -376,7 +377,7 @@ rule plot_elec_statistics: **{ f"{plot}_bar": RESULTS + f"figures/statistics_{plot}_bar_elec_s{{simpl}}_{{clusters}}_ec_l{{ll}}_{{opts}}.pdf" - for plot in STATISTICS_BARPLOTS + for plot in STATISTICS }, barplots_touch=RESULTS + "figures/.statistics_plots_elec_s{simpl}_{clusters}_ec_l{ll}_{opts}", diff --git a/scripts/plot_statistics_comparison.py b/scripts/plot_statistics_comparison.py index 63287c100..79ea41721 100644 --- a/scripts/plot_statistics_comparison.py +++ b/scripts/plot_statistics_comparison.py @@ -59,7 +59,10 @@ def read_csv(input, output): pattern = r"elec_.*?(\d{4})" network_labels = [re.search(pattern, f).group() for f in files] df = pd.concat( - [pd.read_csv(f).set_index(["component", "carrier"]) for f in files], + [ + pd.read_csv(f, skiprows=2).set_index(["component", "carrier"]) + for f in files + ], axis=1, keys=network_labels, ) @@ -91,16 +94,20 @@ def read_csv(input, output): pass continue fig, ax = plt.subplots() - if "energy_balance" not in output: - df = read_csv(snakemake.input, output) - else: - supply = read_csv(snakemake.input, "supply") - withdrawal = read_csv(snakemake.input, "withdrawal") - df = ( - pd.concat([supply, withdrawal.mul(-1)]) - .groupby(["component", "carrier"]) - .sum() - ) + # if output == "energy_balance": + # supply = read_csv(snakemake.input, "supply") + # withdrawal = read_csv(snakemake.input, "withdrawal") + # df = ( + # pd.concat([supply, withdrawal.mul(-1)]) + # .groupby(["component", "carrier"]) + # .sum() + # ) + # elif output == "total_cost": + # opex = read_csv(snakemake.input, "opex") + # capex = read_csv(snakemake.input, "capex") + # df = opex.add(capex, fill_value=0) + # else: + df = read_csv(snakemake.input, output) if df.empty: fig.savefig(snakemake.output[output]) continue diff --git a/scripts/plot_statistics_single.py b/scripts/plot_statistics_single.py index 780fa62cb..47017b801 100644 --- a/scripts/plot_statistics_single.py +++ b/scripts/plot_statistics_single.py @@ -23,7 +23,6 @@ def rename_index(ds): def plot_static_per_carrier(ds, ax): - ds = ds[ds != 0] ds = ds.dropna() c = tech_colors[ds.index.get_level_values("carrier").map(rename_techs)] ds = ds.pipe(rename_index) @@ -33,7 +32,7 @@ def plot_static_per_carrier(ds, ax): def read_csv(input): try: - df = pd.read_csv(input) + df = pd.read_csv(input, skiprows=2) df = df.set_index(["component", "carrier"]).squeeze() except Exception as e: print(f"An error occurred reading file {input}: {e}") @@ -66,12 +65,7 @@ def read_csv(input): pass continue fig, ax = plt.subplots() - if output != "energy_balance": - ds = read_csv(snakemake.input[output]) - else: - supply = read_csv(snakemake.input["supply"]) - withdrawal = read_csv(snakemake.input["withdrawal"]) - ds = pd.concat([supply, withdrawal.mul(-1)]) + ds = read_csv(snakemake.input[output]) if ds.empty: fig.savefig(snakemake.output[output]) continue diff --git a/scripts/write_statistics.py b/scripts/write_statistics.py index 7dfa84044..b49d607c4 100644 --- a/scripts/write_statistics.py +++ b/scripts/write_statistics.py @@ -21,6 +21,16 @@ def groupby_country_and_carrier(n, c, nice_names=False): return [country, carrier] +def call_with_handle(func, **kwargs): + try: + ds = func(**kwargs) + except Exception as e: + print(f"An error occurred: {e}") + ds = pd.Series() + pass + return ds + + if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake @@ -74,25 +84,35 @@ def groupby_country_and_carrier(n, c, nice_names=False): for output in snakemake.output.keys(): if "touch" in output: continue - if output != "energy_balance": + if output == "energy_balance": + supply = call_with_handle(n.statistics.supply, **kwargs) + withdrawal = call_with_handle(n.statistics.withdrawal, **kwargs) + ds = ( + pd.concat([supply, withdrawal.mul(-1)]) + .groupby(["component", "carrier"]) + .sum() + ) + ds.attrs = supply.attrs + ds.attrs["name"] = "Energy Balance" + elif output == "total_cost": + opex = call_with_handle(n.statistics.opex, **kwargs) + capex = call_with_handle(n.statistics.capex, **kwargs) + ds = opex.add(capex, fill_value=0) + ds.attrs["name"] = "Total Cost" + else: func = eval(f"n.statistics.{output}") - try: - ds = func(**kwargs) - if ds.empty: - print( - f"Empty series for {output} with bus carrier {bus_carrier} and country {country}." - ) - pd.Series().to_csv(snakemake.output[output]) - continue - except Exception as e: - print(f"An error occurred: {e}") - pd.Series().to_csv(snakemake.output[output]) - pass - continue - if country and country != "all": - ds = ds.xs(country, level="country") - ds.name = ds.attrs["name"] - ds.dropna().to_csv(snakemake.output[output]) - # touch file + ds = call_with_handle(func, **kwargs) + + if ds.empty: + print( + f"Empty series for {output} with bus carrier {bus_carrier} and country {country}." + ) + pd.Series().to_csv(snakemake.output[output]) + continue + if country and country != "all": + ds = ds.xs(country, level="country") + pd.Series(ds.attrs).to_csv(snakemake.output[output], header=False) + ds.dropna().to_csv(snakemake.output[output], mode="a") + # touch file with open(snakemake.output.csv_touch, "a"): pass From 0e734fd0072cb425a95cf1a378ad6cb2b84c49d4 Mon Sep 17 00:00:00 2001 From: Philipp Glaum Date: Mon, 12 Feb 2024 13:53:13 +0100 Subject: [PATCH 05/18] improve and clean-up statistic plots --- rules/postprocess.smk | 34 +++++++++++++-------------- scripts/plot_statistics_comparison.py | 21 +++++------------ scripts/plot_statistics_single.py | 10 ++++---- scripts/write_statistics.py | 2 +- 4 files changed, 30 insertions(+), 37 deletions(-) diff --git a/rules/postprocess.smk b/rules/postprocess.smk index 1aca7c49a..f56f79ec6 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -280,24 +280,24 @@ STATISTICS_BARPLOTS = [ "market_value", ] -STATISTICS = [ - "capacity_factor", - "installed_capacity", - "optimal_capacity", - "capex", - "opex", - "curtailment", - "supply", - "withdrawal", - "market_value", - "energy_balance", - "total_cost", -] +STATISTICS = { + "capacity_factor": ("-", "p.u."), + "installed_capacity": (1e3, "GW"), + "optimal_capacity": (1e3, "GW"), + "capex": (1e9, "bn €"), + "opex": (1e9, "bn €"), + "total_cost": ("1e9", "bn €"), + "curtailment": (1e3, "GWh"), + "supply": (1e6, "TWh"), + "withdrawal": (1e6, "TWh"), + "energy_balance": (1e6, "TWh"), + "market_value": ("-", "€/MWh"), +} rule save_statistics_csv: params: - barplots=STATISTICS, + statistics=STATISTICS, input: network=RESULTS + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", @@ -318,7 +318,7 @@ rule save_statistics_csv: rule plot_statistics_single: params: plotting=config["plotting"], - barplots=STATISTICS, + statistics=STATISTICS, input: **{ f"{csv}": RESULTS @@ -344,7 +344,7 @@ rule plot_statistics_single: rule plot_statistics_comparison: params: plotting=config["plotting"], - barplots=STATISTICS, + statistics=STATISTICS, input: expand( RESULTS @@ -377,7 +377,7 @@ rule plot_elec_statistics: **{ f"{plot}_bar": RESULTS + f"figures/statistics_{plot}_bar_elec_s{{simpl}}_{{clusters}}_ec_l{{ll}}_{{opts}}.pdf" - for plot in STATISTICS + for plot in STATISTICS_BARPLOTS }, barplots_touch=RESULTS + "figures/.statistics_plots_elec_s{simpl}_{clusters}_ec_l{ll}_{opts}", diff --git a/scripts/plot_statistics_comparison.py b/scripts/plot_statistics_comparison.py index 79ea41721..ce20340b7 100644 --- a/scripts/plot_statistics_comparison.py +++ b/scripts/plot_statistics_comparison.py @@ -7,7 +7,6 @@ import re import matplotlib.pyplot as plt -import matplotlib.ticker as ticker import pandas as pd import seaborn as sns from _helpers import configure_logging @@ -40,11 +39,15 @@ def rename_index(df): def plot_static_comparison(df, ax, stacked=False): + factor, unit = conversion[output] df = df[df != 0] df = df.dropna(axis=0, how="all").fillna(0) + if df.empty: + return c = tech_colors[df.index.get_level_values("carrier").map(rename_techs)] df = df.pipe(rename_index).T - df.plot.bar(color=c.values, ax=ax, stacked=stacked, legend=False) + df = df.div(float(factor)) if factor != "-" else df + df.plot.bar(color=c.values, ax=ax, stacked=stacked, legend=False, ylabel=unit) ax.legend( bbox_to_anchor=(1, 1), loc="upper left", @@ -87,6 +90,7 @@ def read_csv(input, output): configure_logging(snakemake) tech_colors = pd.Series(snakemake.params.plotting["tech_colors"]) + conversion = pd.Series(snakemake.params.statistics) for output in snakemake.output.keys(): if "touch" in output: @@ -94,19 +98,6 @@ def read_csv(input, output): pass continue fig, ax = plt.subplots() - # if output == "energy_balance": - # supply = read_csv(snakemake.input, "supply") - # withdrawal = read_csv(snakemake.input, "withdrawal") - # df = ( - # pd.concat([supply, withdrawal.mul(-1)]) - # .groupby(["component", "carrier"]) - # .sum() - # ) - # elif output == "total_cost": - # opex = read_csv(snakemake.input, "opex") - # capex = read_csv(snakemake.input, "capex") - # df = opex.add(capex, fill_value=0) - # else: df = read_csv(snakemake.input, output) if df.empty: fig.savefig(snakemake.output[output]) diff --git a/scripts/plot_statistics_single.py b/scripts/plot_statistics_single.py index 47017b801..22631f096 100644 --- a/scripts/plot_statistics_single.py +++ b/scripts/plot_statistics_single.py @@ -9,7 +9,6 @@ import seaborn as sns from _helpers import configure_logging from plot_summary import rename_techs -from pypsa.statistics import get_carrier sns.set_theme("paper", style="whitegrid") @@ -22,11 +21,13 @@ def rename_index(ds): return ds.set_axis(index) -def plot_static_per_carrier(ds, ax): +def plot_static_single(ds, ax): + factor, unit = conversion[output] ds = ds.dropna() c = tech_colors[ds.index.get_level_values("carrier").map(rename_techs)] ds = ds.pipe(rename_index) - ds.T.plot.barh(color=c.values, ax=ax) + ds = ds.div(float(factor)) if factor != "-" else ds + ds.T.plot.barh(color=c.values, ax=ax, ylabel=unit) ax.grid(axis="x") @@ -58,6 +59,7 @@ def read_csv(input): configure_logging(snakemake) tech_colors = pd.Series(snakemake.params.plotting["tech_colors"]) + conversion = pd.Series(snakemake.params.statistics) for output in snakemake.output.keys(): if "touch" in output: @@ -69,5 +71,5 @@ def read_csv(input): if ds.empty: fig.savefig(snakemake.output[output]) continue - plot_static_per_carrier(ds, ax) + plot_static_single(ds, ax) fig.savefig(snakemake.output[output], bbox_inches="tight") diff --git a/scripts/write_statistics.py b/scripts/write_statistics.py index b49d607c4..98d9a5c3a 100644 --- a/scripts/write_statistics.py +++ b/scripts/write_statistics.py @@ -89,7 +89,7 @@ def call_with_handle(func, **kwargs): withdrawal = call_with_handle(n.statistics.withdrawal, **kwargs) ds = ( pd.concat([supply, withdrawal.mul(-1)]) - .groupby(["component", "carrier"]) + .groupby(supply.index.names) .sum() ) ds.attrs = supply.attrs From b3ab9bbd84ac203516af290096d191fcafed3c52 Mon Sep 17 00:00:00 2001 From: Philipp Glaum Date: Mon, 12 Feb 2024 14:44:39 +0100 Subject: [PATCH 06/18] revert changes in STATISTICS_BARPLOTS --- rules/postprocess.smk | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rules/postprocess.smk b/rules/postprocess.smk index f56f79ec6..cf49209a3 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -272,8 +272,8 @@ STATISTICS_BARPLOTS = [ "capacity_factor", "installed_capacity", "optimal_capacity", - "capex", - "opex", + "capital_expenditure", + "operational_expenditure", "curtailment", "supply", "withdrawal", From e7b5a0b20bebbc54612bd62bd54228c7ed32b87e Mon Sep 17 00:00:00 2001 From: Philipp Glaum Date: Thu, 15 Feb 2024 17:09:29 +0100 Subject: [PATCH 07/18] plot_statistics_single: fix read csv file when file had only single entry, adjust axis label --- scripts/plot_statistics_single.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/plot_statistics_single.py b/scripts/plot_statistics_single.py index 22631f096..97491f58e 100644 --- a/scripts/plot_statistics_single.py +++ b/scripts/plot_statistics_single.py @@ -27,14 +27,14 @@ def plot_static_single(ds, ax): c = tech_colors[ds.index.get_level_values("carrier").map(rename_techs)] ds = ds.pipe(rename_index) ds = ds.div(float(factor)) if factor != "-" else ds - ds.T.plot.barh(color=c.values, ax=ax, ylabel=unit) - ax.grid(axis="x") + ds.T.plot.barh(color=c.values, ax=ax, xlabel=unit) + ax.grid(axis="y") def read_csv(input): try: df = pd.read_csv(input, skiprows=2) - df = df.set_index(["component", "carrier"]).squeeze() + df = df.set_index(["component", "carrier"]).iloc[:, 0] except Exception as e: print(f"An error occurred reading file {input}: {e}") df = pd.DataFrame() From 962d7221fe929ca53909ef548e08fba8500e1801 Mon Sep 17 00:00:00 2001 From: Philipp Glaum Date: Fri, 16 Feb 2024 12:45:59 +0100 Subject: [PATCH 08/18] plotting scipts: fill carriers with not defined colors with default color --- scripts/plot_statistics_comparison.py | 7 ++++++- scripts/plot_statistics_single.py | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/scripts/plot_statistics_comparison.py b/scripts/plot_statistics_comparison.py index ce20340b7..8222303d7 100644 --- a/scripts/plot_statistics_comparison.py +++ b/scripts/plot_statistics_comparison.py @@ -44,7 +44,12 @@ def plot_static_comparison(df, ax, stacked=False): df = df.dropna(axis=0, how="all").fillna(0) if df.empty: return - c = tech_colors[df.index.get_level_values("carrier").map(rename_techs)] + carriers = df.index.get_level_values("carrier").map(rename_techs) + if not carriers.difference(tech_colors.index).empty: + print( + f"Missing colors for carrier: {carriers.difference(tech_colors.index).values}\n Dark grey used instead." + ) + c = carriers.map(lambda x: tech_colors.get(x, "#808080")) df = df.pipe(rename_index).T df = df.div(float(factor)) if factor != "-" else df df.plot.bar(color=c.values, ax=ax, stacked=stacked, legend=False, ylabel=unit) diff --git a/scripts/plot_statistics_single.py b/scripts/plot_statistics_single.py index 97491f58e..ee1fa4ede 100644 --- a/scripts/plot_statistics_single.py +++ b/scripts/plot_statistics_single.py @@ -24,7 +24,12 @@ def rename_index(ds): def plot_static_single(ds, ax): factor, unit = conversion[output] ds = ds.dropna() - c = tech_colors[ds.index.get_level_values("carrier").map(rename_techs)] + carriers = ds.index.get_level_values("carrier").map(rename_techs) + if not carriers.difference(tech_colors.index).empty: + print( + f"Missing colors for carrier: {carriers.difference(tech_colors.index).values}\n Dark grey used instead." + ) + c = carriers.map(lambda x: tech_colors.get(x, "#808080")) ds = ds.pipe(rename_index) ds = ds.div(float(factor)) if factor != "-" else ds ds.T.plot.barh(color=c.values, ax=ax, xlabel=unit) From 29f1ca7c6723713bb946e285e55c1b50c842ecd3 Mon Sep 17 00:00:00 2001 From: Philipp Glaum Date: Mon, 19 Feb 2024 16:55:52 +0100 Subject: [PATCH 09/18] improve country plotting --- scripts/plot_statistics_comparison.py | 9 +++++++-- scripts/write_statistics.py | 23 +++++++++++++++-------- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/scripts/plot_statistics_comparison.py b/scripts/plot_statistics_comparison.py index 8222303d7..55da50ec1 100644 --- a/scripts/plot_statistics_comparison.py +++ b/scripts/plot_statistics_comparison.py @@ -44,7 +44,8 @@ def plot_static_comparison(df, ax, stacked=False): df = df.dropna(axis=0, how="all").fillna(0) if df.empty: return - carriers = df.index.get_level_values("carrier").map(rename_techs) + df = df.rename(index=rename_techs).groupby(["component", "carrier"]).sum() + carriers = df.index.get_level_values("carrier") if not carriers.difference(tech_colors.index).empty: print( f"Missing colors for carrier: {carriers.difference(tech_colors.index).values}\n Dark grey used instead." @@ -94,7 +95,11 @@ def read_csv(input, output): ) configure_logging(snakemake) - tech_colors = pd.Series(snakemake.params.plotting["tech_colors"]) + tech_colors = ( + pd.Series(snakemake.params.plotting["tech_colors"]) + .groupby(rename_techs) + .first() + ) conversion = pd.Series(snakemake.params.statistics) for output in snakemake.output.keys(): diff --git a/scripts/write_statistics.py b/scripts/write_statistics.py index 98d9a5c3a..8fe13bed6 100644 --- a/scripts/write_statistics.py +++ b/scripts/write_statistics.py @@ -12,11 +12,19 @@ from pypsa.statistics import get_carrier +def get_country(df): + country_map = df.filter(like="bus").apply( + lambda ds: ds.map(n.buses.location.map(n.buses.country)) + ) + country_map = country_map.apply(lambda x: ",".join(x.dropna().unique()), axis=1) + country_map = country_map.rename("country") + return country_map + + # grouperfunctions = hier schreiben und dann in statistics. def groupby_country_and_carrier(n, c, nice_names=False): df = n.df(c) - bus = "bus1" if "bus" not in n.df(c) else "bus" - country = df[bus].map(n.buses.location).map(n.buses.country).rename("country") + country = df.pipe(get_country) carrier = get_carrier(n, c, nice_names) return [country, carrier] @@ -76,10 +84,7 @@ def call_with_handle(func, **kwargs): f"Carrier {carrier} not found in network or carrier group in config." ) - if country == "all" or not country: - kwargs["groupby"] = get_carrier - else: - kwargs["groupby"] = groupby_country_and_carrier + kwargs["groupby"] = groupby_country_and_carrier for output in snakemake.output.keys(): if "touch" in output: @@ -89,7 +94,7 @@ def call_with_handle(func, **kwargs): withdrawal = call_with_handle(n.statistics.withdrawal, **kwargs) ds = ( pd.concat([supply, withdrawal.mul(-1)]) - .groupby(supply.index.names) + .groupby(level=["component", "country", "carrier"]) .sum() ) ds.attrs = supply.attrs @@ -110,7 +115,9 @@ def call_with_handle(func, **kwargs): pd.Series().to_csv(snakemake.output[output]) continue if country and country != "all": - ds = ds.xs(country, level="country") + mask = ds.index.get_level_values("country").str.contains(country) + ds = ds[mask] + ds = ds.groupby(level=["component", "carrier"]).sum() pd.Series(ds.attrs).to_csv(snakemake.output[output], header=False) ds.dropna().to_csv(snakemake.output[output], mode="a") # touch file From 547f9a887a3496dd7729f03e7c095ee855ab4092 Mon Sep 17 00:00:00 2001 From: Philipp Glaum Date: Fri, 15 Mar 2024 10:26:31 +0100 Subject: [PATCH 10/18] make statistics plot scripts compatible with scenario management --- rules/collect.smk | 2 ++ rules/postprocess.smk | 9 +++++++++ scripts/plot_statistics_comparison.py | 2 ++ scripts/plot_statistics_single.py | 3 +++ 4 files changed, 16 insertions(+) diff --git a/rules/collect.smk b/rules/collect.smk index dfe022b14..a5529820d 100644 --- a/rules/collect.smk +++ b/rules/collect.smk @@ -104,6 +104,7 @@ rule plot_statistics: + "statistics/figures/comparison/country_{country}/.statistics_{carrier}_plots", country=config["plotting"].get("countries", "all"), carrier=config["plotting"].get("carriers", ["all"]), + run=config["run"]["name"], ), expand( RESULTS @@ -111,5 +112,6 @@ rule plot_statistics: **config["scenario"], country=config["plotting"].get("countries", "all"), carrier=config["plotting"].get("carriers", ["all"]), + run=config["run"]["name"], ), ], diff --git a/rules/postprocess.smk b/rules/postprocess.smk index 8a00c82df..35082e687 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -306,6 +306,9 @@ rule save_statistics_csv: }, csv_touch=RESULTS + "statistics/csv/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}/country_{country}/.statistics_{carrier}_csv", + log: + RESULTS + + "logs/save_statistics_csv/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_country-{country}_carrier-{carrier}.log", script: "../scripts/write_statistics.py" @@ -332,6 +335,9 @@ rule plot_statistics_single: }, barplots_touch=RESULTS + "statistics/figures/single/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}/country_{country}/.statistics_{carrier}_plots", + log: + RESULTS + + "logs/plot_statistics_single/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_country-{country}_carrier-{carrier}.log", script: "../scripts/plot_statistics_single.py" @@ -358,6 +364,9 @@ rule plot_statistics_comparison: }, barplots_touch=RESULTS + "statistics/figures/comparison/country_{country}/.statistics_{carrier}_plots", + log: + RESULTS + + "logs/plot_statistics_comparison/country-{country}_carrier-{carrier}.log", script: "../scripts/plot_statistics_comparison.py" diff --git a/scripts/plot_statistics_comparison.py b/scripts/plot_statistics_comparison.py index 55da50ec1..6c8885ae2 100644 --- a/scripts/plot_statistics_comparison.py +++ b/scripts/plot_statistics_comparison.py @@ -4,6 +4,7 @@ # # SPDX-License-Identifier: MIT +import logging import re import matplotlib.pyplot as plt @@ -12,6 +13,7 @@ from _helpers import configure_logging from plot_summary import rename_techs +logger = logging.getLogger(__name__) sns.set_theme("paper", style="whitegrid") STACKED = { "capacity_factor": False, diff --git a/scripts/plot_statistics_single.py b/scripts/plot_statistics_single.py index ee1fa4ede..3a2a24ca2 100644 --- a/scripts/plot_statistics_single.py +++ b/scripts/plot_statistics_single.py @@ -4,12 +4,15 @@ # # SPDX-License-Identifier: MIT +import logging + import matplotlib.pyplot as plt import pandas as pd import seaborn as sns from _helpers import configure_logging from plot_summary import rename_techs +logger = logging.getLogger(__name__) sns.set_theme("paper", style="whitegrid") From e782727e4dbd449cc1a679141235847440c5b178 Mon Sep 17 00:00:00 2001 From: Philipp Glaum Date: Fri, 10 May 2024 11:45:59 +0200 Subject: [PATCH 11/18] update write_statisitcs --- scripts/write_statistics.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/scripts/write_statistics.py b/scripts/write_statistics.py index 8fe13bed6..059570b73 100644 --- a/scripts/write_statistics.py +++ b/scripts/write_statistics.py @@ -8,7 +8,11 @@ import pandas as pd import pypsa import seaborn as sns -from _helpers import configure_logging +from _helpers import ( + configure_logging, + set_scenario_config, + update_config_from_wildcards, +) from pypsa.statistics import get_carrier @@ -47,14 +51,16 @@ def call_with_handle(func, **kwargs): "save_statistics_csv", simpl="", ll="v1.5", - clusters="5", + clusters="37", opts="", - sector_opts="24h-T-H-B-I-A-dist1", - planning_horizons="2040", + sector_opts="", + planning_horizons="2050", country="all", carrier="electricity", ) - # configure_logging(snakemake) + configure_logging(snakemake) + set_scenario_config(snakemake) + update_config_from_wildcards(snakemake.config, snakemake.wildcards) config = snakemake.config n = pypsa.Network(snakemake.input.network) From d77bd1bb316bd6cd82b071e38ab88ddaa6d67f2f Mon Sep 17 00:00:00 2001 From: Philipp Glaum Date: Thu, 20 Jun 2024 16:28:09 +0200 Subject: [PATCH 12/18] update statistic plots --- config/config.default.yaml | 15 ++++---- rules/collect.smk | 8 ++-- rules/postprocess.smk | 22 +++++------ scripts/plot_statistics_comparison.py | 39 ++++++++++++++----- scripts/plot_statistics_single.py | 13 ++++++- scripts/write_statistics.py | 55 ++++++++++----------------- 6 files changed, 86 insertions(+), 66 deletions(-) diff --git a/config/config.default.yaml b/config/config.default.yaml index 2c156b0a1..e3b21e1da 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -867,13 +867,14 @@ plotting: energy_max: 20000 energy_min: -20000 energy_threshold: 50. - countries: - - all - carriers: - - electricity - - heat - carrier_groups: - electricity: [AC, low_voltage] + statistics: + countries: + - all + carriers: + - electricity + - heat + carrier_groups: + electricity: [AC, low_voltage] nice_names: OCGT: "Open-Cycle Gas" diff --git a/rules/collect.smk b/rules/collect.smk index a5529820d..5051df472 100644 --- a/rules/collect.smk +++ b/rules/collect.smk @@ -102,16 +102,16 @@ rule plot_statistics: expand( RESULTS + "statistics/figures/comparison/country_{country}/.statistics_{carrier}_plots", - country=config["plotting"].get("countries", "all"), - carrier=config["plotting"].get("carriers", ["all"]), + country=config["plotting"]["statistics"].get("countries", "all"), + carrier=config["plotting"]["statistics"].get("carriers", ["all"]), run=config["run"]["name"], ), expand( RESULTS + "statistics/figures/single/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}/country_{country}/.statistics_{carrier}_plots", **config["scenario"], - country=config["plotting"].get("countries", "all"), - carrier=config["plotting"].get("carriers", ["all"]), + country=config["plotting"]["statistics"].get("countries", "all"), + carrier=config["plotting"]["statistics"].get("carriers", ["all"]), run=config["run"]["name"], ), ], diff --git a/rules/postprocess.smk b/rules/postprocess.smk index 38af0c6da..1c7607c1d 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -274,7 +274,7 @@ STATISTICS = { } -rule save_statistics_csv: +rule write_statistics: params: statistics=STATISTICS, input: @@ -282,17 +282,17 @@ rule save_statistics_csv: + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", output: **{ - f"{csv}": RESULTS + f"{statistic}": RESULTS + "statistics/csv/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}/country_{country}/{carrier}_" - + f"{csv}.csv" - for carrier in config["plotting"].get("carriers", "all") - for csv in STATISTICS + + f"{statistic}.csv" + for carrier in config["plotting"]["statistics"].get("carriers", "all") + for statistic in STATISTICS }, csv_touch=RESULTS + "statistics/csv/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}/country_{country}/.statistics_{carrier}_csv", log: RESULTS - + "logs/save_statistics_csv/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_country-{country}_carrier-{carrier}.log", + + "logs/write_statistics/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_country-{country}_carrier-{carrier}.log", script: "../scripts/write_statistics.py" @@ -303,18 +303,18 @@ rule plot_statistics_single: statistics=STATISTICS, input: **{ - f"{csv}": RESULTS + f"{statistic}": RESULTS + "statistics/csv/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}/country_{country}/{carrier}_" - + f"{csv}.csv" - for carrier in config["plotting"].get("carriers", "all") - for csv in STATISTICS + + f"{statistic}.csv" + for carrier in config["plotting"]["statistics"].get("carriers", "all") + for statistic in STATISTICS }, output: **{ f"{plot}": RESULTS + "statistics/figures/single/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}/country_{country}/{carrier}_" + f"{plot}.pdf" - for carrier in config["plotting"].get("carriers", "all") + for carrier in config["plotting"]["statistics"].get("carriers", "all") for plot in STATISTICS }, barplots_touch=RESULTS diff --git a/scripts/plot_statistics_comparison.py b/scripts/plot_statistics_comparison.py index 6c8885ae2..ac3dd0040 100644 --- a/scripts/plot_statistics_comparison.py +++ b/scripts/plot_statistics_comparison.py @@ -40,22 +40,31 @@ def rename_index(df): return df +def get_carrier_colors(carriers, tech_colors): + if not carriers.difference(tech_colors.index).empty: + print( + f"Missing colors for carrier: {carriers.difference(tech_colors.index).values}\n Dark grey used instead." + ) + carrier_colors = carriers.map(lambda x: tech_colors.get(x, "#808080")).values + return carrier_colors + + def plot_static_comparison(df, ax, stacked=False): factor, unit = conversion[output] df = df[df != 0] df = df.dropna(axis=0, how="all").fillna(0) if df.empty: return + df = df.div(float(factor)) if factor != "-" else df df = df.rename(index=rename_techs).groupby(["component", "carrier"]).sum() + # sort values in descending order + df = df.reindex(df.sum(1).sort_values().index) carriers = df.index.get_level_values("carrier") - if not carriers.difference(tech_colors.index).empty: - print( - f"Missing colors for carrier: {carriers.difference(tech_colors.index).values}\n Dark grey used instead." - ) - c = carriers.map(lambda x: tech_colors.get(x, "#808080")) + carrier_colors = get_carrier_colors(carriers, tech_colors) df = df.pipe(rename_index).T - df = df.div(float(factor)) if factor != "-" else df - df.plot.bar(color=c.values, ax=ax, stacked=stacked, legend=False, ylabel=unit) + # max_carrier_value = df.max(axis=1) + # df.div(max_carrier_value, axis=0).where(lambda x: abs(x)<0.05).all() + df.plot.bar(color=carrier_colors, ax=ax, stacked=stacked, legend=False, ylabel=unit) ax.legend( bbox_to_anchor=(1, 1), loc="upper left", @@ -67,8 +76,8 @@ def read_csv(input, output): try: # filter required csv to plot the wanted output files = list(filter(lambda x: output in x, input)) - pattern = r"elec_.*?(\d{4})" - network_labels = [re.search(pattern, f).group() for f in files] + # retrieves network labels from folder name + network_labels = [file.split("/")[-3] for file in files] df = pd.concat( [ pd.read_csv(f, skiprows=2).set_index(["component", "carrier"]) @@ -92,9 +101,11 @@ def read_csv(input, output): snakemake = mock_snakemake( "plot_statistics_comparison", + run="", country="all", carrier="electricity", ) + configure_logging(snakemake) tech_colors = ( @@ -112,6 +123,16 @@ def read_csv(input, output): fig, ax = plt.subplots() df = read_csv(snakemake.input, output) if df.empty: + ax.text( + 0.5, + 0.5, + "No data available.", + ha="center", + va="center", + transform=ax.transAxes, + fontsize=14, + color="red", + ) fig.savefig(snakemake.output[output]) continue diff --git a/scripts/plot_statistics_single.py b/scripts/plot_statistics_single.py index 3a2a24ca2..f79536913 100644 --- a/scripts/plot_statistics_single.py +++ b/scripts/plot_statistics_single.py @@ -55,6 +55,7 @@ def read_csv(input): snakemake = mock_snakemake( "plot_statistics_single", + run="", simpl="", ll="v1.5", clusters="5", @@ -77,7 +78,17 @@ def read_csv(input): fig, ax = plt.subplots() ds = read_csv(snakemake.input[output]) if ds.empty: - fig.savefig(snakemake.output[output]) + ax.text( + 0.5, + 0.5, + "No data available.", + ha="center", + va="center", + transform=ax.transAxes, + fontsize=18, + color="red", + ) + fig.savefig(snakemake.output[output], bbox_inches="tight") continue plot_static_single(ds, ax) fig.savefig(snakemake.output[output], bbox_inches="tight") diff --git a/scripts/write_statistics.py b/scripts/write_statistics.py index 059570b73..86e6a98ca 100644 --- a/scripts/write_statistics.py +++ b/scripts/write_statistics.py @@ -4,6 +4,8 @@ # # SPDX-License-Identifier: MIT +import logging + import matplotlib.pyplot as plt import pandas as pd import pypsa @@ -13,31 +15,16 @@ set_scenario_config, update_config_from_wildcards, ) -from pypsa.statistics import get_carrier - - -def get_country(df): - country_map = df.filter(like="bus").apply( - lambda ds: ds.map(n.buses.location.map(n.buses.country)) - ) - country_map = country_map.apply(lambda x: ",".join(x.dropna().unique()), axis=1) - country_map = country_map.rename("country") - return country_map +from pypsa.statistics import get_carrier, get_country_and_carrier - -# grouperfunctions = hier schreiben und dann in statistics. -def groupby_country_and_carrier(n, c, nice_names=False): - df = n.df(c) - country = df.pipe(get_country) - carrier = get_carrier(n, c, nice_names) - return [country, carrier] +logger = logging.getLogger(__name__) def call_with_handle(func, **kwargs): try: ds = func(**kwargs) except Exception as e: - print(f"An error occurred: {e}") + logging.info(f"An error occurred: {e}") ds = pd.Series() pass return ds @@ -48,7 +35,8 @@ def call_with_handle(func, **kwargs): from _helpers import mock_snakemake snakemake = mock_snakemake( - "save_statistics_csv", + "write_statistics", + run="", simpl="", ll="v1.5", clusters="37", @@ -64,7 +52,6 @@ def call_with_handle(func, **kwargs): config = snakemake.config n = pypsa.Network(snakemake.input.network) - kwargs = {"nice_names": False} wildcards = dict(snakemake.wildcards) @@ -73,8 +60,8 @@ def call_with_handle(func, **kwargs): if carrier == "all": pass - elif carrier in config["plotting"].get("carrier_groups", []): - bus_carrier = config["plotting"]["carrier_groups"][carrier] + elif carrier in config["plotting"]["statistics"].get("carrier_groups", []): + bus_carrier = config["plotting"]["statistics"]["carrier_groups"][carrier] kwargs["bus_carrier"] = bus_carrier elif n.buses.carrier.str.contains(carrier).any(): if carrier not in n.buses.carrier.unique(): @@ -90,21 +77,21 @@ def call_with_handle(func, **kwargs): f"Carrier {carrier} not found in network or carrier group in config." ) - kwargs["groupby"] = groupby_country_and_carrier + kwargs["groupby"] = get_country_and_carrier for output in snakemake.output.keys(): if "touch" in output: continue - if output == "energy_balance": - supply = call_with_handle(n.statistics.supply, **kwargs) - withdrawal = call_with_handle(n.statistics.withdrawal, **kwargs) - ds = ( - pd.concat([supply, withdrawal.mul(-1)]) - .groupby(level=["component", "country", "carrier"]) - .sum() - ) - ds.attrs = supply.attrs - ds.attrs["name"] = "Energy Balance" + # if output == "energy_balance": + # supply = call_with_handle(n.statistics.supply, **kwargs) + # withdrawal = call_with_handle(n.statistics.withdrawal, **kwargs) + # ds = ( + # pd.concat([supply, withdrawal.mul(-1)]) + # .groupby(level=["component", "country", "carrier"]) + # .sum() + # ) + # ds.attrs = supply.attrs + # ds.attrs["name"] = "Energy Balance" elif output == "total_cost": opex = call_with_handle(n.statistics.opex, **kwargs) capex = call_with_handle(n.statistics.capex, **kwargs) @@ -115,7 +102,7 @@ def call_with_handle(func, **kwargs): ds = call_with_handle(func, **kwargs) if ds.empty: - print( + logging.info( f"Empty series for {output} with bus carrier {bus_carrier} and country {country}." ) pd.Series().to_csv(snakemake.output[output]) From dda6bafb924375d6ece73e3dc09525b86e4b2cb7 Mon Sep 17 00:00:00 2001 From: Philipp Glaum Date: Mon, 8 Jul 2024 16:01:33 +0200 Subject: [PATCH 13/18] minor updates and replace energy balance by new statistics implementation --- scripts/plot_statistics_comparison.py | 13 +++++++++---- scripts/write_statistics.py | 10 ---------- 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/scripts/plot_statistics_comparison.py b/scripts/plot_statistics_comparison.py index ac3dd0040..94501cc56 100644 --- a/scripts/plot_statistics_comparison.py +++ b/scripts/plot_statistics_comparison.py @@ -58,13 +58,18 @@ def plot_static_comparison(df, ax, stacked=False): df = df.div(float(factor)) if factor != "-" else df df = df.rename(index=rename_techs).groupby(["component", "carrier"]).sum() # sort values in descending order - df = df.reindex(df.sum(1).sort_values().index) + df = df.reindex(df.abs().sum(1).sort_values().index) carriers = df.index.get_level_values("carrier") carrier_colors = get_carrier_colors(carriers, tech_colors) df = df.pipe(rename_index).T - # max_carrier_value = df.max(axis=1) - # df.div(max_carrier_value, axis=0).where(lambda x: abs(x)<0.05).all() - df.plot.bar(color=carrier_colors, ax=ax, stacked=stacked, legend=False, ylabel=unit) + df.plot.bar( + color=carrier_colors, + ax=ax, + stacked=stacked, + legend=False, + ylabel=unit, + linewidth=0.1, + ) ax.legend( bbox_to_anchor=(1, 1), loc="upper left", diff --git a/scripts/write_statistics.py b/scripts/write_statistics.py index 86e6a98ca..b64c641ff 100644 --- a/scripts/write_statistics.py +++ b/scripts/write_statistics.py @@ -82,16 +82,6 @@ def call_with_handle(func, **kwargs): for output in snakemake.output.keys(): if "touch" in output: continue - # if output == "energy_balance": - # supply = call_with_handle(n.statistics.supply, **kwargs) - # withdrawal = call_with_handle(n.statistics.withdrawal, **kwargs) - # ds = ( - # pd.concat([supply, withdrawal.mul(-1)]) - # .groupby(level=["component", "country", "carrier"]) - # .sum() - # ) - # ds.attrs = supply.attrs - # ds.attrs["name"] = "Energy Balance" elif output == "total_cost": opex = call_with_handle(n.statistics.opex, **kwargs) capex = call_with_handle(n.statistics.capex, **kwargs) From 683ea19afa1d18a0d9255423a5eb876ccb9ab98c Mon Sep 17 00:00:00 2001 From: Philipp Glaum Date: Mon, 12 Aug 2024 09:08:22 +0200 Subject: [PATCH 14/18] add option to make a scenario comparison plot --- config/config.default.yaml | 7 ++ rules/collect.smk | 32 +++++++- rules/postprocess.smk | 108 +++++++++++++++++++++----- scripts/plot_statistics_comparison.py | 52 ++++++++----- 4 files changed, 155 insertions(+), 44 deletions(-) diff --git a/config/config.default.yaml b/config/config.default.yaml index fd8871ba3..fa5ae2e89 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -954,8 +954,15 @@ plotting: carriers: - electricity - heat + - co2 carrier_groups: electricity: [AC, low_voltage] + metrics: + - energy_balance + - total_cost + scenario_comparison: + - "" + comparison_folder: "scenario_comparison" nice_names: OCGT: "Open-Cycle Gas" diff --git a/rules/collect.smk b/rules/collect.smk index 5051df472..9b38f873a 100644 --- a/rules/collect.smk +++ b/rules/collect.smk @@ -102,16 +102,40 @@ rule plot_statistics: expand( RESULTS + "statistics/figures/comparison/country_{country}/.statistics_{carrier}_plots", - country=config["plotting"]["statistics"].get("countries", "all"), - carrier=config["plotting"]["statistics"].get("carriers", ["all"]), + country=config_provider("plotting", "statistics")(run).get( + "countries", "all" + ), + carrier=config_provider("plotting", "statistics")(run).get( + "carriers", "all" + ), run=config["run"]["name"], ), expand( RESULTS + "statistics/figures/single/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}/country_{country}/.statistics_{carrier}_plots", **config["scenario"], - country=config["plotting"]["statistics"].get("countries", "all"), - carrier=config["plotting"]["statistics"].get("carriers", ["all"]), + country=config_provider("plotting", "statistics")(run).get( + "countries", "all" + ), + carrier=config_provider("plotting", "statistics")(run).get( + "carriers", "all" + ), + run=config["run"]["name"], + ), + expand( + "results/statistics/" + + config_provider("plotting", "statistics")(run).get( + "comparison_folder", "results/scenario_comparison" + ) + + "/" + + "figures/country_{country}/.statistics_{carrier}_plots", + **config["scenario"], + country=config_provider("plotting", "statistics")(run).get( + "countries", "all" + ), + carrier=config_provider("plotting", "statistics")(run).get( + "carriers", "all" + ), run=config["run"]["name"], ), ], diff --git a/rules/postprocess.smk b/rules/postprocess.smk index e6672aaf0..ffc771032 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -282,11 +282,15 @@ rule write_statistics: + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", output: **{ - f"{statistic}": RESULTS + f"{metric}": RESULTS + "statistics/csv/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}/country_{country}/{carrier}_" - + f"{statistic}.csv" - for carrier in config["plotting"]["statistics"].get("carriers", "all") - for statistic in STATISTICS + + f"{metric}.csv" + for carrier in config_provider("plotting", "statistics")(run).get( + "carriers", "all" + ) + for metric in config_provider("plotting", "statistics")(run).get( + "metrics", STATISTICS + ) }, csv_touch=RESULTS + "statistics/csv/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}/country_{country}/.statistics_{carrier}_csv", @@ -299,23 +303,31 @@ rule write_statistics: rule plot_statistics_single: params: - plotting=config["plotting"], + plotting=config_provider("plotting"), statistics=STATISTICS, input: **{ - f"{statistic}": RESULTS + f"{metric}": RESULTS + "statistics/csv/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}/country_{country}/{carrier}_" - + f"{statistic}.csv" - for carrier in config["plotting"]["statistics"].get("carriers", "all") - for statistic in STATISTICS + + f"{metric}.csv" + for carrier in config_provider("plotting", "statistics")(run).get( + "carriers", "all" + ) + for metric in config_provider("plotting", "statistics")(run).get( + "metrics", STATISTICS + ) }, output: **{ - f"{plot}": RESULTS + f"{metric}": RESULTS + "statistics/figures/single/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}/country_{country}/{carrier}_" - + f"{plot}.pdf" - for carrier in config["plotting"]["statistics"].get("carriers", "all") - for plot in STATISTICS + + f"{metric}.pdf" + for carrier in config_provider("plotting", "statistics")(run).get( + "carriers", "all" + ) + for metric in config_provider("plotting", "statistics")(run).get( + "metrics", STATISTICS + ) }, barplots_touch=RESULTS + "statistics/figures/single/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}/country_{country}/.statistics_{carrier}_plots", @@ -328,23 +340,29 @@ rule plot_statistics_single: rule plot_statistics_comparison: params: - plotting=config["plotting"], + plotting=config_provider("plotting"), statistics=STATISTICS, input: expand( RESULTS - + "statistics/csv/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}/country_{country}/{carrier}_{csv}.csv", + + "statistics/csv/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}/country_{country}/{carrier}_{metric}.csv", **config["scenario"], - csv=STATISTICS, + metric=config_provider("plotting", "statistics")(run).get( + "metrics", STATISTICS + ), allow_missing=True, ), output: **{ - f"{plot}": RESULTS + f"{metric}": RESULTS + "statistics/figures/comparison/country_{country}/{carrier}_" - + f"{plot}.pdf" - for carrier in config["plotting"].get("carriers", "all") - for plot in STATISTICS + + f"{metric}.pdf" + for carrier in config_provider("plotting", "statistics")(run).get( + "carriers", "all" + ) + for metric in config_provider("plotting", "statistics")(run).get( + "metrics", STATISTICS + ) }, barplots_touch=RESULTS + "statistics/figures/comparison/country_{country}/.statistics_{carrier}_plots", @@ -355,6 +373,56 @@ rule plot_statistics_comparison: "../scripts/plot_statistics_comparison.py" +rule plot_statistics_scenario_comparison: + params: + plotting=config_provider("plotting"), + statistics=STATISTICS, + input: + expand( + RESULTS + + "statistics/csv/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}/country_{country}/{carrier}_{metric}.csv", + **config["scenario"], + metric=config_provider("plotting", "statistics")(run).get( + "metrics", STATISTICS + ), + run=config_provider("plotting", "statistics")(run).get( + "scenario_comparison", config["run"]["name"] + ), + allow_missing=True, + ), + output: + **{ + f"{metric}": "results/statistics/" + + config_provider("plotting", "statistics")(run).get( + "comparison_folder", "scenario_comparison" + ) + + "/" + + "figures/country_{country}/{carrier}_" + + f"{metric}.pdf" + for carrier in config_provider("plotting", "statistics")(run).get( + "carriers", "all" + ) + for metric in config_provider("plotting", "statistics")(run).get( + "metrics", STATISTICS + ) + }, + barplots_touch="results/statistics/" + + config_provider("plotting", "statistics")(run).get( + "comparison_folder", "results/scenario_comparison" + ) + + "/" + + "figures/country_{country}/.statistics_{carrier}_plots", + log: + "results/statistics/" + + config_provider("plotting", "statistics")(run).get( + "comparison_folder", "scenario_comparison" + ) + + "/" + + "logs/plot_statistics_scenario_comparison/country-{country}_carrier-{carrier}.log", + script: + "../scripts/plot_statistics_comparison.py" + + rule plot_elec_statistics: params: plotting=config_provider("plotting"), diff --git a/scripts/plot_statistics_comparison.py b/scripts/plot_statistics_comparison.py index 94501cc56..9f57f7eb6 100644 --- a/scripts/plot_statistics_comparison.py +++ b/scripts/plot_statistics_comparison.py @@ -5,9 +5,9 @@ # SPDX-License-Identifier: MIT import logging -import re import matplotlib.pyplot as plt +import numpy as np import pandas as pd import seaborn as sns from _helpers import configure_logging @@ -29,13 +29,17 @@ def rename_index(df): index = specific.where(duplicated, generic) df = df.set_axis(index) # rename columns and drop duplicates - columns = df.columns.str.split("_", expand=True) - columns = [ - columns.get_level_values(level).unique() - for level in range(columns.nlevels) - if not columns.get_level_values(level).duplicated(keep=False).all() + opts = df.columns.get_level_values(-1).str.split("_", expand=True) + scenario = df.columns.get_level_values(0) + opts = [ + opts.get_level_values(level).unique() + for level in range(opts.nlevels) + if not opts.get_level_values(level).duplicated(keep=False).all() ] - columns = pd.MultiIndex.from_arrays(columns) + opts = [""] if not opts else opts # in case all opts are the same + if df.columns.nlevels > 1: + columns = pd.MultiIndex.from_product([scenario, opts]) + # columns = pd.MultiIndex.from_arrays(columns) df.columns = columns.map("\n".join) return df @@ -49,8 +53,9 @@ def get_carrier_colors(carriers, tech_colors): return carrier_colors -def plot_static_comparison(df, ax, stacked=False): +def plot_static_comparison(df, ax, stacked=False, tech_colors=None): factor, unit = conversion[output] + df = df.round(2) df = df[df != 0] df = df.dropna(axis=0, how="all").fillna(0) if df.empty: @@ -77,12 +82,17 @@ def plot_static_comparison(df, ax, stacked=False): ax.grid(axis="x") -def read_csv(input, output): +def read_csv(input, output, num_run): try: # filter required csv to plot the wanted output files = list(filter(lambda x: output in x, input)) # retrieves network labels from folder name - network_labels = [file.split("/")[-3] for file in files] + if num_run == 1: + network_labels = [file.split("/")[-3] for file in files] + else: + network_labels = [ + (file.split("/")[-6], file.split("/")[-3]) for file in files + ] df = pd.concat( [ pd.read_csv(f, skiprows=2).set_index(["component", "carrier"]) @@ -90,10 +100,9 @@ def read_csv(input, output): ], axis=1, keys=network_labels, + names=["scenario", "opts", "to_drop"], ) - # get plot label and drop from index - label = df.columns.get_level_values(1).unique()[0] - df.columns = df.columns.droplevel(1) + df.columns = df.columns.droplevel("to_drop") except Exception as e: print(f"Error reading csv file for {output}: {e}") df = pd.DataFrame() @@ -113,20 +122,21 @@ def read_csv(input, output): configure_logging(snakemake) - tech_colors = ( - pd.Series(snakemake.params.plotting["tech_colors"]) - .groupby(rename_techs) - .first() - ) + plotting = snakemake.params.plotting + tech_colors = pd.Series(plotting["tech_colors"]).groupby(rename_techs).first() conversion = pd.Series(snakemake.params.statistics) + all_runs = snakemake.config["run"]["name"] + runs = plotting["statistics"].get("scenario_comparison", all_runs) + num_run = len(np.atleast_1d(runs)) + for output in snakemake.output.keys(): if "touch" in output: with open(snakemake.output.barplots_touch, "a"): pass continue fig, ax = plt.subplots() - df = read_csv(snakemake.input, output) + df = read_csv(snakemake.input, output, num_run) if df.empty: ax.text( 0.5, @@ -141,6 +151,8 @@ def read_csv(input, output): fig.savefig(snakemake.output[output]) continue - plot_static_comparison(df, ax, stacked=STACKED.get(output, True)) + plot_static_comparison( + df, ax, stacked=STACKED.get(output, True), tech_colors=tech_colors + ) fig.savefig(snakemake.output[output], bbox_inches="tight") From 75170595c00872fbf19e7ebc7850029dc0fec0d7 Mon Sep 17 00:00:00 2001 From: Philipp Glaum Date: Fri, 13 Sep 2024 15:10:51 +0200 Subject: [PATCH 15/18] update scenario plotting --- config/config.default.yaml | 1 + rules/postprocess.smk | 13 ++++++-- scripts/plot_statistics_comparison.py | 48 +++++++++++++++++---------- 3 files changed, 41 insertions(+), 21 deletions(-) diff --git a/config/config.default.yaml b/config/config.default.yaml index e2faf7f57..cfa28417a 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -1017,6 +1017,7 @@ plotting: countries: - all carriers: + - all - electricity - heat - co2 diff --git a/rules/postprocess.smk b/rules/postprocess.smk index ffc771032..c92e5a36f 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -373,6 +373,15 @@ rule plot_statistics_comparison: "../scripts/plot_statistics_comparison.py" +def get_scnario_copmarison_run(w): + run = config_provider("plotting", "statistics")(w).get( + "scenario_comparison", config["run"]["name"] + ) + if run == [""] or run == "": + run = config["run"]["name"] + return run + + rule plot_statistics_scenario_comparison: params: plotting=config_provider("plotting"), @@ -385,9 +394,7 @@ rule plot_statistics_scenario_comparison: metric=config_provider("plotting", "statistics")(run).get( "metrics", STATISTICS ), - run=config_provider("plotting", "statistics")(run).get( - "scenario_comparison", config["run"]["name"] - ), + run=get_scnario_copmarison_run(run), allow_missing=True, ), output: diff --git a/scripts/plot_statistics_comparison.py b/scripts/plot_statistics_comparison.py index 9f57f7eb6..0332357f8 100644 --- a/scripts/plot_statistics_comparison.py +++ b/scripts/plot_statistics_comparison.py @@ -21,6 +21,10 @@ } +def find_duplicate_substrings(idx): + idx.split("_") + + def rename_index(df): # rename index and drop duplicates specific = df.index.map(lambda x: f"{x[1]}({x[0]})") @@ -29,18 +33,22 @@ def rename_index(df): index = specific.where(duplicated, generic) df = df.set_axis(index) # rename columns and drop duplicates - opts = df.columns.get_level_values(-1).str.split("_", expand=True) - scenario = df.columns.get_level_values(0) + opts = df.columns.get_level_values("opts").str.split("_", expand=True) + # drops column level if all values are the same, if not keep unique values opts = [ - opts.get_level_values(level).unique() + opts.get_level_values(level).to_list() for level in range(opts.nlevels) - if not opts.get_level_values(level).duplicated(keep=False).all() + if (~opts.get_level_values(level).duplicated(keep="first")).sum() > 1 ] - opts = [""] if not opts else opts # in case all opts are the same - if df.columns.nlevels > 1: - columns = pd.MultiIndex.from_product([scenario, opts]) - # columns = pd.MultiIndex.from_arrays(columns) - df.columns = columns.map("\n".join) + opts = ["_".join(x) for x in zip(*opts)] + + if df.columns.nlevels == 1: + columns = opts + else: + scenarios = df.columns.get_level_values("scenario") + columns = pd.MultiIndex.from_arrays([scenarios, opts]) + columns = columns.map("\n".join) + df.columns = columns return df @@ -82,17 +90,20 @@ def plot_static_comparison(df, ax, stacked=False, tech_colors=None): ax.grid(axis="x") -def read_csv(input, output, num_run): +def read_csv(input, output, single_run): + # TODO since always read_csv always assigns column names, even if they do not exist, we drop them later. Maybe there is a better way. try: # filter required csv to plot the wanted output files = list(filter(lambda x: output in x, input)) # retrieves network labels from folder name - if num_run == 1: + if single_run: # only single scenario which can have opts network_labels = [file.split("/")[-3] for file in files] - else: + names = ["opts", "to_drop"] + else: # multiple scenarios which can have different opts network_labels = [ (file.split("/")[-6], file.split("/")[-3]) for file in files ] + names = ["scenario", "opts", "to_drop"] df = pd.concat( [ pd.read_csv(f, skiprows=2).set_index(["component", "carrier"]) @@ -100,7 +111,7 @@ def read_csv(input, output, num_run): ], axis=1, keys=network_labels, - names=["scenario", "opts", "to_drop"], + names=names, ) df.columns = df.columns.droplevel("to_drop") except Exception as e: @@ -115,7 +126,7 @@ def read_csv(input, output, num_run): snakemake = mock_snakemake( "plot_statistics_comparison", - run="", + run="solar", country="all", carrier="electricity", ) @@ -126,9 +137,10 @@ def read_csv(input, output, num_run): tech_colors = pd.Series(plotting["tech_colors"]).groupby(rename_techs).first() conversion = pd.Series(snakemake.params.statistics) - all_runs = snakemake.config["run"]["name"] - runs = plotting["statistics"].get("scenario_comparison", all_runs) - num_run = len(np.atleast_1d(runs)) + if "run" in snakemake.wildcards.keys(): + single_run = True + else: + single_run = False for output in snakemake.output.keys(): if "touch" in output: @@ -136,7 +148,7 @@ def read_csv(input, output, num_run): pass continue fig, ax = plt.subplots() - df = read_csv(snakemake.input, output, num_run) + df = read_csv(snakemake.input, output, single_run) if df.empty: ax.text( 0.5, From eb166115a1945ff36335f5dc450aadaa4fb03651 Mon Sep 17 00:00:00 2001 From: Philipp Glaum Date: Mon, 16 Sep 2024 10:08:18 +0200 Subject: [PATCH 16/18] only run scenario comparison if folder is defined --- config/config.default.yaml | 2 +- rules/collect.smk | 35 +++++++++++++++++++++-------------- rules/postprocess.smk | 2 +- 3 files changed, 23 insertions(+), 16 deletions(-) diff --git a/config/config.default.yaml b/config/config.default.yaml index cfa28417a..8ae37f9f3 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -1026,9 +1026,9 @@ plotting: metrics: - energy_balance - total_cost + comparison_folder: "" scenario_comparison: - "" - comparison_folder: "scenario_comparison" nice_names: OCGT: "Open-Cycle Gas" diff --git a/rules/collect.smk b/rules/collect.smk index 9b38f873a..3958d88f3 100644 --- a/rules/collect.smk +++ b/rules/collect.smk @@ -122,20 +122,27 @@ rule plot_statistics: ), run=config["run"]["name"], ), - expand( - "results/statistics/" - + config_provider("plotting", "statistics")(run).get( - "comparison_folder", "results/scenario_comparison" + ( + expand( + "results/statistics/" + + config_provider("plotting", "statistics")(run).get( + "comparison_folder", "''" + ) + + "/" + + "figures/country_{country}/.statistics_{carrier}_plots", + **config["scenario"], + country=config_provider("plotting", "statistics")(run).get( + "countries", "all" + ), + carrier=config_provider("plotting", "statistics")(run).get( + "carriers", "all" + ), + run=config["run"]["name"], ) - + "/" - + "figures/country_{country}/.statistics_{carrier}_plots", - **config["scenario"], - country=config_provider("plotting", "statistics")(run).get( - "countries", "all" - ), - carrier=config_provider("plotting", "statistics")(run).get( - "carriers", "all" - ), - run=config["run"]["name"], + if config_provider("plotting", "statistics")(run).get( + "comparison_folder", "" + ) + != "" + else [] ), ], diff --git a/rules/postprocess.smk b/rules/postprocess.smk index c92e5a36f..dedc5a41d 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -401,7 +401,7 @@ rule plot_statistics_scenario_comparison: **{ f"{metric}": "results/statistics/" + config_provider("plotting", "statistics")(run).get( - "comparison_folder", "scenario_comparison" + "comparison_folder", "" ) + "/" + "figures/country_{country}/{carrier}_" From b1b07a167a7c02de4d1cee2e33652a495b2f158b Mon Sep 17 00:00:00 2001 From: Philipp Glaum Date: Mon, 16 Sep 2024 10:22:20 +0200 Subject: [PATCH 17/18] update documentation --- doc/configtables/plotting.csv | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/doc/configtables/plotting.csv b/doc/configtables/plotting.csv index 3124ce306..d6eb1fa01 100644 --- a/doc/configtables/plotting.csv +++ b/doc/configtables/plotting.csv @@ -9,8 +9,11 @@ costs_threshold,bn Euro,float,Threshold below which technologies will not be sho energy_max,TWh,float,Upper y-axis limit in energy bar plots. energy_min,TWh,float,Lower y-axis limit in energy bar plots. energy_threshold,TWh,float,Threshold below which technologies will not be shown in energy bar plots. -countries,--,[str],List of countries you want to plot in your statistics figures. Default is all coutries but also single countries can be defined like ``DE``. -carriers,--,[str],Carrier you want to plot from in your statistics figures. This carrier must be an available bus carrier in your network or substring of a carrier or a defined carrier group. +countries,--,[str],List of countries you want to create the statistcs for. Statistics includes csv files and figures. Default is all coutries but also single countries can be defined like ``DE``. +carriers,--,[str],"Carrier you want to create the statistcs for, i.e. csv files and figures. This carrier must be an available bus carrier in your network or substring of a carrier or a defined carrier group." carrier_groups,--,str -> carrier,Mapping from ``carrier_groups`` to individual bus carriers in the network. +metrics,--,[str],"List of metrics you want to create your statistics for. Default is `energy_balance' and 'total_cost'. However, you can choose other metrics from the list below:", +comparison_folder, --,str,"Folder name where the scenario comparison statistics are stored. If this is empty, no senario comparison plots will be generated Defaultis an empty string ''." +scenario_comparison,--,[str],"List of scenarios you want to compare. If the list is empty, all scenarios from ``run:name`` are compared. Only works, when a ``comparison_folder`` is defined. Default is an empty list." nice_names,--,str -> str,Mapping from network ``carrier`` to a more readable name. tech_colors,--,carrier -> HEX colour code,Mapping from network ``carrier`` to a colour (`HEX colour code `_). From b04c47447b88ae16028bda30e1d8f6affb72604a Mon Sep 17 00:00:00 2001 From: Philipp Glaum Date: Tue, 12 Nov 2024 16:30:22 +0100 Subject: [PATCH 18/18] update rules to work with new workflow --- rules/collect.smk | 96 +++++++++++++++++++++++-------------------- rules/postprocess.smk | 78 +++++++++++++++-------------------- 2 files changed, 84 insertions(+), 90 deletions(-) diff --git a/rules/collect.smk b/rules/collect.smk index e988d5847..0101b9c93 100644 --- a/rules/collect.smk +++ b/rules/collect.smk @@ -87,51 +87,59 @@ rule validate_elec_networks: rule plot_statistics: input: - [ - expand( + lambda w: expand( + ( RESULTS - + "statistics/figures/comparison/country_{country}/.statistics_{carrier}_plots", - country=config_provider("plotting", "statistics")(run).get( - "countries", "all" - ), - carrier=config_provider("plotting", "statistics")(run).get( - "carriers", "all" - ), - run=config["run"]["name"], + + "statistics/csv/base_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}/country_{country}/.statistics_{carrier}_csv" ), - expand( - RESULTS - + "statistics/figures/single/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}/country_{country}/.statistics_{carrier}_plots", - **config["scenario"], - country=config_provider("plotting", "statistics")(run).get( - "countries", "all" - ), - carrier=config_provider("plotting", "statistics")(run).get( - "carriers", "all" - ), - run=config["run"]["name"], + **config["scenario"], + run=config["run"]["name"], + country=config_provider("plotting", "statistics")(w).get( + "countries", "all" ), - ( - expand( - "results/statistics/" - + config_provider("plotting", "statistics")(run).get( - "comparison_folder", "''" - ) - + "/" - + "figures/country_{country}/.statistics_{carrier}_plots", - **config["scenario"], - country=config_provider("plotting", "statistics")(run).get( - "countries", "all" - ), - carrier=config_provider("plotting", "statistics")(run).get( - "carriers", "all" - ), - run=config["run"]["name"], - ) - if config_provider("plotting", "statistics")(run).get( - "comparison_folder", "" - ) - != "" - else [] + carrier=config_provider("plotting", "statistics")(w).get( + "carriers", "all" + ), + allow_missing=True, + ), + expand( + RESULTS + + "statistics/figures/single/base_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}/country_{country}/.statistics_{carrier}_plots", + **config["scenario"], + country=config_provider("plotting", "statistics")(run).get( + "countries", "all" + ), + carrier=config_provider("plotting", "statistics")(run).get( + "carriers", "all" ), - ], + run=config["run"]["name"], + ), + expand( + RESULTS + + "statistics/figures/comparison/country_{country}/.statistics_{carrier}_plots", + country=config_provider("plotting", "statistics")(run).get( + "countries", "all" + ), + carrier=config_provider("plotting", "statistics")(run).get( + "carriers", "all" + ), + run=config["run"]["name"], + ), + expand( + "results/statistics/" + + config_provider("plotting", "statistics")(run).get( + "comparison_folder", "''" + ) + + "/" + + "figures/country_{country}/.statistics_{carrier}_plots", + country=config_provider("plotting", "statistics")(run).get( + "countries", "all" + ), + carrier=config_provider("plotting", "statistics")(run).get( + "carriers", "all" + ), + run=config["run"]["name"], + ) + if config_provider("plotting", "statistics")(run).get("comparison_folder", "") + != "" + else [], diff --git a/rules/postprocess.smk b/rules/postprocess.smk index 97722d72c..fa9f82977 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -257,6 +257,25 @@ STATISTICS_BARPLOTS = [ "market_value", ] + +rule plot_base_statistics: + params: + plotting=config_provider("plotting"), + barplots=STATISTICS_BARPLOTS, + input: + network=RESULTS + "networks/base_s_{clusters}_elec_l{ll}_{opts}.nc", + output: + **{ + f"{plot}_bar": RESULTS + + f"figures/statistics_{plot}_bar_base_s_{{clusters}}_elec_l{{ll}}_{{opts}}.pdf" + for plot in STATISTICS_BARPLOTS + }, + barplots_touch=RESULTS + + "figures/.statistics_plots_base_s_{clusters}_elec_l{ll}_{opts}", + script: + "../scripts/plot_statistics.py" + + STATISTICS = { "capacity_factor": ("-", "p.u."), "installed_capacity": (1e3, "GW"), @@ -277,11 +296,11 @@ rule write_statistics: statistics=STATISTICS, input: network=RESULTS - + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", + + "postnetworks/base_s_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", output: **{ f"{metric}": RESULTS - + "statistics/csv/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}/country_{country}/{carrier}_" + + "statistics/csv/base_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}/country_{country}/{carrier}_" + f"{metric}.csv" for carrier in config_provider("plotting", "statistics")(run).get( "carriers", "all" @@ -291,10 +310,10 @@ rule write_statistics: ) }, csv_touch=RESULTS - + "statistics/csv/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}/country_{country}/.statistics_{carrier}_csv", + + "statistics/csv/base_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}/country_{country}/.statistics_{carrier}_csv", log: RESULTS - + "logs/write_statistics/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_country-{country}_carrier-{carrier}.log", + + "logs/write_statistics/base_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_country-{country}_carrier-{carrier}.log", script: "../scripts/write_statistics.py" @@ -306,7 +325,7 @@ rule plot_statistics_single: input: **{ f"{metric}": RESULTS - + "statistics/csv/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}/country_{country}/{carrier}_" + + "statistics/csv/base_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}/country_{country}/{carrier}_" + f"{metric}.csv" for carrier in config_provider("plotting", "statistics")(run).get( "carriers", "all" @@ -318,7 +337,7 @@ rule plot_statistics_single: output: **{ f"{metric}": RESULTS - + "statistics/figures/single/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}/country_{country}/{carrier}_" + + "statistics/figures/single/base_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}/country_{country}/{carrier}_" + f"{metric}.pdf" for carrier in config_provider("plotting", "statistics")(run).get( "carriers", "all" @@ -328,10 +347,10 @@ rule plot_statistics_single: ) }, barplots_touch=RESULTS - + "statistics/figures/single/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}/country_{country}/.statistics_{carrier}_plots", + + "statistics/figures/single/base_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}/country_{country}/.statistics_{carrier}_plots", log: RESULTS - + "logs/plot_statistics_single/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_country-{country}_carrier-{carrier}.log", + + "logs/plot_statistics_single/base_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_country-{country}_carrier-{carrier}.log", script: "../scripts/plot_statistics_single.py" @@ -343,7 +362,7 @@ rule plot_statistics_comparison: input: expand( RESULTS - + "statistics/csv/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}/country_{country}/{carrier}_{metric}.csv", + + "statistics/csv/base_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}/country_{country}/{carrier}_{metric}.csv", **config["scenario"], metric=config_provider("plotting", "statistics")(run).get( "metrics", STATISTICS @@ -387,7 +406,7 @@ rule plot_statistics_scenario_comparison: input: expand( RESULTS - + "statistics/csv/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}/country_{country}/{carrier}_{metric}.csv", + + "statistics/csv/base_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}/country_{country}/{carrier}_{metric}.csv", **config["scenario"], metric=config_provider("plotting", "statistics")(run).get( "metrics", STATISTICS @@ -397,12 +416,7 @@ rule plot_statistics_scenario_comparison: ), output: **{ - f"{metric}": "results/statistics/" - + config_provider("plotting", "statistics")(run).get( - "comparison_folder", "" - ) - + "/" - + "figures/country_{country}/{carrier}_" + f"{metric}": "results/statistics/{comparison_folder}/figures/country_{country}/{carrier}_" + f"{metric}.pdf" for carrier in config_provider("plotting", "statistics")(run).get( "carriers", "all" @@ -411,36 +425,8 @@ rule plot_statistics_scenario_comparison: "metrics", STATISTICS ) }, - barplots_touch="results/statistics/" - + config_provider("plotting", "statistics")(run).get( - "comparison_folder", "results/scenario_comparison" - ) - + "/" - + "figures/country_{country}/.statistics_{carrier}_plots", + barplots_touch="results/statistics/{comparison_folder}/figures/country_{country}/.statistics_{carrier}_plots", log: - "results/statistics/" - + config_provider("plotting", "statistics")(run).get( - "comparison_folder", "scenario_comparison" - ) - + "/" - + "logs/plot_statistics_scenario_comparison/country-{country}_carrier-{carrier}.log", + "results/logs/{comparison_folder}/plot_statistics_scenario_comparison/country-{country}_carrier-{carrier}.log", script: "../scripts/plot_statistics_comparison.py" - - -rule plot_base_statistics: - params: - plotting=config_provider("plotting"), - barplots=STATISTICS_BARPLOTS, - input: - network=RESULTS + "networks/base_s_{clusters}_elec_l{ll}_{opts}.nc", - output: - **{ - f"{plot}_bar": RESULTS - + f"figures/statistics_{plot}_bar_base_s_{{clusters}}_elec_l{{ll}}_{{opts}}.pdf" - for plot in STATISTICS_BARPLOTS - }, - barplots_touch=RESULTS - + "figures/.statistics_plots_base_s_{clusters}_elec_l{ll}_{opts}", - script: - "../scripts/plot_statistics.py"