diff --git a/src/egon/data/datasets/chp/__init__.py b/src/egon/data/datasets/chp/__init__.py index 0c20319f9..69d410b1f 100644 --- a/src/egon/data/datasets/chp/__init__.py +++ b/src/egon/data/datasets/chp/__init__.py @@ -36,6 +36,8 @@ scale_prox2now, ) +from egon.data.datasets.pypsaeur import read_network + Base = declarative_base() @@ -588,17 +590,10 @@ def insert_chp_egon100re(): if config.settings()["egon-data"]["--dataset-boundary"] != "Everything": additional_capacity /= 16 - target_file = ( - Path(".") - / "data_bundle_egon_data" - / "pypsa_eur_sec" - / "2022-07-26-egondata-integration" - / "postnetworks" - / "elec_s_37_lv2.0__Co2L0-1H-T-H-B-I-dist1_2050.nc" - ) - network = pypsa.Network(str(target_file)) - chp_index = "DE0 0 urban central gas CHP" + network = read_network() + + chp_index = "DE0 0 urban central gas CHP-2045" standard_chp_th = 10 standard_chp_el = ( diff --git a/src/egon/data/datasets/gas_neighbours/eGon2035.py b/src/egon/data/datasets/gas_neighbours/eGon2035.py index 04329d95b..44afe851d 100755 --- a/src/egon/data/datasets/gas_neighbours/eGon2035.py +++ b/src/egon/data/datasets/gas_neighbours/eGon2035.py @@ -19,6 +19,7 @@ from egon.data.datasets.gas_neighbours.gas_abroad import ( insert_gas_grid_capacities, ) +from egon.data.datasets.pypsaeur import prepared_network from egon.data.datasets.scenario_parameters import get_sector_parameters countries = [ @@ -638,17 +639,7 @@ def import_ch4_demandTS(): """ - cwd = Path(".") - target_file = ( - cwd - / "data_bundle_egon_data" - / "pypsa_eur_sec" - / "2022-07-26-egondata-integration" - / "postnetworks" - / "elec_s_37_lv2.0__Co2L0-1H-T-H-B-I-dist1_2050.nc" - ) - - network = pypsa.Network(str(target_file)) + network = prepared_network() # Set country tag for all buses network.buses.country = network.buses.index.str[:2] diff --git a/src/egon/data/datasets/industrial_gas_demand.py b/src/egon/data/datasets/industrial_gas_demand.py index f372a4176..2a883fdef 100755 --- a/src/egon/data/datasets/industrial_gas_demand.py +++ b/src/egon/data/datasets/industrial_gas_demand.py @@ -27,7 +27,7 @@ initialise_bus_insertion, ) from egon.data.datasets.etrago_setup import link_geom_from_buses -from egon.data.datasets.pypsaeur import read_network +from egon.data.datasets.pypsaeur import (read_network, prepared_network) from egon.data.datasets.scenario_parameters import get_sector_parameters logger = logging.getLogger(__name__) @@ -48,7 +48,7 @@ class IndustrialGasDemand(Dataset): #: name: str = "IndustrialGasDemand" #: - version: str = "0.0.5" + version: str = "0.0.6" def __init__(self, dependencies): super().__init__( @@ -487,7 +487,8 @@ def insert_industrial_gas_demand_egon100RE(): # adjust H2 and CH4 total demands (values from PES) # CH4 demand = 0 in 100RE, therefore scale H2 ts # fallback values see https://github.com/openego/eGon-data/issues/626 - n = read_network() + n = prepared_network() + solved_network = read_network() try: H2_total_PES = ( @@ -495,6 +496,13 @@ def insert_industrial_gas_demand_egon100RE(): "DE0 0 H2 for industry", "p_set" ] * 8760 + # Add h2 demand of Fischer-Tropsch process from pypsa-eur + + solved_network.links_t.p0[ + solved_network.links.loc[ + solved_network.links.index.str.contains( + "DE0 0 Fischer-Tropsch")].index].mul( + solved_network.snapshot_weightings.generators, + axis= 0).sum().sum() ) except KeyError: H2_total_PES = 42090000 diff --git a/src/egon/data/datasets/pypsaeur/__init__.py b/src/egon/data/datasets/pypsaeur/__init__.py index a6baf6e9c..2fc120fbf 100755 --- a/src/egon/data/datasets/pypsaeur/__init__.py +++ b/src/egon/data/datasets/pypsaeur/__init__.py @@ -30,7 +30,7 @@ class PreparePypsaEur(Dataset): def __init__(self, dependencies): super().__init__( name="PreparePypsaEur", - version="0.0.40", + version="0.0.41", dependencies=dependencies, tasks=( download, @@ -43,7 +43,7 @@ class RunPypsaEur(Dataset): def __init__(self, dependencies): super().__init__( name="SolvePypsaEur", - version="0.0.38", + version="0.0.39", dependencies=dependencies, tasks=( prepare_network_2, @@ -385,10 +385,10 @@ def read_network(planning_horizon=3): Path(".") / "data_bundle_powerd_data" / "pypsa_eur" - / "2024-08-02-egondata-integration" + / "21122024_3h_clean_run" / "results" / "postnetworks" - / "elec_s_37_lv1.5__Co2L0-1H-T-H-B-I-A-solar+p3_2050.nc" + / "base_s_39_lc1.25__cb40ex0-T-H-I-B-solar+p3-dist1_2045.nc" ) return pypsa.Network(target_file) @@ -497,7 +497,8 @@ def electrical_neighbours_egon100(): def neighbor_reduction(): - network = read_network() + network_solved = read_network() + network_prepared = prepared_network(planning_horizon="2045") # network.links.drop("pipe_retrofit", axis="columns", inplace=True) @@ -516,164 +517,152 @@ def neighbor_reduction(): "FR", "LU", ] - foreign_buses = network.buses[ - ~network.buses.index.str.contains("|".join(wanted_countries)) + foreign_buses = network_solved.buses[ + ~network_solved.buses.index.str.contains("|".join(wanted_countries)) ] - network.buses = network.buses.drop( - network.buses.loc[foreign_buses.index].index + network_solved.buses = network_solved.buses.drop( + network_solved.buses.loc[foreign_buses.index].index ) + # Add H2 demand of Fischer-Tropsch process to industrial H2 demands + industrial_hydrogen = network_prepared.loads.loc[ + network_prepared.loads.carrier== "H2 for industry"] + fischer_tropsch = network_solved.links_t.p0[ + network_solved.links.loc[ + network_solved.links.carrier=="Fischer-Tropsch"].index + ].mul(network_solved.snapshot_weightings.generators, axis=0).sum() + for i, row in industrial_hydrogen.iterrows(): + network_prepared.loads.loc[i, "p_set"] += fischer_tropsch[ + fischer_tropsch.index.str.startswith(row.bus[:5])].sum()/8760 + # drop foreign lines and links from the 2nd row - network.lines = network.lines.drop( - network.lines[ - (network.lines["bus0"].isin(network.buses.index) == False) - & (network.lines["bus1"].isin(network.buses.index) == False) + network_solved.lines = network_solved.lines.drop( + network_solved.lines[ + (network_solved.lines["bus0"].isin(network_solved.buses.index) == False) + & (network_solved.lines["bus1"].isin(network_solved.buses.index) == False) ].index ) # select all lines which have at bus1 the bus which is kept - lines_cb_1 = network.lines[ - (network.lines["bus0"].isin(network.buses.index) == False) + lines_cb_1 = network_solved.lines[ + (network_solved.lines["bus0"].isin(network_solved.buses.index) == False) ] # create a load at bus1 with the line's hourly loading for i, k in zip(lines_cb_1.bus1.values, lines_cb_1.index): - network.add( + + # Copy loading of lines into hourly resolution + pset = pd.Series( + index=network_prepared.snapshots, + data = network_solved.lines_t.p1[k].resample("H").ffill()) + pset["2011-12-31 22:00:00"] = pset["2011-12-31 21:00:00"] + pset["2011-12-31 23:00:00"] = pset["2011-12-31 21:00:00"] + + # Loads are all imported from the prepared network in the end + network_prepared.add( "Load", "slack_fix " + i + " " + k, bus=i, - p_set=network.lines_t.p1[k], - ) - network.loads.carrier.loc["slack_fix " + i + " " + k] = ( - lines_cb_1.carrier[k] + p_set=pset, + carrier = lines_cb_1.loc[k, "carrier"] ) # select all lines which have at bus0 the bus which is kept - lines_cb_0 = network.lines[ - (network.lines["bus1"].isin(network.buses.index) == False) + lines_cb_0 = network_solved.lines[ + (network_solved.lines["bus1"].isin(network_solved.buses.index) == False) ] # create a load at bus0 with the line's hourly loading for i, k in zip(lines_cb_0.bus0.values, lines_cb_0.index): - network.add( + # Copy loading of lines into hourly resolution + pset = pd.Series( + index=network_prepared.snapshots, + data = network_solved.lines_t.p0[k].resample("H").ffill()) + pset["2011-12-31 22:00:00"] = pset["2011-12-31 21:00:00"] + pset["2011-12-31 23:00:00"] = pset["2011-12-31 21:00:00"] + + network_prepared.add( "Load", "slack_fix " + i + " " + k, bus=i, - p_set=network.lines_t.p0[k], - ) - network.loads.carrier.loc["slack_fix " + i + " " + k] = ( - lines_cb_0.carrier[k] + p_set=pset, + carrier = lines_cb_0.loc[k, "carrier"] ) - # do the same for links - network.links = network.links.drop( - network.links[ - (network.links["bus0"].isin(network.buses.index) == False) - & (network.links["bus1"].isin(network.buses.index) == False) - ].index - ) + # do the same for links + network_solved.mremove( + "Link", + network_solved.links[ + (~network_solved.links.bus0.isin(network_solved.buses.index)) + | (~network_solved.links.bus1.isin(network_solved.buses.index))].index + ) # select all links which have at bus1 the bus which is kept - links_cb_1 = network.links[ - (network.links["bus0"].isin(network.buses.index) == False) + links_cb_1 = network_solved.links[ + (network_solved.links["bus0"].isin(network_solved.buses.index) == False) ] # create a load at bus1 with the link's hourly loading for i, k in zip(links_cb_1.bus1.values, links_cb_1.index): - network.add( + pset = pd.Series( + index=network_prepared.snapshots, + data = network_solved.links_t.p1[k].resample("H").ffill()) + pset["2011-12-31 22:00:00"] = pset["2011-12-31 21:00:00"] + pset["2011-12-31 23:00:00"] = pset["2011-12-31 21:00:00"] + + network_prepared.add( "Load", "slack_fix_links " + i + " " + k, bus=i, - p_set=network.links_t.p1[k], - ) - network.loads.carrier.loc["slack_fix_links " + i + " " + k] = ( - links_cb_1.carrier[k] + p_set=pset, + carrier = links_cb_1.loc[k, "carrier"] ) # select all links which have at bus0 the bus which is kept - links_cb_0 = network.links[ - (network.links["bus1"].isin(network.buses.index) == False) + links_cb_0 = network_solved.links[ + (network_solved.links["bus1"].isin(network_solved.buses.index) == False) ] # create a load at bus0 with the link's hourly loading for i, k in zip(links_cb_0.bus0.values, links_cb_0.index): - network.add( + pset = pd.Series( + index=network_prepared.snapshots, + data = network_solved.links_t.p0[k].resample("H").ffill()) + pset["2011-12-31 22:00:00"] = pset["2011-12-31 21:00:00"] + pset["2011-12-31 23:00:00"] = pset["2011-12-31 21:00:00"] + + network_prepared.add( "Load", "slack_fix_links " + i + " " + k, bus=i, - p_set=network.links_t.p0[k], - ) - network.loads.carrier.loc["slack_fix_links " + i + " " + k] = ( - links_cb_0.carrier[k] + p_set=pset, + carrier = links_cb_0.carrier[k], ) # drop remaining foreign components - - network.lines = network.lines.drop( - network.lines[ - (network.lines["bus0"].isin(network.buses.index) == False) - | (network.lines["bus1"].isin(network.buses.index) == False) - ].index - ) - - network.links = network.links.drop( - network.links[ - (network.links["bus0"].isin(network.buses.index) == False) - | (network.links["bus1"].isin(network.buses.index) == False) - ].index - ) - - network.transformers = network.transformers.drop( - network.transformers[ - (network.transformers["bus0"].isin(network.buses.index) == False) - | (network.transformers["bus1"].isin(network.buses.index) == False) - ].index - ) - network.generators = network.generators.drop( - network.generators[ - (network.generators["bus"].isin(network.buses.index) == False) - ].index - ) - - network.loads = network.loads.drop( - network.loads[ - (network.loads["bus"].isin(network.buses.index) == False) - ].index - ) - - network.storage_units = network.storage_units.drop( - network.storage_units[ - (network.storage_units["bus"].isin(network.buses.index) == False) - ].index - ) - - components = [ - "loads", - "generators", - "lines", - "buses", - "transformers", - "links", - ] - for g in components: # loads_t - h = g + "_t" - nw = getattr(network, h) # network.loads_t - for i in nw.keys(): # network.loads_t.p - cols = [ - j - for j in getattr(nw, i).columns - if j not in getattr(network, g).index - ] - for k in cols: - del getattr(nw, i)[k] + for comp in network_solved.iterate_components(): + if "bus0" in comp.df.columns: + network_solved.mremove( + comp.name, + comp.df[~comp.df.bus0.isin(network_solved.buses.index)].index + ) + network_solved.mremove( + comp.name, + comp.df[~comp.df.bus1.isin(network_solved.buses.index)].index + ) + elif "bus" in comp.df.columns: + network_solved.mremove( + comp.name, + comp.df[~comp.df.bus.isin(network_solved.buses.index)].index + ) # writing components of neighboring countries to etrago tables # Set country tag for all buses - network.buses.country = network.buses.index.str[:2] - neighbors = network.buses[network.buses.country != "DE"] + network_solved.buses.country = network_solved.buses.index.str[:2] + neighbors = network_solved.buses[network_solved.buses.country != "DE"] neighbors["new_index"] = ( db.next_etrago_id("bus") + neighbors.reset_index().index @@ -701,12 +690,12 @@ def neighbor_reduction(): # lines, the foreign crossborder lines # (without crossborder lines to Germany!) - neighbor_lines = network.lines[ - network.lines.bus0.isin(neighbors.index) - & network.lines.bus1.isin(neighbors.index) + neighbor_lines = network_solved.lines[ + network_solved.lines.bus0.isin(neighbors.index) + & network_solved.lines.bus1.isin(neighbors.index) ] - if not network.lines_t["s_max_pu"].empty: - neighbor_lines_t = network.lines_t["s_max_pu"][neighbor_lines.index] + if not network_solved.lines_t["s_max_pu"].empty: + neighbor_lines_t = network_prepared.lines_t["s_max_pu"][neighbor_lines.index] neighbor_lines.reset_index(inplace=True) neighbor_lines.bus0 = ( @@ -717,15 +706,15 @@ def neighbor_reduction(): ) neighbor_lines.index += db.next_etrago_id("line") - if not network.lines_t["s_max_pu"].empty: + if not network_solved.lines_t["s_max_pu"].empty: for i in neighbor_lines_t.columns: new_index = neighbor_lines[neighbor_lines["name"] == i].index neighbor_lines_t.rename(columns={i: new_index[0]}, inplace=True) # links - neighbor_links = network.links[ - network.links.bus0.isin(neighbors.index) - & network.links.bus1.isin(neighbors.index) + neighbor_links = network_solved.links[ + network_solved.links.bus0.isin(neighbors.index) + & network_solved.links.bus1.isin(neighbors.index) ] neighbor_links.reset_index(inplace=True) @@ -738,12 +727,12 @@ def neighbor_reduction(): neighbor_links.index += db.next_etrago_id("link") # generators - neighbor_gens = network.generators[ - network.generators.bus.isin(neighbors.index) + neighbor_gens = network_solved.generators[ + network_solved.generators.bus.isin(neighbors.index) ] - neighbor_gens_t = network.generators_t["p_max_pu"][ + neighbor_gens_t = network_prepared.generators_t["p_max_pu"][ neighbor_gens[ - neighbor_gens.index.isin(network.generators_t["p_max_pu"].columns) + neighbor_gens.index.isin(network_prepared.generators_t["p_max_pu"].columns) ].index ] @@ -758,12 +747,12 @@ def neighbor_reduction(): neighbor_gens_t.rename(columns={i: new_index[0]}, inplace=True) # loads - - neighbor_loads = network.loads[network.loads.bus.isin(neighbors.index)] + # imported from prenetwork in 1h-resolution + neighbor_loads = network_prepared.loads[network_prepared.loads.bus.isin(neighbors.index)] neighbor_loads_t_index = neighbor_loads.index[ - neighbor_loads.index.isin(network.loads_t.p_set.columns) + neighbor_loads.index.isin(network_prepared.loads_t.p_set.columns) ] - neighbor_loads_t = network.loads_t["p_set"][neighbor_loads_t_index] + neighbor_loads_t = network_prepared.loads_t["p_set"][neighbor_loads_t_index] neighbor_loads.reset_index(inplace=True) neighbor_loads.bus = ( @@ -776,11 +765,11 @@ def neighbor_reduction(): neighbor_loads_t.rename(columns={i: new_index[0]}, inplace=True) # stores - neighbor_stores = network.stores[network.stores.bus.isin(neighbors.index)] + neighbor_stores = network_solved.stores[network_solved.stores.bus.isin(neighbors.index)] neighbor_stores_t_index = neighbor_stores.index[ - neighbor_stores.index.isin(network.stores_t.e_min_pu.columns) + neighbor_stores.index.isin(network_solved.stores_t.e_min_pu.columns) ] - neighbor_stores_t = network.stores_t["e_min_pu"][neighbor_stores_t_index] + neighbor_stores_t = network_prepared.stores_t["e_min_pu"][neighbor_stores_t_index] neighbor_stores.reset_index(inplace=True) neighbor_stores.bus = ( @@ -793,13 +782,13 @@ def neighbor_reduction(): neighbor_stores_t.rename(columns={i: new_index[0]}, inplace=True) # storage_units - neighbor_storage = network.storage_units[ - network.storage_units.bus.isin(neighbors.index) + neighbor_storage = network_solved.storage_units[ + network_solved.storage_units.bus.isin(neighbors.index) ] neighbor_storage_t_index = neighbor_storage.index[ - neighbor_storage.index.isin(network.storage_units_t.inflow.columns) + neighbor_storage.index.isin(network_solved.storage_units_t.inflow.columns) ] - neighbor_storage_t = network.storage_units_t["inflow"][ + neighbor_storage_t = network_prepared.storage_units_t["inflow"][ neighbor_storage_t_index ] @@ -1330,7 +1319,7 @@ def links_to_etrago(neighbor_links, scn="eGon100RE", extendable=True): ) # writing neighboring lines_t s_max_pu to etrago tables - if not network.lines_t["s_max_pu"].empty: + if not network_solved.lines_t["s_max_pu"].empty: neighbor_lines_t_etrago = pd.DataFrame( columns=["scn_name", "s_max_pu"], index=neighbor_lines_t.columns ) @@ -1377,10 +1366,11 @@ def prepared_network(planning_horizon=3): Path(".") / "data_bundle_powerd_data" / "pypsa_eur" - / "2024-08-02-egondata-integration" + / "21122024_3h_clean_run" / "results" - / "postnetworks" - / "elec_s_37_lv1.5__Co2L0-1H-T-H-B-I-A-solar+p3_2050.nc" + / "prenetworks" + / "prenetwork_post-manipulate_pre-solve" + / "base_s_39_lc1.25__cb40ex0-T-H-I-B-solar+p3-dist1_2035.nc" ) return pypsa.Network(target_file.absolute().as_posix()) diff --git a/src/egon/data/datasets/sanity_checks.py b/src/egon/data/datasets/sanity_checks.py index 859bcec5c..68033bca1 100644 --- a/src/egon/data/datasets/sanity_checks.py +++ b/src/egon/data/datasets/sanity_checks.py @@ -74,6 +74,7 @@ def __init__(self, dependencies): sanitycheck_pv_rooftop_buildings, sanitycheck_home_batteries, sanitycheck_dsm, + etrago_timeseries_length, }, ) @@ -1542,3 +1543,31 @@ def df_from_series(s: pd.Series): assert np.allclose(e_max_df, individual_e_max_df) assert np.allclose(e_min_df, individual_e_min_df) + +def etrago_timeseries_length(): + + for component in ["generator", "load", "link", "store", "storage"]: + + columns = db.select_dataframe( + f""" + SELECT * + FROM information_schema.columns + WHERE table_schema = 'grid' + AND table_name = 'egon_etrago_{component}_timeseries' + """ + ) + columns = columns[columns.data_type=="ARRAY"].column_name.values + + for col in columns: + lengths = db.select_dataframe( + f""" + SELECT array_length({col}, 1) + FROM grid.egon_etrago_{component}_timeseries; + """ + )["array_length"] + + if not lengths.dropna().empty: + assert(lengths.dropna()==8760).all(), ( + f"Timeseries with a length != 8760 for {component} {col}") + else: + print(f"Empty timeseries for {component} {col}") diff --git a/src/egon/data/datasets/scenario_capacities.py b/src/egon/data/datasets/scenario_capacities.py index 609e1b16f..d2ddabee6 100755 --- a/src/egon/data/datasets/scenario_capacities.py +++ b/src/egon/data/datasets/scenario_capacities.py @@ -677,7 +677,8 @@ def eGon100_capacities(): cwd / "data_bundle_powerd_data" / "pypsa_eur" - / "2024-08-02-egondata-integration" + / "21122024_3h_clean_run" + / "results" / "csvs" / sources["eGon100RE"]["capacities"] )