diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 75e024d72..dc392cfa0 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -124,6 +124,8 @@ Added `#237 `_ * Merge electrical loads per bus and export to etrago tables `#328 `_ +* Insert industial gas demand + `#321 `_ * Integrate existing CHP and extdended CHP > 10MW_el `#266 `_ @@ -237,5 +239,10 @@ Bug fixes `#387 `_ * Add xlrd and openpyxl to installation setup `#400 `_ +* Remove hard-coded slashes in file paths to ensure Windows compatibility + `#398 `_ +* Add missing dependency in pipeline.py + `#412 `_ * Add prefix egon to MV grid district tables `#349 `_ + diff --git a/src/egon/data/airflow/dags/pipeline.py b/src/egon/data/airflow/dags/pipeline.py index d80bed738..fc0475bbb 100755 --- a/src/egon/data/airflow/dags/pipeline.py +++ b/src/egon/data/airflow/dags/pipeline.py @@ -37,6 +37,7 @@ from egon.data.datasets.zensus_mv_grid_districts import ZensusMvGridDistricts from egon.data.datasets.zensus_vg250 import ZensusVg250 from egon.data.datasets.gas_prod import GasProduction +from egon.data.datasets.industrial_gas_demand import IndustrialGasDemand import airflow import egon.data.importing.zensus as import_zs @@ -236,19 +237,6 @@ heat_demand_Germany = HeatDemandImport( dependencies=[vg250, scenario_parameters, zensus_vg250]) - # Distribute electrical CTS demands to zensus grid - cts_electricity_demand_annual = CtsElectricityDemand( - dependencies=[ - demandregio, - zensus_vg250, - heat_demand_Germany, - etrago_input_data, - household_electricity_demand_annual, - ] - ) - - elec_cts_demands_zensus = tasks[ - 'electricity_demand.distribute-cts-demands'] # Gas grid import gas_grid_insert_data = PythonOperator( @@ -281,6 +269,10 @@ gas_production_insert_data = GasProduction( dependencies=[create_gas_polygons]) + # Insert industrial gas demand + industrial_gas_demand = IndustrialGasDemand( + dependencies=[create_gas_polygons]) + # Extract landuse areas from osm data set create_landuse_table = PythonOperator( task_id="create-landuse-table", @@ -331,6 +323,22 @@ ) map_zensus_grid_districts = tasks["zensus_mv_grid_districts.mapping"] + + # Distribute electrical CTS demands to zensus grid + cts_electricity_demand_annual = CtsElectricityDemand( + dependencies=[ + demandregio, + zensus_vg250, + heat_demand_Germany, + etrago_input_data, + household_electricity_demand_annual, + zensus_mv_grid_districts + ] + ) + + elec_cts_demands_zensus = tasks[ + 'electricity_demand.distribute-cts-demands'] + # Map federal states to mv_grid_districts vg250_mv_grid_districts = Vg250MvGridDistricts( diff --git a/src/egon/data/datasets/demandregio/__init__.py b/src/egon/data/datasets/demandregio/__init__.py index dc5ec0f61..3e47719ce 100644 --- a/src/egon/data/datasets/demandregio/__init__.py +++ b/src/egon/data/datasets/demandregio/__init__.py @@ -2,7 +2,6 @@ adjusting data from demandRegio """ -import os import pandas as pd import numpy as np import egon.data.config @@ -14,6 +13,7 @@ from egon.data.datasets.demandregio.install_disaggregator import ( clone_and_install) from egon.data.datasets import Dataset +from pathlib import Path try: from disaggregator import data, spatial, config @@ -40,7 +40,7 @@ class DemandRegio(Dataset): def __init__(self, dependencies): super().__init__( name="DemandRegio", - version="0.0.0", + version="0.0.1", dependencies=dependencies, tasks=(clone_and_install, create_tables, insert_society_data, insert_household_demand, @@ -168,14 +168,19 @@ def insert_cts_ind_wz_definitions(): engine = db.engine() for sector in source['wz_definitions']: + + file_path = ( + Path(".") / + "data_bundle_egon_data" / + "WZ_definition" / + source['wz_definitions'][sector]) + if sector=='CTS': delimiter=';' else: delimiter=',' df = pd.read_csv( - os.path.join( - "data_bundle_egon_data/WZ_definition/", - source['wz_definitions'][sector]), + file_path, delimiter=delimiter, header=None).rename( {0: 'wz', 1: 'definition'}, @@ -233,10 +238,14 @@ def adjust_cts_ind_nep(ec_cts_ind, sector): sources = (egon.data.config.datasets() ['demandregio_cts_ind_demand']['sources']) + file_path = ( + Path(".") / + "data_bundle_egon_data" / + "nep2035_version2021" / + sources['new_consumers_2035']) + # get data from NEP per federal state - new_con = pd.read_csv(os.path.join( - "data_bundle_egon_data/nep2035_version2021/", - sources['new_consumers_2035']), + new_con = pd.read_csv(file_path, delimiter=';', decimal=',', index_col=0) # match nuts3 regions to federal states diff --git a/src/egon/data/datasets/gas_prod.py b/src/egon/data/datasets/gas_prod.py index e20a7bcaa..fbb2596c9 100755 --- a/src/egon/data/datasets/gas_prod.py +++ b/src/egon/data/datasets/gas_prod.py @@ -2,7 +2,7 @@ """ The central module containing all code dealing with importing gas production data """ -import os +#import os import ast import pandas as pd import geopandas as gpd @@ -10,16 +10,16 @@ import geopandas from egon.data import db -from egon.data.importing.gas_grid import next_id from egon.data.config import settings from egon.data.datasets import Dataset from urllib.request import urlretrieve +from pathlib import Path class GasProduction(Dataset): def __init__(self, dependencies): super().__init__( name="GasProduction", - version="0.0.2", + version="0.0.3", dependencies=dependencies, tasks=(import_gas_generators), ) @@ -34,9 +34,12 @@ def load_NG_generators(): Dataframe containing the natural gas producion units in Germany """ - target_file = os.path.join( - "datasets/gas_data/", - 'data/IGGIELGN_Productions.csv') + target_file = ( + Path(".") / + "datasets" / + "gas_data" / + "data" / + "IGGIELGN_Productions.csv") NG_generators_list = pd.read_csv(target_file, delimiter=';', decimal='.', @@ -100,7 +103,11 @@ def load_biogas_generators(): # Download file basename = "Biogaspartner_Einspeiseatlas_Deutschland_2021.xlsx" url = "https://www.biogaspartner.de/fileadmin/Biogaspartner/Dokumente/Einspeiseatlas/" + basename - target_file = "datasets/gas_data/" + basename + target_file = ( + Path(".") / + "datasets" / + "gas_data" / + basename) urlretrieve(url, target_file) @@ -210,7 +217,7 @@ def import_gas_generators(): ) # Select next id value - new_id = next_id('generator') + new_id = db.next_etrago_id('generator') CH4_generators_list = pd.concat([load_NG_generators(), load_biogas_generators()]) CH4_generators_list['generator_id'] = range(new_id, new_id + len(CH4_generators_list)) diff --git a/src/egon/data/datasets/heat_supply/geothermal.py b/src/egon/data/datasets/heat_supply/geothermal.py index a9f0ea192..10a179524 100644 --- a/src/egon/data/datasets/heat_supply/geothermal.py +++ b/src/egon/data/datasets/heat_supply/geothermal.py @@ -9,6 +9,7 @@ import geopandas as gpd import pandas as pd import numpy as np +from pathlib import Path from egon.data import db, config @@ -24,8 +25,12 @@ def calc_geothermal_potentials(): data={'NDB': 35, 'ORG': 90, 'SMB': 125}, name = 'm_flow') ## geothermal potentials per temperature (p. 94) - potentials = gpd.read_file( - 'data_bundle_egon_data/geothermal_potential/geothermal_potential_germany.shp') + file_path = ( + Path(".") / + "data_bundle_egon_data" / + "geothermal_potential" / + "geothermal_potential_germany.shp") + potentials = gpd.read_file(file_path) ## temperature heating system in °C (p. 95) sys_temp = 60 ## temeprature losses heat recuperator in °C (p. 95) diff --git a/src/egon/data/datasets/industrial_gas_demand.py b/src/egon/data/datasets/industrial_gas_demand.py new file mode 100755 index 000000000..8ce5bfcd9 --- /dev/null +++ b/src/egon/data/datasets/industrial_gas_demand.py @@ -0,0 +1,250 @@ +# -*- coding: utf-8 -*- +""" +The central module containing all code dealing with importing gas industrial demand +""" +import requests +import pandas as pd +import numpy as np + + +from shapely import wkt +from egon.data import db +from egon.data.datasets.gas_prod import assign_gas_bus_id +from egon.data.config import settings +from egon.data.datasets import Dataset + +class IndustrialGasDemand(Dataset): + def __init__(self, dependencies): + super().__init__( + name="IndustrialGasDemand", + version="0.0.0", + dependencies=dependencies, + tasks=(insert_industrial_gas_demand), + ) + +def download_CH4_industrial_demand(): + """Download the CH4 industrial demand in Germany from the FfE open data portal + + Returns + ------- + CH4_industrial_demand : + Dataframe containing the CH4 industrial demand in Germany + + """ + # Download the data and the id_region correspondance table + correspondance_url = 'http://opendata.ffe.de:3000/region?id_region_type=eq.38' + url = 'http://opendata.ffe.de:3000/opendata?id_opendata=eq.66&&year=eq.' + + internal_id = '2,11' # Natural_Gas + year = '2035' # 2050 + datafilter = '&&internal_id=eq.{'+internal_id+'}' + request = url + year + datafilter + + result = requests.get(request) + industrial_loads_list = pd.read_json(result.content) + industrial_loads_list = industrial_loads_list[['id_region', 'values']].copy() + industrial_loads_list = industrial_loads_list.set_index('id_region') + + result_corr = requests.get(correspondance_url) + df_corr = pd.read_json(result_corr.content) + df_corr = df_corr[['id_region', 'name_short']].copy() + df_corr = df_corr.set_index('id_region') + + # Match the id_region to obtain the NUT3 region names + industrial_loads_list = pd.concat([industrial_loads_list, df_corr], axis=1, join="inner") + industrial_loads_list['NUTS0'] = (industrial_loads_list['name_short'].str)[0:2] + industrial_loads_list['NUTS1'] = (industrial_loads_list['name_short'].str)[0:3] + industrial_loads_list = industrial_loads_list[industrial_loads_list['NUTS0'].str.match('DE')] + + # Cut data to federal state if in testmode + boundary = settings()['egon-data']['--dataset-boundary'] + if boundary != 'Everything': + map_states = {'Baden-Württemberg':'DE1', 'Nordrhein-Westfalen': 'DEA', + 'Hessen': 'DE7', 'Brandenburg': 'DE4', 'Bremen':'DE5', + 'Rheinland-Pfalz': 'DEB', 'Sachsen-Anhalt': 'DEE', + 'Schleswig-Holstein':'DEF', 'Mecklenburg-Vorpommern': 'DE8', + 'Thüringen': 'DEG', 'Niedersachsen': 'DE9', + 'Sachsen': 'DED', 'Hamburg': 'DE6', 'Saarland': 'DEC', + 'Berlin': 'DE3', 'Bayern': 'DE2'} + + industrial_loads_list = industrial_loads_list[industrial_loads_list['NUTS1'].isin([map_states[boundary], np.nan])] + + industrial_loads_list = industrial_loads_list.rename(columns={"name_short": "nuts3", "values": "p_set"}) + industrial_loads_list = industrial_loads_list.set_index('nuts3') + + # Add the centroid point to each NUTS3 area + sql_vg250 = """SELECT nuts as nuts3, geometry as geom + FROM boundaries.vg250_krs + WHERE gf = 4 ;""" + gdf_vg250 = db.select_geodataframe(sql_vg250 , epsg=4326) + + point = [] + for index, row in gdf_vg250.iterrows(): + point.append(wkt.loads(str(row['geom'])).centroid) + gdf_vg250['point'] = point + gdf_vg250 = gdf_vg250.set_index('nuts3') + gdf_vg250 = gdf_vg250 .drop(columns=['geom']) + + # Match the load to the NUTS3 points + industrial_loads_list = pd.concat([industrial_loads_list, gdf_vg250], axis=1, join="inner") + industrial_loads_list = industrial_loads_list.rename(columns={'point': 'geom'}).set_geometry('geom', crs=4326) + + # Match to associated gas bus + industrial_loads_list = assign_gas_bus_id(industrial_loads_list) + + # Remove useless columns + industrial_loads_list = industrial_loads_list.drop(columns=['geom', 'NUTS0', 'NUTS1', 'bus_id']) + + return industrial_loads_list + + +def download_H2_industrial_demand(): + """Download the H2 industrial demand in Germany from the FfE open data portal + + Returns + ------- + H2_industrial_demand : + Dataframe containing the H2 industrial demand in Germany + + """ + # Download the data and the id_region correspondance table + correspondance_url = 'http://opendata.ffe.de:3000/region?id_region_type=eq.38' + url = 'http://opendata.ffe.de:3000/opendata?id_opendata=eq.66&&year=eq.' + + internal_id = '2,162' # Hydrogen + year = '2035' # 2050 + datafilter = '&&internal_id=eq.{'+internal_id+'}' + request = url + year + datafilter + + result = requests.get(request) + industrial_loads_list = pd.read_json(result.content) + industrial_loads_list = industrial_loads_list[['id_region', 'values']].copy() + industrial_loads_list = industrial_loads_list.set_index('id_region') + + result_corr = requests.get(correspondance_url) + df_corr = pd.read_json(result_corr.content) + df_corr = df_corr[['id_region', 'name_short']].copy() + df_corr = df_corr.set_index('id_region') + + # Match the id_region to obtain the NUT3 region names + industrial_loads_list = pd.concat([industrial_loads_list, df_corr], axis=1, join="inner") + industrial_loads_list['NUTS0'] = (industrial_loads_list['name_short'].str)[0:2] + industrial_loads_list['NUTS1'] = (industrial_loads_list['name_short'].str)[0:3] + industrial_loads_list = industrial_loads_list[industrial_loads_list['NUTS0'].str.match('DE')] + + # Cut data to federal state if in testmode + boundary = settings()['egon-data']['--dataset-boundary'] + if boundary != 'Everything': + map_states = {'Baden-Württemberg':'DE1', 'Nordrhein-Westfalen': 'DEA', + 'Hessen': 'DE7', 'Brandenburg': 'DE4', 'Bremen':'DE5', + 'Rheinland-Pfalz': 'DEB', 'Sachsen-Anhalt': 'DEE', + 'Schleswig-Holstein':'DEF', 'Mecklenburg-Vorpommern': 'DE8', + 'Thüringen': 'DEG', 'Niedersachsen': 'DE9', + 'Sachsen': 'DED', 'Hamburg': 'DE6', 'Saarland': 'DEC', + 'Berlin': 'DE3', 'Bayern': 'DE2'} + + industrial_loads_list = industrial_loads_list[industrial_loads_list['NUTS1'].isin([map_states[boundary], np.nan])] + + industrial_loads_list = industrial_loads_list.rename(columns={"name_short": "nuts3", "values": "p_set"}) + industrial_loads_list = industrial_loads_list.set_index('nuts3') + + # Add the centroid point to each NUTS3 area + sql_vg250 = """SELECT nuts as nuts3, geometry as geom + FROM boundaries.vg250_krs + WHERE gf = 4 ;""" + gdf_vg250 = db.select_geodataframe(sql_vg250 , epsg=4326) + + point = [] + for index, row in gdf_vg250.iterrows(): + point.append(wkt.loads(str(row['geom'])).centroid) + gdf_vg250['point'] = point + gdf_vg250 = gdf_vg250.set_index('nuts3') + gdf_vg250 = gdf_vg250 .drop(columns=['geom']) + + # Match the load to the NUTS3 points + industrial_loads_list = pd.concat([industrial_loads_list, gdf_vg250], axis=1, join="inner") + industrial_loads_list = industrial_loads_list.rename(columns={'point': 'geom'}).set_geometry('geom', crs=4326) + + # Match to associated gas bus + industrial_loads_list = assign_gas_bus_id(industrial_loads_list) + + # Remove useless columns + industrial_loads_list = industrial_loads_list.drop(columns=['geom', 'NUTS0', 'NUTS1', 'bus_id']) + + return industrial_loads_list + + +def import_industrial_gas_demand(): + """Insert list of industrial gas demand (one per NUTS3) in database + Returns + industrial_gas_demand : Dataframe containing the industrial gas demand in Germany + """ + # Connect to local database + engine = db.engine() + + # Clean table + db.execute_sql( + """ + DELETE FROM grid.egon_etrago_load WHERE "carrier" = 'gas'; + """ + ) + + # Select next id value + new_id = db.next_etrago_id('load') + + industrial_gas_demand = pd.concat([download_CH4_industrial_demand(), download_H2_industrial_demand()]) + industrial_gas_demand['load_id'] = range(new_id, new_id + len(industrial_gas_demand)) + + # Add missing columns + c = {'scn_name':'eGon2035', 'carrier':'gas'} + industrial_gas_demand = industrial_gas_demand.assign(**c) + + industrial_gas_demand = industrial_gas_demand.reset_index(drop=True) + + # Remove useless columns + egon_etrago_load_gas = industrial_gas_demand.drop(columns=['p_set']) + + # Insert data to db + egon_etrago_load_gas.to_sql('egon_etrago_load', + engine, + schema ='grid', + index = False, + if_exists = 'append') + + return industrial_gas_demand + + +def import_industrial_gas_demand_time_series(egon_etrago_load_gas): + """Insert list of industrial gas demand time series (one per NUTS3) in database + Returns + ------- + None. + """ + egon_etrago_load_gas_timeseries = egon_etrago_load_gas + + # Connect to local database + engine = db.engine() + + # Adjust columns + egon_etrago_load_gas_timeseries = egon_etrago_load_gas_timeseries.drop(columns=['carrier', 'bus']) + egon_etrago_load_gas_timeseries['temp_id'] = 1 + + # Insert data to db + egon_etrago_load_gas_timeseries.to_sql('egon_etrago_load_timeseries', + engine, + schema ='grid', + index = False, + if_exists = 'append') + + +def insert_industrial_gas_demand(): + """Overall function for inserting the industrial gas demand + Returns + ------- + None. + """ + + egon_etrago_load_gas = import_industrial_gas_demand() + + import_industrial_gas_demand_time_series(egon_etrago_load_gas) + \ No newline at end of file diff --git a/src/egon/data/datasets/re_potential_areas/__init__.py b/src/egon/data/datasets/re_potential_areas/__init__.py index 3d321935d..69ac83efd 100644 --- a/src/egon/data/datasets/re_potential_areas/__init__.py +++ b/src/egon/data/datasets/re_potential_areas/__init__.py @@ -4,6 +4,7 @@ from functools import partial from urllib.request import urlretrieve +from pathlib import Path import os from geoalchemy2 import Geometry @@ -57,10 +58,16 @@ def ve(s): else ve(f"'{dataset}' is not a valid dataset boundary.") ) + download_directory = Path(".") / "re_potential_areas" + # Create the folder, if it does not exists already + if not os.path.exists(download_directory): + os.mkdir(download_directory) + url_target_file_map = zip( pa_config["original_data"]["source"][url_section], [ - os.path.join(os.path.dirname(__file__), file) + # os.path.join(os.path.dirname(__file__), file) + Path(".") / "re_potential_areas" / Path(file).name for file in pa_config["original_data"]["target"][ "path_table_map" ].keys() @@ -106,7 +113,7 @@ def insert_data(): pa_config = data_config["re_potential_areas"] file_table_map = { - os.path.join(os.path.dirname(__file__), file): table + Path(".") / "re_potential_areas" / Path(file).name: table for file, table in pa_config["original_data"]["target"][ "path_table_map" ].items() diff --git a/src/egon/data/datasets/scenario_capacities.py b/src/egon/data/datasets/scenario_capacities.py index 76dd96e1b..528cb0b6f 100755 --- a/src/egon/data/datasets/scenario_capacities.py +++ b/src/egon/data/datasets/scenario_capacities.py @@ -2,7 +2,6 @@ Netzentwicklungsplan 2035, Version 2031, Szenario C """ -import os import egon.data.config import pandas as pd import numpy as np @@ -12,6 +11,7 @@ from sqlalchemy import Column, String, Float, Integer from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import sessionmaker +from pathlib import Path ### will be later imported from another file ### Base = declarative_base() @@ -55,7 +55,7 @@ class ScenarioCapacities(Dataset): def __init__(self, dependencies): super().__init__( name="ScenarioCapacities", - version="0.0.2", + version="0.0.3", dependencies=dependencies, tasks=( create_table, @@ -118,8 +118,10 @@ def insert_capacities_per_federal_state_nep(): "AND nuts != 'DE'") # read-in installed capacities per federal state of germany - target_file = os.path.join( - "data_bundle_egon_data/nep2035_version2021", + target_file = ( + Path(".") / + "data_bundle_egon_data" / + "nep2035_version2021" / scenario_config('eGon2035')['paths']['capacities']) df = pd.read_excel(target_file, sheet_name='1.Entwurf_NEP2035_V2021', @@ -235,9 +237,12 @@ def insert_nep_list_powerplants(): engine = db.engine() # Read-in data from csv-file - target_file = os.path.join( - "data_bundle_egon_data/nep2035_version2021/", + target_file = ( + Path(".") / + "data_bundle_egon_data" / + "nep2035_version2021" / scenario_config('eGon2035')['paths']['list_conv_pp']) + kw_liste_nep = pd.read_csv(target_file, delimiter=';', decimal=',') @@ -298,8 +303,10 @@ def district_heating_input(): """ # import data to dataframe - file = os.path.join( - "data_bundle_egon_data/nep2035_version2021", + file = ( + Path(".") / + "data_bundle_egon_data" / + "nep2035_version2021" / scenario_config('eGon2035')['paths']['capacities']) df = pd.read_excel(file, sheet_name='Kurzstudie_KWK', dtype={'Wert':float}) df.set_index(['Energietraeger', 'Name'], inplace=True) diff --git a/src/egon/data/importing/gas_grid.py b/src/egon/data/importing/gas_grid.py index 34bcfc8c6..88374c1be 100755 --- a/src/egon/data/importing/gas_grid.py +++ b/src/egon/data/importing/gas_grid.py @@ -16,50 +16,35 @@ from urllib.request import urlretrieve from zipfile import ZipFile from geoalchemy2.shape import from_shape - -def next_id(component): - """ Select next id value for components in pf-tables - Parameters - ---------- - component : str - Name of componenet - Returns - ------- - next_id : int - Next index value - """ - max_id = db.select_dataframe( - f""" - SELECT MAX({component}_id) FROM grid.egon_etrago_{component} - """)['max'][0] - - if max_id: - next_id = max_id + 1 - else: - next_id = 1 - - return next_id - +from pathlib import Path def download_SciGRID_gas_data(): """ Download SciGRID_gas IGGIELGN data from Zenodo """ - path = "datasets/gas_data/" + path = ( + Path(".") / + "datasets" / + "gas_data" ) os.makedirs(path, exist_ok=True) basename = "IGGIELGN" + zip_file = ( + Path(".") / + "datasets" / + "gas_data" / + "IGGIELGN.zip") zenodo_zip_file_url = ("https://zenodo.org/record/4767098/files/"+ basename + ".zip") - if not os.path.isfile(path + basename + ".zip"): - urlretrieve(zenodo_zip_file_url, path + basename + ".zip") + if not os.path.isfile(zip_file): + urlretrieve(zenodo_zip_file_url, zip_file) components = ['Nodes', 'PipeSegments', 'Productions', 'Storages'] #'Compressors' files = [] for i in components: files.append('data/'+ basename + "_" + i + ".csv") - with ZipFile(path + basename + ".zip", 'r') as zipObj: + with ZipFile(zip_file, 'r') as zipObj: listOfFileNames = zipObj.namelist() for fileName in listOfFileNames: if fileName in files: @@ -76,11 +61,14 @@ def define_gas_nodes_list(): """ # Select next id value - new_id = next_id('bus') + new_id = db.next_etrago_id('bus') - target_file = os.path.join( - "datasets/gas_data/data/", - 'IGGIELGN_Nodes.csv') + target_file = ( + Path(".") / + "datasets" / + "gas_data" / + "data" / + "IGGIELGN_Nodes.csv") gas_nodes_list = pd.read_csv(target_file, delimiter=';', decimal='.', @@ -173,19 +161,24 @@ def insert_gas_pipeline_list(gas_nodes_list): engine = db.engine() # Select next id value - new_id = next_id('link') + new_id = db.next_etrago_id('link') - classifiaction_file = os.path.join( - "data_bundle_egon_data/pipeline_classification_gas/", - 'pipeline_classification.csv') + classifiaction_file = ( + Path(".") / + "data_bundle_egon_data" / + "pipeline_classification_gas" / + "pipeline_classification.csv") classification = pd.read_csv(classifiaction_file, delimiter=',', usecols = ['classification', 'max_transport_capacity_Gwh/d']) - target_file = os.path.join( - "datasets/gas_data/data/", - 'IGGIELGN_PipeSegments.csv') + target_file = ( + Path(".") / + "datasets" / + "gas_data" / + "data" / + "IGGIELGN_PipeSegments.csv") gas_pipelines_list = pd.read_csv(target_file, delimiter=';', decimal='.', diff --git a/src/egon/data/processing/calculate_dlr.py b/src/egon/data/processing/calculate_dlr.py index df835f044..150ac09e8 100644 --- a/src/egon/data/processing/calculate_dlr.py +++ b/src/egon/data/processing/calculate_dlr.py @@ -13,7 +13,7 @@ import rioxarray from shapely.geometry import Point import psycopg2 - +from pathlib import Path def Calculate_DLR(): """Calculate DLR and assign values to each line in the db @@ -24,10 +24,14 @@ def Calculate_DLR(): """ - weather_info_path = "cutouts/europe-2011-era5/201101.nc" + weather_info_path = ( + Path(".") / "cutouts" / "germany-2011-era5.nc") + regions_shape_path = ( - "data_bundle_egon_data/regions_dynamic_line_rating/Germany_regions.shp" - ) + Path(".") / + "data_bundle_egon_data" / + "regions_dynamic_line_rating" / + "Germany_regions.shp") # Calculate hourly DLR per region dlr_hourly_dic, dlr_hourly = DLR_Regions(weather_info_path, regions_shape_path) @@ -131,7 +135,8 @@ def DLR_Regions(weather_info_path, regions_shape_path): regions = regions.sort_values(by=["Region"]) # The data downloaded using Atlite is loaded in 'weather_data_raw'. - weather_data_raw = xr.open_mfdataset("cutouts/germany-2011-era5.nc") + path = (Path(".") / "cutouts" / "germany-2011-era5.nc") + weather_data_raw = xr.open_mfdataset(str(path)) weather_data_raw = weather_data_raw.rio.write_crs(4326) weather_data_raw = weather_data_raw.rio.clip_box( minx=5.5, diff --git a/src/egon/data/processing/power2gas.py b/src/egon/data/processing/power2gas.py index 5e7085e58..efd2eeb56 100755 --- a/src/egon/data/processing/power2gas.py +++ b/src/egon/data/processing/power2gas.py @@ -10,7 +10,7 @@ from geoalchemy2.types import Geometry from scipy.spatial import cKDTree from egon.data import db -from egon.data.importing.gas_grid import next_id +#from egon.data.importing.gas_grid import next_id def insert_power2gas(): """Function defining the potential power-to-gas capacities and inserting them in the etrago_link table. @@ -24,7 +24,7 @@ def insert_power2gas(): engine = db.engine() # Select next id value - new_id = next_id('link') + new_id = db.next_etrago_id('link') # Create dataframes containing all gas buses and all the HV power buses sql_AC = """SELECT bus_id, geom