From d577531262c055a35e0a11a55e45ba5868f7531b Mon Sep 17 00:00:00 2001 From: AmeliaNadal Date: Mon, 16 Aug 2021 14:08:29 +0200 Subject: [PATCH 01/13] Resolve conflict in pipeline.py --- src/egon/data/airflow/dags/pipeline.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/egon/data/airflow/dags/pipeline.py b/src/egon/data/airflow/dags/pipeline.py index 713cde0bd..f4d036f6f 100755 --- a/src/egon/data/airflow/dags/pipeline.py +++ b/src/egon/data/airflow/dags/pipeline.py @@ -34,6 +34,7 @@ from egon.data.datasets.zensus_mv_grid_districts import ZensusMvGridDistricts from egon.data.datasets.zensus_vg250 import ZensusVg250 from egon.data.datasets.gas_prod import GasProduction +from egon.data.datasets.industrial_gas_demand import IndustrialGasDemand import airflow import egon.data.importing.heat_demand_data as import_hd @@ -293,6 +294,10 @@ gas_production_insert_data = GasProduction( dependencies=[create_gas_polygons]) + # Insert industrial gas demand + industrial_gas_demand = IndustrialGasDemand( + dependencies=[create_gas_polygons]) + # Extract landuse areas from osm data set create_landuse_table = PythonOperator( task_id="create-landuse-table", From 9a3ae4fbe8a75154b2cebc4a44fff6b97720da7d Mon Sep 17 00:00:00 2001 From: AmeliaNadal Date: Tue, 17 Aug 2021 09:17:51 +0200 Subject: [PATCH 02/13] Temporary changes in gas_prod --- src/egon/data/datasets/gas_prod.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/egon/data/datasets/gas_prod.py b/src/egon/data/datasets/gas_prod.py index 14bfcb29c..c4804ee28 100755 --- a/src/egon/data/datasets/gas_prod.py +++ b/src/egon/data/datasets/gas_prod.py @@ -15,14 +15,14 @@ from egon.data.datasets import Dataset from urllib.request import urlretrieve -class GasProduction(Dataset): - def __init__(self, dependencies): - super().__init__( - name="GasProduction", - version="0.0.0", - dependencies=dependencies, - tasks=(import_gas_generators), - ) +class GasProduction(Dataset): + def __init__(self, dependencies): + super().__init__( + name="GasProduction", + version="0.0.1.dev", + dependencies=dependencies, + tasks=(import_gas_generators), + ) def load_NG_generators(): @@ -144,6 +144,7 @@ def load_biogas_generators(): AND ST_Contains(ST_Transform(vg.geometry,4326), egon_biogas_generator.geom)''' biogas_generators_list = gpd.GeoDataFrame.from_postgis(sql, con=engine, geom_col="geom", crs=4326) + biogas_generators_list = biogas_generators_list.drop(columns=['gid', 'bez', 'area_ha', 'geometry']) db.execute_sql( """ DROP TABLE IF EXISTS grid.egon_biogas_generator CASCADE; @@ -155,12 +156,11 @@ def load_biogas_generators(): Total_biogas_capacity_2035 = Total_biogas_extracted_2035 * 1000000 / (24 * 365) # Regionalization of the production - biogas_generators_list['p_nom'] = (biogas_generators_list['Einspeisung Biomethan [(N*m^3)/h)]'] - / biogas_generators_list['Einspeisung Biomethan [(N*m^3)/h)]'].sum() - * Total_biogas_capacity_2035) - # Remove useless columns - biogas_generators_list = biogas_generators_list.drop(columns=['x', 'y', 'gid', 'bez', 'Koordinaten', - 'area_ha', 'geometry', + biogas_generators_list['p_nom'] = (biogas_generators_list['Einspeisung Biomethan [(N*m^3)/h)]'] + / biogas_generators_list['Einspeisung Biomethan [(N*m^3)/h)]'].sum() + * Total_biogas_capacity_2035) + # Remove useless columns + biogas_generators_list = biogas_generators_list.drop(columns=['x', 'y', 'Koordinaten', 'Einspeisung Biomethan [(N*m^3)/h)]']) return biogas_generators_list From a7a84f455fdefc2902c3cd8f2903fd7a97bf214f Mon Sep 17 00:00:00 2001 From: AmeliaNadal Date: Wed, 18 Aug 2021 14:29:48 +0200 Subject: [PATCH 03/13] Update CHANGELOG --- CHANGELOG.rst | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index f2b67f08d..f0ca11aee 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -124,6 +124,8 @@ Added `#237 `_ * Merge electrical loads per bus and export to etrago tables `#328 `_ +* Insert industial gas demand + `#321 `_ .. _PR #159: https://github.com/openego/eGon-data/pull/159 @@ -181,6 +183,10 @@ Changed `#326 `_ * Migrate osmTGmod to datasets `#305 `_ +* Filter osm landuse areas, rename industrial sites tables and update load curve function + `#378 `_ +* Remove version columns from eTraGo tables and related code + `#384 `_ Bug fixes --------- @@ -218,4 +224,9 @@ Bug fixes * Set current working directory as java's temp dir when executing osmosis `#344 `_ * Fix border gas voronoi polygons which had no bus_id -`#362 `_ + `#362 `_ +* Add dependency from WeatherData to Vg250 + `#387 `_ +* Fix unnecessary columns in normal mode for inserting the gas production + `#387 `_ + From b055dae8743d0eefcdb0232ae2e73fccb1c28846 Mon Sep 17 00:00:00 2001 From: AmeliaNadal Date: Wed, 18 Aug 2021 14:31:44 +0200 Subject: [PATCH 04/13] Insert industrial gas demand load and associated time series --- .../data/datasets/industrial_gas_demand.py | 254 ++++++++++++++++++ 1 file changed, 254 insertions(+) create mode 100755 src/egon/data/datasets/industrial_gas_demand.py diff --git a/src/egon/data/datasets/industrial_gas_demand.py b/src/egon/data/datasets/industrial_gas_demand.py new file mode 100755 index 000000000..347160e95 --- /dev/null +++ b/src/egon/data/datasets/industrial_gas_demand.py @@ -0,0 +1,254 @@ +# -*- coding: utf-8 -*- +""" +The central module containing all code dealing with importing gas industrial demand +""" +import requests +import os +import ast +import pandas as pd +import geopandas as gpd +import numpy as np +import geopandas + +from shapely import geometry, wkt +from egon.data import db +from egon.data.importing.gas_grid import next_id +from egon.data.datasets.gas_prod import assign_gas_bus_id +from egon.data.config import settings +from egon.data.datasets import Dataset + +class IndustrialGasDemand(Dataset): + def __init__(self, dependencies): + super().__init__( + name="IndustrialGasDemand", + version="0.0.0.dev", + dependencies=dependencies, + tasks=(insert_industrial_gas_demand), + ) + +def download_CH4_industrial_demand(): + """Download the CH4 industrial demand in Germany from the FfE open data portal + + Returns + ------- + CH4_industrial_demand : + Dataframe containing the CH4 industrial demand in Germany + + """ + # Download the data and the id_region correspondance table + correspondance_url = 'http://opendata.ffe.de:3000/region?id_region_type=eq.38' + url = 'http://opendata.ffe.de:3000/opendata?id_opendata=eq.66&&year=eq.' + + internal_id = '2,11' # Natural_Gas + year = '2035' # 2050 + datafilter = '&&internal_id=eq.{'+internal_id+'}' + request = url + year + datafilter + + result = requests.get(request) + industrial_loads_list = pd.read_json(result.content) + industrial_loads_list = industrial_loads_list[['id_region', 'values']].copy() + industrial_loads_list = industrial_loads_list.set_index('id_region') + + result_corr = requests.get(correspondance_url) + df_corr = pd.read_json(result_corr.content) + df_corr = df_corr[['id_region', 'name_short']].copy() + df_corr = df_corr.set_index('id_region') + + # Match the id_region to obtain the NUT3 region names + industrial_loads_list = pd.concat([industrial_loads_list, df_corr], axis=1, join="inner") + industrial_loads_list['NUTS0'] = (industrial_loads_list['name_short'].str)[0:2] + industrial_loads_list['NUTS1'] = (industrial_loads_list['name_short'].str)[0:3] + industrial_loads_list = industrial_loads_list[industrial_loads_list['NUTS0'].str.match('DE')] + + # Cut data to federal state if in testmode + boundary = settings()['egon-data']['--dataset-boundary'] + if boundary != 'Everything': + map_states = {'Baden-Württemberg':'DE1', 'Nordrhein-Westfalen': 'DEA', + 'Hessen': 'DE7', 'Brandenburg': 'DE4', 'Bremen':'DE5', + 'Rheinland-Pfalz': 'DEB', 'Sachsen-Anhalt': 'DEE', + 'Schleswig-Holstein':'DEF', 'Mecklenburg-Vorpommern': 'DE8', + 'Thüringen': 'DEG', 'Niedersachsen': 'DE9', + 'Sachsen': 'DED', 'Hamburg': 'DE6', 'Saarland': 'DEC', + 'Berlin': 'DE3', 'Bayern': 'DE2'} + + industrial_loads_list = industrial_loads_list[industrial_loads_list['NUTS1'].isin([map_states[boundary], np.nan])] + + industrial_loads_list = industrial_loads_list.rename(columns={"name_short": "nuts3", "values": "p_set"}) + industrial_loads_list = industrial_loads_list.set_index('nuts3') + + # Add the centroid point to each NUTS3 area + sql_vg250 = """SELECT nuts as nuts3, geometry as geom + FROM boundaries.vg250_krs + WHERE gf = 4 ;""" + gdf_vg250 = db.select_geodataframe(sql_vg250 , epsg=4326) + + point = [] + for index, row in gdf_vg250.iterrows(): + point.append(wkt.loads(str(row['geom'])).centroid) + gdf_vg250['point'] = point + gdf_vg250 = gdf_vg250.set_index('nuts3') + gdf_vg250 = gdf_vg250 .drop(columns=['geom']) + + # Match the load to the NUTS3 points + industrial_loads_list = pd.concat([industrial_loads_list, gdf_vg250], axis=1, join="inner") + industrial_loads_list = industrial_loads_list.rename(columns={'point': 'geom'}).set_geometry('geom', crs=4326) + + # Match to associated gas bus + industrial_loads_list = assign_gas_bus_id(industrial_loads_list) + + # Remove useless columns + industrial_loads_list = industrial_loads_list.drop(columns=['geom', 'NUTS0', 'NUTS1', 'bus_id']) + + return industrial_loads_list + + +def download_H2_industrial_demand(): + """Download the H2 industrial demand in Germany from the FfE open data portal + + Returns + ------- + H2_industrial_demand : + Dataframe containing the H2 industrial demand in Germany + + """ + # Download the data and the id_region correspondance table + correspondance_url = 'http://opendata.ffe.de:3000/region?id_region_type=eq.38' + url = 'http://opendata.ffe.de:3000/opendata?id_opendata=eq.66&&year=eq.' + + internal_id = '2,162' # Hydrogen + year = '2035' # 2050 + datafilter = '&&internal_id=eq.{'+internal_id+'}' + request = url + year + datafilter + + result = requests.get(request) + industrial_loads_list = pd.read_json(result.content) + industrial_loads_list = industrial_loads_list[['id_region', 'values']].copy() + industrial_loads_list = industrial_loads_list.set_index('id_region') + + result_corr = requests.get(correspondance_url) + df_corr = pd.read_json(result_corr.content) + df_corr = df_corr[['id_region', 'name_short']].copy() + df_corr = df_corr.set_index('id_region') + + # Match the id_region to obtain the NUT3 region names + industrial_loads_list = pd.concat([industrial_loads_list, df_corr], axis=1, join="inner") + industrial_loads_list['NUTS0'] = (industrial_loads_list['name_short'].str)[0:2] + industrial_loads_list['NUTS1'] = (industrial_loads_list['name_short'].str)[0:3] + industrial_loads_list = industrial_loads_list[industrial_loads_list['NUTS0'].str.match('DE')] + + # Cut data to federal state if in testmode + boundary = settings()['egon-data']['--dataset-boundary'] + if boundary != 'Everything': + map_states = {'Baden-Württemberg':'DE1', 'Nordrhein-Westfalen': 'DEA', + 'Hessen': 'DE7', 'Brandenburg': 'DE4', 'Bremen':'DE5', + 'Rheinland-Pfalz': 'DEB', 'Sachsen-Anhalt': 'DEE', + 'Schleswig-Holstein':'DEF', 'Mecklenburg-Vorpommern': 'DE8', + 'Thüringen': 'DEG', 'Niedersachsen': 'DE9', + 'Sachsen': 'DED', 'Hamburg': 'DE6', 'Saarland': 'DEC', + 'Berlin': 'DE3', 'Bayern': 'DE2'} + + industrial_loads_list = industrial_loads_list[industrial_loads_list['NUTS1'].isin([map_states[boundary], np.nan])] + + industrial_loads_list = industrial_loads_list.rename(columns={"name_short": "nuts3", "values": "p_set"}) + industrial_loads_list = industrial_loads_list.set_index('nuts3') + + # Add the centroid point to each NUTS3 area + sql_vg250 = """SELECT nuts as nuts3, geometry as geom + FROM boundaries.vg250_krs + WHERE gf = 4 ;""" + gdf_vg250 = db.select_geodataframe(sql_vg250 , epsg=4326) + + point = [] + for index, row in gdf_vg250.iterrows(): + point.append(wkt.loads(str(row['geom'])).centroid) + gdf_vg250['point'] = point + gdf_vg250 = gdf_vg250.set_index('nuts3') + gdf_vg250 = gdf_vg250 .drop(columns=['geom']) + + # Match the load to the NUTS3 points + industrial_loads_list = pd.concat([industrial_loads_list, gdf_vg250], axis=1, join="inner") + industrial_loads_list = industrial_loads_list.rename(columns={'point': 'geom'}).set_geometry('geom', crs=4326) + + # Match to associated gas bus + industrial_loads_list = assign_gas_bus_id(industrial_loads_list) + + # Remove useless columns + industrial_loads_list = industrial_loads_list.drop(columns=['geom', 'NUTS0', 'NUTS1', 'bus_id']) + + return industrial_loads_list + + +def import_industrial_gas_demand(): + """Insert list of industrial gas demand (one per NUTS3) in database + Returns + industrial_gas_demand : Dataframe containing the industrial gas demand in Germany + """ + # Connect to local database + engine = db.engine() + + # Clean table + db.execute_sql( + """ + DELETE FROM grid.egon_etrago_load WHERE "carrier" = 'gas'; + """ + ) + + # Select next id value + new_id = next_id('load') + + industrial_gas_demand = pd.concat([download_CH4_industrial_demand(), download_H2_industrial_demand()]) + industrial_gas_demand['load_id'] = range(new_id, new_id + len(industrial_gas_demand)) + + # Add missing columns + c = {'version':'0.0.0','scn_name':'eGon2035', 'carrier':'gas'} + industrial_gas_demand = industrial_gas_demand.assign(**c) + + industrial_gas_demand = industrial_gas_demand.reset_index(drop=True) + + # Remove useless columns + egon_etrago_load_gas = industrial_gas_demand.drop(columns=['p_set']) + + # Insert data to db + egon_etrago_load_gas.to_sql('egon_etrago_load', + engine, + schema ='grid', + index = False, + if_exists = 'append') + + return industrial_gas_demand + + +def import_industrial_gas_demand_time_series(egon_etrago_load_gas): + """Insert list of industrial gas demand time series (one per NUTS3) in database + Returns + ------- + None. + """ + egon_etrago_load_gas_timeseries = egon_etrago_load_gas + + # Connect to local database + engine = db.engine() + + # Adjust columns + egon_etrago_load_gas_timeseries = egon_etrago_load_gas_timeseries.drop(columns=['carrier', 'bus']) + egon_etrago_load_gas_timeseries['temp_id'] = 1 + + # Insert data to db + egon_etrago_load_gas_timeseries.to_sql('egon_etrago_load_timeseries', + engine, + schema ='grid', + index = False, + if_exists = 'append') + + +def insert_industrial_gas_demand(): + """Overall function for inserting the industrial gas demand + Returns + ------- + None. + """ + + egon_etrago_load_gas = import_industrial_gas_demand() + + import_industrial_gas_demand_time_series(egon_etrago_load_gas) + \ No newline at end of file From 0ea52f8ca6d6206856ad9319a544bafee0e485c6 Mon Sep 17 00:00:00 2001 From: AmeliaNadal Date: Wed, 18 Aug 2021 16:00:24 +0200 Subject: [PATCH 05/13] Remove version column --- src/egon/data/datasets/industrial_gas_demand.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/egon/data/datasets/industrial_gas_demand.py b/src/egon/data/datasets/industrial_gas_demand.py index 347160e95..7b2c41156 100755 --- a/src/egon/data/datasets/industrial_gas_demand.py +++ b/src/egon/data/datasets/industrial_gas_demand.py @@ -200,7 +200,7 @@ def import_industrial_gas_demand(): industrial_gas_demand['load_id'] = range(new_id, new_id + len(industrial_gas_demand)) # Add missing columns - c = {'version':'0.0.0','scn_name':'eGon2035', 'carrier':'gas'} + c = {'scn_name':'eGon2035', 'carrier':'gas'} industrial_gas_demand = industrial_gas_demand.assign(**c) industrial_gas_demand = industrial_gas_demand.reset_index(drop=True) From d8d78fd138a83000580a7c30ce508576cfd8b5df Mon Sep 17 00:00:00 2001 From: AmeliaNadal Date: Wed, 18 Aug 2021 16:27:48 +0200 Subject: [PATCH 06/13] Replace next_id function by db.next_etrago_id --- src/egon/data/datasets/gas_prod.py | 3 +- .../data/datasets/industrial_gas_demand.py | 12 +++----- src/egon/data/importing/gas_grid.py | 28 ++----------------- src/egon/data/processing/power2gas.py | 4 +-- 4 files changed, 9 insertions(+), 38 deletions(-) diff --git a/src/egon/data/datasets/gas_prod.py b/src/egon/data/datasets/gas_prod.py index e20a7bcaa..907334dfa 100755 --- a/src/egon/data/datasets/gas_prod.py +++ b/src/egon/data/datasets/gas_prod.py @@ -10,7 +10,6 @@ import geopandas from egon.data import db -from egon.data.importing.gas_grid import next_id from egon.data.config import settings from egon.data.datasets import Dataset from urllib.request import urlretrieve @@ -210,7 +209,7 @@ def import_gas_generators(): ) # Select next id value - new_id = next_id('generator') + new_id = db.next_etrago_id('generator') CH4_generators_list = pd.concat([load_NG_generators(), load_biogas_generators()]) CH4_generators_list['generator_id'] = range(new_id, new_id + len(CH4_generators_list)) diff --git a/src/egon/data/datasets/industrial_gas_demand.py b/src/egon/data/datasets/industrial_gas_demand.py index 7b2c41156..8ce5bfcd9 100755 --- a/src/egon/data/datasets/industrial_gas_demand.py +++ b/src/egon/data/datasets/industrial_gas_demand.py @@ -3,16 +3,12 @@ The central module containing all code dealing with importing gas industrial demand """ import requests -import os -import ast import pandas as pd -import geopandas as gpd import numpy as np -import geopandas -from shapely import geometry, wkt + +from shapely import wkt from egon.data import db -from egon.data.importing.gas_grid import next_id from egon.data.datasets.gas_prod import assign_gas_bus_id from egon.data.config import settings from egon.data.datasets import Dataset @@ -21,7 +17,7 @@ class IndustrialGasDemand(Dataset): def __init__(self, dependencies): super().__init__( name="IndustrialGasDemand", - version="0.0.0.dev", + version="0.0.0", dependencies=dependencies, tasks=(insert_industrial_gas_demand), ) @@ -194,7 +190,7 @@ def import_industrial_gas_demand(): ) # Select next id value - new_id = next_id('load') + new_id = db.next_etrago_id('load') industrial_gas_demand = pd.concat([download_CH4_industrial_demand(), download_H2_industrial_demand()]) industrial_gas_demand['load_id'] = range(new_id, new_id + len(industrial_gas_demand)) diff --git a/src/egon/data/importing/gas_grid.py b/src/egon/data/importing/gas_grid.py index 34bcfc8c6..109f6417a 100755 --- a/src/egon/data/importing/gas_grid.py +++ b/src/egon/data/importing/gas_grid.py @@ -17,30 +17,6 @@ from zipfile import ZipFile from geoalchemy2.shape import from_shape -def next_id(component): - """ Select next id value for components in pf-tables - Parameters - ---------- - component : str - Name of componenet - Returns - ------- - next_id : int - Next index value - """ - max_id = db.select_dataframe( - f""" - SELECT MAX({component}_id) FROM grid.egon_etrago_{component} - """)['max'][0] - - if max_id: - next_id = max_id + 1 - else: - next_id = 1 - - return next_id - - def download_SciGRID_gas_data(): """ Download SciGRID_gas IGGIELGN data from Zenodo @@ -76,7 +52,7 @@ def define_gas_nodes_list(): """ # Select next id value - new_id = next_id('bus') + new_id = db.next_etrago_id('bus') target_file = os.path.join( "datasets/gas_data/data/", @@ -173,7 +149,7 @@ def insert_gas_pipeline_list(gas_nodes_list): engine = db.engine() # Select next id value - new_id = next_id('link') + new_id = db.next_etrago_id('link') classifiaction_file = os.path.join( "data_bundle_egon_data/pipeline_classification_gas/", diff --git a/src/egon/data/processing/power2gas.py b/src/egon/data/processing/power2gas.py index 5e7085e58..efd2eeb56 100755 --- a/src/egon/data/processing/power2gas.py +++ b/src/egon/data/processing/power2gas.py @@ -10,7 +10,7 @@ from geoalchemy2.types import Geometry from scipy.spatial import cKDTree from egon.data import db -from egon.data.importing.gas_grid import next_id +#from egon.data.importing.gas_grid import next_id def insert_power2gas(): """Function defining the potential power-to-gas capacities and inserting them in the etrago_link table. @@ -24,7 +24,7 @@ def insert_power2gas(): engine = db.engine() # Select next id value - new_id = next_id('link') + new_id = db.next_etrago_id('link') # Create dataframes containing all gas buses and all the HV power buses sql_AC = """SELECT bus_id, geom From e0de29fc09221d9abd33b19ce167dec1fc80beb5 Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Thu, 19 Aug 2021 16:21:13 +0200 Subject: [PATCH 07/13] Use Path object to define paths --- .../data/datasets/demandregio/__init__.py | 25 +++++++++++++------ .../data/datasets/heat_supply/__init__.py | 2 +- .../data/datasets/heat_supply/geothermal.py | 9 +++++-- src/egon/data/datasets/scenario_capacities.py | 23 +++++++++++------ src/egon/data/processing/calculate_dlr.py | 15 +++++++---- 5 files changed, 50 insertions(+), 24 deletions(-) diff --git a/src/egon/data/datasets/demandregio/__init__.py b/src/egon/data/datasets/demandregio/__init__.py index dc5ec0f61..3e47719ce 100644 --- a/src/egon/data/datasets/demandregio/__init__.py +++ b/src/egon/data/datasets/demandregio/__init__.py @@ -2,7 +2,6 @@ adjusting data from demandRegio """ -import os import pandas as pd import numpy as np import egon.data.config @@ -14,6 +13,7 @@ from egon.data.datasets.demandregio.install_disaggregator import ( clone_and_install) from egon.data.datasets import Dataset +from pathlib import Path try: from disaggregator import data, spatial, config @@ -40,7 +40,7 @@ class DemandRegio(Dataset): def __init__(self, dependencies): super().__init__( name="DemandRegio", - version="0.0.0", + version="0.0.1", dependencies=dependencies, tasks=(clone_and_install, create_tables, insert_society_data, insert_household_demand, @@ -168,14 +168,19 @@ def insert_cts_ind_wz_definitions(): engine = db.engine() for sector in source['wz_definitions']: + + file_path = ( + Path(".") / + "data_bundle_egon_data" / + "WZ_definition" / + source['wz_definitions'][sector]) + if sector=='CTS': delimiter=';' else: delimiter=',' df = pd.read_csv( - os.path.join( - "data_bundle_egon_data/WZ_definition/", - source['wz_definitions'][sector]), + file_path, delimiter=delimiter, header=None).rename( {0: 'wz', 1: 'definition'}, @@ -233,10 +238,14 @@ def adjust_cts_ind_nep(ec_cts_ind, sector): sources = (egon.data.config.datasets() ['demandregio_cts_ind_demand']['sources']) + file_path = ( + Path(".") / + "data_bundle_egon_data" / + "nep2035_version2021" / + sources['new_consumers_2035']) + # get data from NEP per federal state - new_con = pd.read_csv(os.path.join( - "data_bundle_egon_data/nep2035_version2021/", - sources['new_consumers_2035']), + new_con = pd.read_csv(file_path, delimiter=';', decimal=',', index_col=0) # match nuts3 regions to federal states diff --git a/src/egon/data/datasets/heat_supply/__init__.py b/src/egon/data/datasets/heat_supply/__init__.py index 461269828..17387a2eb 100644 --- a/src/egon/data/datasets/heat_supply/__init__.py +++ b/src/egon/data/datasets/heat_supply/__init__.py @@ -111,7 +111,7 @@ class HeatSupply(Dataset): def __init__(self, dependencies): super().__init__( name="HeatSupply", - version="0.0.0", + version="0.0.1", dependencies=dependencies, tasks=(create_tables, district_heating, individual_heating, potential_germany), diff --git a/src/egon/data/datasets/heat_supply/geothermal.py b/src/egon/data/datasets/heat_supply/geothermal.py index a9f0ea192..10a179524 100644 --- a/src/egon/data/datasets/heat_supply/geothermal.py +++ b/src/egon/data/datasets/heat_supply/geothermal.py @@ -9,6 +9,7 @@ import geopandas as gpd import pandas as pd import numpy as np +from pathlib import Path from egon.data import db, config @@ -24,8 +25,12 @@ def calc_geothermal_potentials(): data={'NDB': 35, 'ORG': 90, 'SMB': 125}, name = 'm_flow') ## geothermal potentials per temperature (p. 94) - potentials = gpd.read_file( - 'data_bundle_egon_data/geothermal_potential/geothermal_potential_germany.shp') + file_path = ( + Path(".") / + "data_bundle_egon_data" / + "geothermal_potential" / + "geothermal_potential_germany.shp") + potentials = gpd.read_file(file_path) ## temperature heating system in °C (p. 95) sys_temp = 60 ## temeprature losses heat recuperator in °C (p. 95) diff --git a/src/egon/data/datasets/scenario_capacities.py b/src/egon/data/datasets/scenario_capacities.py index 76dd96e1b..528cb0b6f 100755 --- a/src/egon/data/datasets/scenario_capacities.py +++ b/src/egon/data/datasets/scenario_capacities.py @@ -2,7 +2,6 @@ Netzentwicklungsplan 2035, Version 2031, Szenario C """ -import os import egon.data.config import pandas as pd import numpy as np @@ -12,6 +11,7 @@ from sqlalchemy import Column, String, Float, Integer from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import sessionmaker +from pathlib import Path ### will be later imported from another file ### Base = declarative_base() @@ -55,7 +55,7 @@ class ScenarioCapacities(Dataset): def __init__(self, dependencies): super().__init__( name="ScenarioCapacities", - version="0.0.2", + version="0.0.3", dependencies=dependencies, tasks=( create_table, @@ -118,8 +118,10 @@ def insert_capacities_per_federal_state_nep(): "AND nuts != 'DE'") # read-in installed capacities per federal state of germany - target_file = os.path.join( - "data_bundle_egon_data/nep2035_version2021", + target_file = ( + Path(".") / + "data_bundle_egon_data" / + "nep2035_version2021" / scenario_config('eGon2035')['paths']['capacities']) df = pd.read_excel(target_file, sheet_name='1.Entwurf_NEP2035_V2021', @@ -235,9 +237,12 @@ def insert_nep_list_powerplants(): engine = db.engine() # Read-in data from csv-file - target_file = os.path.join( - "data_bundle_egon_data/nep2035_version2021/", + target_file = ( + Path(".") / + "data_bundle_egon_data" / + "nep2035_version2021" / scenario_config('eGon2035')['paths']['list_conv_pp']) + kw_liste_nep = pd.read_csv(target_file, delimiter=';', decimal=',') @@ -298,8 +303,10 @@ def district_heating_input(): """ # import data to dataframe - file = os.path.join( - "data_bundle_egon_data/nep2035_version2021", + file = ( + Path(".") / + "data_bundle_egon_data" / + "nep2035_version2021" / scenario_config('eGon2035')['paths']['capacities']) df = pd.read_excel(file, sheet_name='Kurzstudie_KWK', dtype={'Wert':float}) df.set_index(['Energietraeger', 'Name'], inplace=True) diff --git a/src/egon/data/processing/calculate_dlr.py b/src/egon/data/processing/calculate_dlr.py index df835f044..150ac09e8 100644 --- a/src/egon/data/processing/calculate_dlr.py +++ b/src/egon/data/processing/calculate_dlr.py @@ -13,7 +13,7 @@ import rioxarray from shapely.geometry import Point import psycopg2 - +from pathlib import Path def Calculate_DLR(): """Calculate DLR and assign values to each line in the db @@ -24,10 +24,14 @@ def Calculate_DLR(): """ - weather_info_path = "cutouts/europe-2011-era5/201101.nc" + weather_info_path = ( + Path(".") / "cutouts" / "germany-2011-era5.nc") + regions_shape_path = ( - "data_bundle_egon_data/regions_dynamic_line_rating/Germany_regions.shp" - ) + Path(".") / + "data_bundle_egon_data" / + "regions_dynamic_line_rating" / + "Germany_regions.shp") # Calculate hourly DLR per region dlr_hourly_dic, dlr_hourly = DLR_Regions(weather_info_path, regions_shape_path) @@ -131,7 +135,8 @@ def DLR_Regions(weather_info_path, regions_shape_path): regions = regions.sort_values(by=["Region"]) # The data downloaded using Atlite is loaded in 'weather_data_raw'. - weather_data_raw = xr.open_mfdataset("cutouts/germany-2011-era5.nc") + path = (Path(".") / "cutouts" / "germany-2011-era5.nc") + weather_data_raw = xr.open_mfdataset(str(path)) weather_data_raw = weather_data_raw.rio.write_crs(4326) weather_data_raw = weather_data_raw.rio.clip_box( minx=5.5, From 9c678cdd33f441c8262629353ca9dcf824b0c875 Mon Sep 17 00:00:00 2001 From: AmeliaNadal Date: Mon, 23 Aug 2021 14:23:24 +0200 Subject: [PATCH 08/13] Use Path object to define paths --- src/egon/data/datasets/gas_prod.py | 20 ++++++++++---- src/egon/data/importing/gas_grid.py | 43 ++++++++++++++++++++--------- 2 files changed, 44 insertions(+), 19 deletions(-) diff --git a/src/egon/data/datasets/gas_prod.py b/src/egon/data/datasets/gas_prod.py index e20a7bcaa..38865e684 100755 --- a/src/egon/data/datasets/gas_prod.py +++ b/src/egon/data/datasets/gas_prod.py @@ -2,7 +2,7 @@ """ The central module containing all code dealing with importing gas production data """ -import os +#import os import ast import pandas as pd import geopandas as gpd @@ -14,12 +14,13 @@ from egon.data.config import settings from egon.data.datasets import Dataset from urllib.request import urlretrieve +from pathlib import Path class GasProduction(Dataset): def __init__(self, dependencies): super().__init__( name="GasProduction", - version="0.0.2", + version="0.0.3", dependencies=dependencies, tasks=(import_gas_generators), ) @@ -34,9 +35,12 @@ def load_NG_generators(): Dataframe containing the natural gas producion units in Germany """ - target_file = os.path.join( - "datasets/gas_data/", - 'data/IGGIELGN_Productions.csv') + target_file = ( + Path(".") / + "datasets" / + "gas_data" / + "data" / + "IGGIELGN_Productions.csv") NG_generators_list = pd.read_csv(target_file, delimiter=';', decimal='.', @@ -100,7 +104,11 @@ def load_biogas_generators(): # Download file basename = "Biogaspartner_Einspeiseatlas_Deutschland_2021.xlsx" url = "https://www.biogaspartner.de/fileadmin/Biogaspartner/Dokumente/Einspeiseatlas/" + basename - target_file = "datasets/gas_data/" + basename + target_file = ( + Path(".") / + "datasets" / + "gas_data" / + basename) urlretrieve(url, target_file) diff --git a/src/egon/data/importing/gas_grid.py b/src/egon/data/importing/gas_grid.py index 34bcfc8c6..a0b2a6c27 100755 --- a/src/egon/data/importing/gas_grid.py +++ b/src/egon/data/importing/gas_grid.py @@ -16,6 +16,7 @@ from urllib.request import urlretrieve from zipfile import ZipFile from geoalchemy2.shape import from_shape +from pathlib import Path def next_id(component): """ Select next id value for components in pf-tables @@ -46,20 +47,28 @@ def download_SciGRID_gas_data(): Download SciGRID_gas IGGIELGN data from Zenodo """ - path = "datasets/gas_data/" + path = ( + Path(".") / + "datasets" / + "gas_data" ) os.makedirs(path, exist_ok=True) basename = "IGGIELGN" + zip_file = ( + Path(".") / + "datasets" / + "gas_data" / + "IGGIELGN.zip") zenodo_zip_file_url = ("https://zenodo.org/record/4767098/files/"+ basename + ".zip") - if not os.path.isfile(path + basename + ".zip"): - urlretrieve(zenodo_zip_file_url, path + basename + ".zip") + if not os.path.isfile(zip_file): + urlretrieve(zenodo_zip_file_url, zip_file) components = ['Nodes', 'PipeSegments', 'Productions', 'Storages'] #'Compressors' files = [] for i in components: files.append('data/'+ basename + "_" + i + ".csv") - with ZipFile(path + basename + ".zip", 'r') as zipObj: + with ZipFile(zip_file, 'r') as zipObj: listOfFileNames = zipObj.namelist() for fileName in listOfFileNames: if fileName in files: @@ -78,9 +87,12 @@ def define_gas_nodes_list(): # Select next id value new_id = next_id('bus') - target_file = os.path.join( - "datasets/gas_data/data/", - 'IGGIELGN_Nodes.csv') + target_file = ( + Path(".") / + "datasets" / + "gas_data" / + "data" / + "IGGIELGN_Nodes.csv") gas_nodes_list = pd.read_csv(target_file, delimiter=';', decimal='.', @@ -175,17 +187,22 @@ def insert_gas_pipeline_list(gas_nodes_list): # Select next id value new_id = next_id('link') - classifiaction_file = os.path.join( - "data_bundle_egon_data/pipeline_classification_gas/", - 'pipeline_classification.csv') + classifiaction_file = ( + Path(".") / + "data_bundle_egon_data" / + "pipeline_classification_gas" / + "pipeline_classification.csv") classification = pd.read_csv(classifiaction_file, delimiter=',', usecols = ['classification', 'max_transport_capacity_Gwh/d']) - target_file = os.path.join( - "datasets/gas_data/data/", - 'IGGIELGN_PipeSegments.csv') + target_file = ( + Path(".") / + "datasets" / + "gas_data" / + "data" / + "IGGIELGN_PipeSegments.csv") gas_pipelines_list = pd.read_csv(target_file, delimiter=';', decimal='.', From 9868c1b6b5961807206b77109099cffe57d62924 Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Wed, 25 Aug 2021 12:15:43 +0200 Subject: [PATCH 09/13] Store data in working dir (see #341) --- src/egon/data/datasets/re_potential_areas/__init__.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/egon/data/datasets/re_potential_areas/__init__.py b/src/egon/data/datasets/re_potential_areas/__init__.py index 3d321935d..69ac83efd 100644 --- a/src/egon/data/datasets/re_potential_areas/__init__.py +++ b/src/egon/data/datasets/re_potential_areas/__init__.py @@ -4,6 +4,7 @@ from functools import partial from urllib.request import urlretrieve +from pathlib import Path import os from geoalchemy2 import Geometry @@ -57,10 +58,16 @@ def ve(s): else ve(f"'{dataset}' is not a valid dataset boundary.") ) + download_directory = Path(".") / "re_potential_areas" + # Create the folder, if it does not exists already + if not os.path.exists(download_directory): + os.mkdir(download_directory) + url_target_file_map = zip( pa_config["original_data"]["source"][url_section], [ - os.path.join(os.path.dirname(__file__), file) + # os.path.join(os.path.dirname(__file__), file) + Path(".") / "re_potential_areas" / Path(file).name for file in pa_config["original_data"]["target"][ "path_table_map" ].keys() @@ -106,7 +113,7 @@ def insert_data(): pa_config = data_config["re_potential_areas"] file_table_map = { - os.path.join(os.path.dirname(__file__), file): table + Path(".") / "re_potential_areas" / Path(file).name: table for file, table in pa_config["original_data"]["target"][ "path_table_map" ].items() From a3235b3aab4c2e8b9a5ca41fd2bb7dcde96e25d0 Mon Sep 17 00:00:00 2001 From: nesnoj Date: Thu, 26 Aug 2021 09:16:04 +0200 Subject: [PATCH 10/13] Add changelog entry --- CHANGELOG.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 78bfb9411..df20439c0 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -237,5 +237,5 @@ Bug fixes `#387 `_ * Add xlrd and openpyxl to installation setup `#400 `_ - - +* Remove hard-coded slashes in file paths to ensure Windows compatibility + `#398 `_ From c244ab0c02b13a6c3f60180cf61366cb18857822 Mon Sep 17 00:00:00 2001 From: IlkaCu Date: Fri, 27 Aug 2021 13:29:43 +0200 Subject: [PATCH 11/13] Integrate missing dependency --- src/egon/data/airflow/dags/pipeline.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/egon/data/airflow/dags/pipeline.py b/src/egon/data/airflow/dags/pipeline.py index d80bed738..01607a541 100755 --- a/src/egon/data/airflow/dags/pipeline.py +++ b/src/egon/data/airflow/dags/pipeline.py @@ -244,6 +244,7 @@ heat_demand_Germany, etrago_input_data, household_electricity_demand_annual, + zensus_mv_grid_districts ] ) From 140fca6d061e8d1d42d12043443f03cc5534da64 Mon Sep 17 00:00:00 2001 From: IlkaCu Date: Fri, 27 Aug 2021 16:07:07 +0200 Subject: [PATCH 12/13] Update pipeline.py --- src/egon/data/airflow/dags/pipeline.py | 30 ++++++++++++++------------ 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/src/egon/data/airflow/dags/pipeline.py b/src/egon/data/airflow/dags/pipeline.py index 01607a541..cfae39781 100755 --- a/src/egon/data/airflow/dags/pipeline.py +++ b/src/egon/data/airflow/dags/pipeline.py @@ -236,20 +236,6 @@ heat_demand_Germany = HeatDemandImport( dependencies=[vg250, scenario_parameters, zensus_vg250]) - # Distribute electrical CTS demands to zensus grid - cts_electricity_demand_annual = CtsElectricityDemand( - dependencies=[ - demandregio, - zensus_vg250, - heat_demand_Germany, - etrago_input_data, - household_electricity_demand_annual, - zensus_mv_grid_districts - ] - ) - - elec_cts_demands_zensus = tasks[ - 'electricity_demand.distribute-cts-demands'] # Gas grid import gas_grid_insert_data = PythonOperator( @@ -332,6 +318,22 @@ ) map_zensus_grid_districts = tasks["zensus_mv_grid_districts.mapping"] + + # Distribute electrical CTS demands to zensus grid + cts_electricity_demand_annual = CtsElectricityDemand( + dependencies=[ + demandregio, + zensus_vg250, + heat_demand_Germany, + etrago_input_data, + household_electricity_demand_annual, + zensus_mv_grid_districts + ] + ) + + elec_cts_demands_zensus = tasks[ + 'electricity_demand.distribute-cts-demands'] + # Map federal states to mv_grid_districts vg250_mv_grid_districts = Vg250MvGridDistricts( From ff9aa23f85e02869367809d48fe930d87e05d9bf Mon Sep 17 00:00:00 2001 From: IlkaCu Date: Mon, 30 Aug 2021 11:25:47 +0200 Subject: [PATCH 13/13] Update CHANGELOG --- CHANGELOG.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 78bfb9411..41d5b9519 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -237,5 +237,7 @@ Bug fixes `#387 `_ * Add xlrd and openpyxl to installation setup `#400 `_ +* Add missing dependency in pipeline.py + `#412 `_