Merge branch 'dev' into fixes/#341-osm-vg250-store-data-in-working-dir

CHANGELOG.rst

@@ -124,11 +124,11 @@ Added
   `#237 <https://github.com/openego/eGon-data/issues/237>`_
 * Merge electrical loads per bus and export to etrago tables
   `#328 <https://github.com/openego/eGon-data/issues/328>`_
+* Insert industial gas demand
+  `#321 <https://github.com/openego/eGon-data/issues/358>`_
 * Integrate existing CHP and extdended CHP > 10MW_el
   `#266 <https://github.com/openego/eGon-data/issues/266>`_
 
-.. _PR #159: https://github.com/openego/eGon-data/pull/159
-
 
 Changed
 -------

src/egon/data/airflow/dags/pipeline.py

@@ -38,6 +38,12 @@
 from egon.data.datasets.vg250_mv_grid_districts import Vg250MvGridDistricts
 from egon.data.datasets.zensus_mv_grid_districts import ZensusMvGridDistricts
 from egon.data.datasets.zensus_vg250 import ZensusVg250
+
+from egon.data.datasets.gas_prod import GasProduction
+from egon.data.datasets.industrial_gas_demand import IndustrialGasDemand
+import airflow
+
+import egon.data.importing.zensus as import_
 import egon.data.importing.gas_grid as gas_grid
 import egon.data.importing.zensus as import_zs
 import egon.data.processing.calculate_dlr as dlr
@@ -267,6 +273,10 @@
         dependencies=[create_gas_polygons]
     )
 
+    # Insert industrial gas demand
+    industrial_gas_demand = IndustrialGasDemand( 
+     dependencies=[create_gas_polygons]) 
+
     # Extract landuse areas from osm data set
     create_landuse_table = PythonOperator(
         task_id="create-landuse-table",

src/egon/data/datasets/gas_prod.py

@@ -10,7 +10,6 @@
 import geopandas
 
 from egon.data import db
-from egon.data.importing.gas_grid import next_id
 from egon.data.config import settings
 from egon.data.datasets import Dataset
 from urllib.request import urlretrieve
@@ -218,7 +217,7 @@ def import_gas_generators():
     )
 
     # Select next id value
-    new_id = next_id('generator')
+    new_id = db.next_etrago_id('generator')
 
     CH4_generators_list = pd.concat([load_NG_generators(), load_biogas_generators()])
     CH4_generators_list['generator_id'] = range(new_id, new_id + len(CH4_generators_list))

src/egon/data/datasets/industrial_gas_demand.py

@@ -0,0 +1,250 @@
+# -*- coding: utf-8 -*-
+"""
+The central module containing all code dealing with importing gas industrial demand
+"""
+import requests
+import pandas as pd
+import numpy as np
+
+
+from shapely import wkt 
+from egon.data import db
+from egon.data.datasets.gas_prod import assign_gas_bus_id
+from egon.data.config import settings
+from egon.data.datasets import Dataset
+
+class IndustrialGasDemand(Dataset): 
+     def __init__(self, dependencies): 
+         super().__init__( 
+             name="IndustrialGasDemand", 
+             version="0.0.0", 
+             dependencies=dependencies, 
+             tasks=(insert_industrial_gas_demand), 
+         )       
+
+def download_CH4_industrial_demand():
+    """Download the CH4 industrial demand in Germany from the FfE open data portal
+    
+    Returns
+    -------
+    CH4_industrial_demand : 
+        Dataframe containing the CH4 industrial demand in Germany
+        
+    """
+    # Download the data and the id_region correspondance table
+    correspondance_url = 'http://opendata.ffe.de:3000/region?id_region_type=eq.38'
+    url = 'http://opendata.ffe.de:3000/opendata?id_opendata=eq.66&&year=eq.'
+
+    internal_id = '2,11' # Natural_Gas
+    year = '2035' # 2050
+    datafilter = '&&internal_id=eq.{'+internal_id+'}'
+    request = url + year + datafilter
+
+    result = requests.get(request)
+    industrial_loads_list = pd.read_json(result.content)
+    industrial_loads_list = industrial_loads_list[['id_region', 'values']].copy()
+    industrial_loads_list = industrial_loads_list.set_index('id_region')
+
+    result_corr = requests.get(correspondance_url)
+    df_corr = pd.read_json(result_corr.content)
+    df_corr = df_corr[['id_region', 'name_short']].copy()
+    df_corr = df_corr.set_index('id_region')
+
+    # Match the id_region to obtain the NUT3 region names
+    industrial_loads_list = pd.concat([industrial_loads_list, df_corr], axis=1, join="inner")
+    industrial_loads_list['NUTS0'] = (industrial_loads_list['name_short'].str)[0:2] 
+    industrial_loads_list['NUTS1'] = (industrial_loads_list['name_short'].str)[0:3]   
+    industrial_loads_list = industrial_loads_list[industrial_loads_list['NUTS0'].str.match('DE')]
+
+    # Cut data to federal state if in testmode
+    boundary = settings()['egon-data']['--dataset-boundary']
+    if boundary != 'Everything':
+        map_states = {'Baden-Württemberg':'DE1', 'Nordrhein-Westfalen': 'DEA',
+                'Hessen': 'DE7', 'Brandenburg': 'DE4', 'Bremen':'DE5',
+                'Rheinland-Pfalz': 'DEB', 'Sachsen-Anhalt': 'DEE',
+                'Schleswig-Holstein':'DEF', 'Mecklenburg-Vorpommern': 'DE8',
+                'Thüringen': 'DEG', 'Niedersachsen': 'DE9',
+                'Sachsen': 'DED', 'Hamburg': 'DE6', 'Saarland': 'DEC',
+                'Berlin': 'DE3', 'Bayern': 'DE2'}
+
+        industrial_loads_list = industrial_loads_list[industrial_loads_list['NUTS1'].isin([map_states[boundary], np.nan])]
+
+    industrial_loads_list = industrial_loads_list.rename(columns={"name_short": "nuts3", "values": "p_set"})
+    industrial_loads_list = industrial_loads_list.set_index('nuts3')
+
+    # Add the centroid point to each NUTS3 area
+    sql_vg250 = """SELECT nuts as nuts3, geometry as geom
+                    FROM boundaries.vg250_krs 
+                    WHERE gf = 4 ;"""   
+    gdf_vg250 = db.select_geodataframe(sql_vg250 , epsg=4326)
+
+    point = []
+    for index, row in gdf_vg250.iterrows():
+        point.append(wkt.loads(str(row['geom'])).centroid)
+    gdf_vg250['point'] = point
+    gdf_vg250 = gdf_vg250.set_index('nuts3')
+    gdf_vg250  = gdf_vg250 .drop(columns=['geom'])  
+
+    # Match the load to the NUTS3 points
+    industrial_loads_list = pd.concat([industrial_loads_list, gdf_vg250], axis=1, join="inner")
+    industrial_loads_list = industrial_loads_list.rename(columns={'point': 'geom'}).set_geometry('geom', crs=4326)
+
+    # Match to associated gas bus   
+    industrial_loads_list = assign_gas_bus_id(industrial_loads_list)
+
+    # Remove useless columns
+    industrial_loads_list = industrial_loads_list.drop(columns=['geom', 'NUTS0', 'NUTS1', 'bus_id']) 
+
+    return industrial_loads_list
+
+
+def download_H2_industrial_demand():
+    """Download the H2 industrial demand in Germany from the FfE open data portal
+    
+    Returns
+    -------
+    H2_industrial_demand : 
+        Dataframe containing the H2 industrial demand in Germany
+        
+    """
+    # Download the data and the id_region correspondance table
+    correspondance_url = 'http://opendata.ffe.de:3000/region?id_region_type=eq.38'
+    url = 'http://opendata.ffe.de:3000/opendata?id_opendata=eq.66&&year=eq.'
+
+    internal_id = '2,162' # Hydrogen
+    year = '2035' # 2050
+    datafilter = '&&internal_id=eq.{'+internal_id+'}'
+    request = url + year + datafilter
+
+    result = requests.get(request)
+    industrial_loads_list = pd.read_json(result.content)
+    industrial_loads_list = industrial_loads_list[['id_region', 'values']].copy()
+    industrial_loads_list = industrial_loads_list.set_index('id_region')
+
+    result_corr = requests.get(correspondance_url)
+    df_corr = pd.read_json(result_corr.content)
+    df_corr = df_corr[['id_region', 'name_short']].copy()
+    df_corr = df_corr.set_index('id_region')
+
+    # Match the id_region to obtain the NUT3 region names
+    industrial_loads_list = pd.concat([industrial_loads_list, df_corr], axis=1, join="inner")
+    industrial_loads_list['NUTS0'] = (industrial_loads_list['name_short'].str)[0:2] 
+    industrial_loads_list['NUTS1'] = (industrial_loads_list['name_short'].str)[0:3]   
+    industrial_loads_list = industrial_loads_list[industrial_loads_list['NUTS0'].str.match('DE')]
+
+    # Cut data to federal state if in testmode
+    boundary = settings()['egon-data']['--dataset-boundary']
+    if boundary != 'Everything':
+        map_states = {'Baden-Württemberg':'DE1', 'Nordrhein-Westfalen': 'DEA',
+                'Hessen': 'DE7', 'Brandenburg': 'DE4', 'Bremen':'DE5',
+                'Rheinland-Pfalz': 'DEB', 'Sachsen-Anhalt': 'DEE',
+                'Schleswig-Holstein':'DEF', 'Mecklenburg-Vorpommern': 'DE8',
+                'Thüringen': 'DEG', 'Niedersachsen': 'DE9',
+                'Sachsen': 'DED', 'Hamburg': 'DE6', 'Saarland': 'DEC',
+                'Berlin': 'DE3', 'Bayern': 'DE2'}
+
+        industrial_loads_list = industrial_loads_list[industrial_loads_list['NUTS1'].isin([map_states[boundary], np.nan])]
+
+    industrial_loads_list = industrial_loads_list.rename(columns={"name_short": "nuts3", "values": "p_set"})
+    industrial_loads_list = industrial_loads_list.set_index('nuts3')
+
+    # Add the centroid point to each NUTS3 area
+    sql_vg250 = """SELECT nuts as nuts3, geometry as geom
+                    FROM boundaries.vg250_krs 
+                    WHERE gf = 4 ;"""   
+    gdf_vg250 = db.select_geodataframe(sql_vg250 , epsg=4326)
+
+    point = []
+    for index, row in gdf_vg250.iterrows():
+        point.append(wkt.loads(str(row['geom'])).centroid)
+    gdf_vg250['point'] = point
+    gdf_vg250 = gdf_vg250.set_index('nuts3')
+    gdf_vg250  = gdf_vg250 .drop(columns=['geom'])  
+
+    # Match the load to the NUTS3 points
+    industrial_loads_list = pd.concat([industrial_loads_list, gdf_vg250], axis=1, join="inner")
+    industrial_loads_list = industrial_loads_list.rename(columns={'point': 'geom'}).set_geometry('geom', crs=4326)
+
+    # Match to associated gas bus   
+    industrial_loads_list = assign_gas_bus_id(industrial_loads_list)
+
+    # Remove useless columns
+    industrial_loads_list = industrial_loads_list.drop(columns=['geom', 'NUTS0', 'NUTS1', 'bus_id']) 
+
+    return industrial_loads_list
+
+
+def import_industrial_gas_demand():
+    """Insert list of industrial gas demand (one per NUTS3) in database
+    Returns
+        industrial_gas_demand : Dataframe containing the industrial gas demand in Germany
+    """
+    # Connect to local database
+    engine = db.engine()
+
+    # Clean table
+    db.execute_sql(
+        """
+    DELETE FROM grid.egon_etrago_load WHERE "carrier" = 'gas';
+    """
+    )
+
+    # Select next id value
+    new_id = db.next_etrago_id('load')
+
+    industrial_gas_demand = pd.concat([download_CH4_industrial_demand(), download_H2_industrial_demand()])
+    industrial_gas_demand['load_id'] = range(new_id, new_id + len(industrial_gas_demand))
+
+    # Add missing columns
+    c = {'scn_name':'eGon2035', 'carrier':'gas'}
+    industrial_gas_demand = industrial_gas_demand.assign(**c)
+
+    industrial_gas_demand =  industrial_gas_demand.reset_index(drop=True)
+
+    # Remove useless columns
+    egon_etrago_load_gas = industrial_gas_demand.drop(columns=['p_set'])  
+
+    # Insert data to db
+    egon_etrago_load_gas.to_sql('egon_etrago_load',
+                              engine,
+                              schema ='grid',
+                              index = False,
+                              if_exists = 'append')
+
+    return industrial_gas_demand
+
+
+def import_industrial_gas_demand_time_series(egon_etrago_load_gas):
+    """Insert list of industrial gas demand time series (one per NUTS3) in database
+    Returns
+    -------
+    None.
+    """
+    egon_etrago_load_gas_timeseries = egon_etrago_load_gas
+
+    # Connect to local database
+    engine = db.engine()
+
+    # Adjust columns
+    egon_etrago_load_gas_timeseries = egon_etrago_load_gas_timeseries.drop(columns=['carrier', 'bus']) 
+    egon_etrago_load_gas_timeseries['temp_id'] = 1
+
+    # Insert data to db
+    egon_etrago_load_gas_timeseries.to_sql('egon_etrago_load_timeseries',
+                              engine,
+                              schema ='grid',
+                              index = False,
+                              if_exists = 'append')
+
+
+def insert_industrial_gas_demand():
+    """Overall function for inserting the industrial gas demand
+    Returns
+    -------
+    None.
+    """
+
+    egon_etrago_load_gas = import_industrial_gas_demand()
+
+    import_industrial_gas_demand_time_series(egon_etrago_load_gas)
+

src/egon/data/importing/gas_grid.py

@@ -18,30 +18,6 @@
 from geoalchemy2.shape import from_shape
 from pathlib import Path
 
-def next_id(component):
-    """ Select next id value for components in pf-tables
-    Parameters
-    ----------
-    component : str
-        Name of componenet
-    Returns
-    -------
-    next_id : int
-        Next index value
-    """
-    max_id = db.select_dataframe(
-        f"""
-        SELECT MAX({component}_id) FROM grid.egon_etrago_{component}
-        """)['max'][0]
-
-    if max_id:
-        next_id = max_id + 1
-    else:
-        next_id = 1
-
-    return next_id
-
-
 def download_SciGRID_gas_data():
     """
     Download SciGRID_gas IGGIELGN data from Zenodo
@@ -85,7 +61,7 @@ def define_gas_nodes_list():
 
     """
     # Select next id value
-    new_id = next_id('bus')
+    new_id = db.next_etrago_id('bus')
 
     target_file = (
             Path(".") /
@@ -185,7 +161,7 @@ def insert_gas_pipeline_list(gas_nodes_list):
     engine = db.engine()
 
     # Select next id value
-    new_id = next_id('link')
+    new_id = db.next_etrago_id('link')
 
     classifiaction_file = (
             Path(".") /

src/egon/data/processing/power2gas.py

@@ -10,7 +10,7 @@
 from geoalchemy2.types import Geometry
 from scipy.spatial import cKDTree
 from egon.data import db
-from egon.data.importing.gas_grid import next_id
+#from egon.data.importing.gas_grid import next_id
 
 def insert_power2gas():
     """Function defining the potential power-to-gas capacities and inserting them in the etrago_link table.
@@ -24,7 +24,7 @@ def insert_power2gas():
     engine = db.engine()
 
     # Select next id value
-    new_id = next_id('link')
+    new_id = db.next_etrago_id('link')
 
     # Create dataframes containing all gas buses and all the HV power buses
     sql_AC = """SELECT bus_id, geom