Features/otg substation mapping

src/egon/data/airflow/dags/pipeline.py

@@ -322,39 +322,47 @@
         autocommit=True,
     )
 
-    create_voronoi = PythonOperator(
-        task_id="create_voronoi",
-        python_callable=substation.create_voronoi
-    )
     osm_add_metadata >> substation_tables >> substation_functions
-    substation_functions >> hvmv_substation_extraction >> create_voronoi
-    substation_functions >> ehv_substation_extraction >> create_voronoi
+    substation_functions >> hvmv_substation_extraction
+    substation_functions >> ehv_substation_extraction
     vg250_clean_and_prepare >> hvmv_substation_extraction
     vg250_clean_and_prepare >> ehv_substation_extraction
 
-    # MV grid districts
-    define_mv_grid_districts = PythonOperator(
-        task_id="define_mv_grid_districts",
-        python_callable=mvgd.define_mv_grid_districts
-    )
-    create_voronoi >> define_mv_grid_districts
-
     # osmTGmod ehv/hv grid model generation
     run_osmtgmod = PythonOperator(
         task_id="run_osmtgmod",
         python_callable=osmtgmod.run_osmtgmod,
     )
 
-
     osmtgmod_pypsa = PythonOperator(
         task_id="osmtgmod_pypsa",
         python_callable=osmtgmod.osmtgmmod_to_pypsa,
     )
 
+    osmtgmod_substation = PostgresOperator(
+        task_id="osmtgmod_substation",
+        sql=resources.read_text(osmtgmod, "substation_otg.sql"),
+        postgres_conn_id="egon_data",
+        autocommit=True,
+    )
     ehv_substation_extraction >> run_osmtgmod
     hvmv_substation_extraction >> run_osmtgmod
     run_osmtgmod >> osmtgmod_pypsa
+    run_osmtgmod >> osmtgmod_substation
 
+    # MV grid districts
+    create_voronoi = PythonOperator(
+        task_id="create_voronoi",
+        python_callable=substation.create_voronoi
+    )
+    osmtgmod_substation >> create_voronoi
+
+
+    define_mv_grid_districts = PythonOperator(
+        task_id="define_mv_grid_districts",
+        python_callable=mvgd.define_mv_grid_districts
+    )
+    create_voronoi >> define_mv_grid_districts
 
     # Import potential areas for wind onshore and ground-mounted PV
     download_re_potential_areas = PythonOperator(

src/egon/data/processing/mv_grid_districts.py

@@ -29,7 +29,8 @@
 
 from egon.data import db
 from egon.data.db import session_scope
-from egon.data.processing.substation import EgonHvmvSubstationVoronoi
+from egon.data.processing.substation import (EgonHvmvSubstationVoronoi,
+                                             EgonHvmvSubstation)
 
 Base = declarative_base()
 metadata = Base.metadata
@@ -54,28 +55,6 @@ class Vg250GemClean(Base):
     geometry = Column(Geometry("POLYGON", 3035), index=True)
 
 
-class EgonHvmvSubstation(Base):
-    __tablename__ = "egon_hvmv_substation"
-    __table_args__ = {"schema": "grid"}
-
-    subst_id = Column(Integer, primary_key=True)
-    lon = Column(Float)
-    lat = Column(Float)
-    point = Column(Geometry("POINT", 4326), index=True)
-    polygon = Column(Geometry, index=True)
-    voltage = Column(String)
-    power_type = Column(String)
-    substation = Column(String)
-    osm_id = Column(String)
-    osm_www = Column(String)
-    frequency = Column(String)
-    subst_name = Column(String)
-    ref = Column(String)
-    operator = Column(String)
-    dbahn = Column(String)
-    status = Column(Integer)
-
-
 class HvmvSubstPerMunicipality(Base):
     __tablename__ = "hvmv_subst_per_municipality"
     __table_args__ = {"schema": "grid"}
@@ -261,7 +240,7 @@ def split_multi_substation_municipalities():
                         ),
                     )
                 ).geom.label("geom"),
-                EgonHvmvSubstationVoronoi.subst_id,
+                EgonHvmvSubstationVoronoi.bus_id,
                 EgonHvmvSubstationVoronoi.id.label("voronoi_id"),
             )
             .filter(HvmvSubstPerMunicipality.subst_count > 1)
@@ -290,7 +269,7 @@ def split_multi_substation_municipalities():
         cuts_substation_subquery = (
             session.query(
                 VoronoiMunicipalityCuts.id,
-                EgonHvmvSubstation.subst_id,
+                EgonHvmvSubstation.bus_id,
                 func.ST_Transform(EgonHvmvSubstation.point, 3035).label(
                     "geom_sub"
                 ),
@@ -304,7 +283,7 @@ def split_multi_substation_municipalities():
             )
             .group_by(
                 VoronoiMunicipalityCuts.id,
-                EgonHvmvSubstation.subst_id,
+                EgonHvmvSubstation.bus_id,
                 EgonHvmvSubstation.point,
             )
             .subquery()
@@ -314,7 +293,7 @@ def split_multi_substation_municipalities():
         ).update(
             {
                 "subst_count": cuts_substation_subquery.c.subst_count,
-                "subst_id": cuts_substation_subquery.c.subst_id,
+                "subst_id": cuts_substation_subquery.c.bus_id,
                 "geom_sub": cuts_substation_subquery.c.geom_sub,
             },
             synchronize_session="fetch",
@@ -567,7 +546,7 @@ def merge_polygons_to_grid_district():
         # Step 2: Insert municipality polygons with exactly one substation
         one_substation = (
             session.query(
-                EgonHvmvSubstation.subst_id,
+                EgonHvmvSubstation.bus_id,
                 func.ST_Multi(HvmvSubstPerMunicipality.geometry).label("geom"),
                 func.ST_Area(
                     func.ST_Multi(HvmvSubstPerMunicipality.geometry)
@@ -717,7 +696,7 @@ def nearest_polygon_with_substation(
             ),
             #with_substation.c.id
             func.ST_Distance(
-                func.ST_Centroid(without_substation.c.geom), 
+                func.ST_Centroid(without_substation.c.geom),
                 func.ST_Centroid(with_substation.c.geom)
             )
         )

src/egon/data/processing/osmtgmod/__init__.py

@@ -429,7 +429,7 @@ def osmtgmod(
         CREATE TABLE transfer_busses_complete as
         SELECT DISTINCT ON (osm_id) * FROM
         (SELECT * FROM grid.egon_ehv_substation
-        UNION SELECT subst_id, lon, lat, point, polygon, voltage,
+        UNION SELECT bus_id, lon, lat, point, polygon, voltage,
         power_type, substation, osm_id, osm_www, frequency, subst_name,
         ref, operator, dbahn, status
         FROM grid.egon_hvmv_substation ORDER BY osm_id) as foo;

src/egon/data/processing/osmtgmod/substation_otg.sql

@@ -0,0 +1,41 @@
+ALTER TABLE grid.egon_hvmv_substation
+	ADD COLUMN otg_id bigint;
+
+-- fill table with bus_i from osmtgmod
+UPDATE grid.egon_hvmv_substation
+	SET 	otg_id = osmtgmod_results.bus_data.bus_i
+	FROM 	osmtgmod_results.bus_data
+	WHERE 	osmtgmod_results.bus_data.base_kv <= 110 AND (SELECT TRIM(leading 'n' FROM TRIM(leading 'w' FROM grid.egon_hvmv_substation.osm_id))::BIGINT)=osmtgmod_results.bus_data.osm_substation_id; 
+
+DELETE FROM grid.egon_hvmv_substation WHERE otg_id IS NULL;
+
+UPDATE grid.egon_hvmv_substation
+	SET 	bus_id = otg_id;
+
+ALTER TABLE grid.egon_hvmv_substation
+	DROP COLUMN otg_id;
+
+
+-- do the same with model_draft.ego_grid_ehv_substation
+
+-- update model_draft.ego_grid_ehv_substation table with new column of respective osmtgmod bus_i
+ALTER TABLE grid.egon_ehv_substation
+	ADD COLUMN otg_id bigint;
+
+-- fill table with bus_i from osmtgmod
+UPDATE grid.egon_ehv_substation
+	SET otg_id = osmtgmod_results.bus_data.bus_i
+	FROM osmtgmod_results.bus_data
+	WHERE osmtgmod_results.bus_data.base_kv > 110 AND(SELECT TRIM(leading 'n' FROM TRIM(leading 'w' FROM TRIM(leading 'r' FROM grid.egon_ehv_substation.osm_id)))::BIGINT)=osmtgmod_results.bus_data.osm_substation_id; 
+
+DELETE FROM grid.egon_ehv_substation WHERE otg_id IS NULL;
+
+UPDATE grid.egon_ehv_substation
+	SET 	bus_id = otg_id;
+
+ALTER TABLE grid.egon_ehv_substation
+	DROP COLUMN otg_id;
+
+
+
+

src/egon/data/processing/substation/__init__.py

@@ -13,10 +13,10 @@
 class EgonEhvSubstation(Base):
     __tablename__ = 'egon_ehv_substation'
     __table_args__ = {'schema': 'grid'}
-    subst_id = Column(Integer,
-        Sequence('egon_ehv_substation_subst_id_seq', schema='grid'),
+    bus_id = Column(Integer,
+        Sequence('egon_ehv_substation_bus_id_seq', schema='grid'),
         server_default=
-            Sequence('egon_ehv_substation_subst_id_seq', schema='grid').next_value(),
+            Sequence('egon_ehv_substation_bus_id_seq', schema='grid').next_value(),
         primary_key=True)
     lon = Column(Float(53))
     lat = Column(Float(53))
@@ -38,10 +38,10 @@ class EgonEhvSubstation(Base):
 class EgonHvmvSubstation(Base):
     __tablename__ = 'egon_hvmv_substation'
     __table_args__ = {'schema': 'grid'}
-    subst_id = Column(Integer,
-        Sequence('egon_hvmv_substation_subst_id_seq', schema='grid'),
+    bus_id = Column(Integer,
+        Sequence('egon_hvmv_substation_bus_id_seq', schema='grid'),
         server_default=
-            Sequence('egon_hvmv_substation_subst_id_seq', schema='grid').next_value(),
+            Sequence('egon_hvmv_substation_bus_id_seq', schema='grid').next_value(),
         primary_key=True)
     lon = Column(Float(53))
     lat = Column(Float(53))
@@ -67,7 +67,7 @@ class EgonHvmvSubstationVoronoi(Base):
         server_default=
             Sequence('egon_hvmv_substation_voronoi_id_seq', schema='grid').next_value(),
         primary_key=True)
-    subst_id = Column(Integer)
+    bus_id = Column(Integer)
     geom = Column(Geometry('Multipolygon', 4326))
 
 
@@ -79,7 +79,7 @@ class EgonEhvSubstationVoronoi(Base):
         server_default=
             Sequence('egon_ehv_substation_voronoi_id_seq', schema='grid').next_value(),
         primary_key=True)
-    subst_id = Column(Integer)
+    bus_id = Column(Integer)
     geom = Column(Geometry('Multipolygon', 4326))
 
 def create_tables():
@@ -136,13 +136,13 @@ def create_tables():
     db.execute_sql(
         f"""DROP SEQUENCE IF EXISTS
             {cfg_hvmv['processed']['schema']}.
-            {cfg_hvmv['processed']['table']}_subst_id_seq CASCADE;"""
+            {cfg_hvmv['processed']['table']}_bus_id_seq CASCADE;"""
     )
 
     db.execute_sql(
         f"""DROP SEQUENCE IF EXISTS
             {cfg_ehv['processed']['schema']}.
-            {cfg_ehv['processed']['table']}_subst_id_seq CASCADE;"""
+            {cfg_ehv['processed']['table']}_bus_id_seq CASCADE;"""
     )
 
     engine = db.engine()
@@ -289,14 +289,14 @@ def create_voronoi():
         db.execute_sql(
             f"""
             UPDATE {schema}.{voronoi_table} AS t1
-                SET  	subst_id = t2.subst_id
+                SET  	bus_id = t2.bus_id
 	            FROM	(SELECT	voi.id AS id,
-			                sub.subst_id ::integer AS subst_id
+			                sub.bus_id ::integer AS bus_id
 		            FROM	{schema}.{voronoi_table} AS voi,
 			                {schema}.{substation_table} AS sub
 		            WHERE  	voi.geom && sub.point AND
 			                ST_CONTAINS(voi.geom,sub.point)
-		           GROUP BY voi.id,sub.subst_id
+		           GROUP BY voi.id,sub.bus_id
 		           )AS t2
 	            WHERE  	t1.id = t2.id;
             """