diff --git a/src/egon/data/airflow/dags/pipeline.py b/src/egon/data/airflow/dags/pipeline.py index 64f986461..a703b1f08 100644 --- a/src/egon/data/airflow/dags/pipeline.py +++ b/src/egon/data/airflow/dags/pipeline.py @@ -322,39 +322,47 @@ autocommit=True, ) - create_voronoi = PythonOperator( - task_id="create_voronoi", - python_callable=substation.create_voronoi - ) osm_add_metadata >> substation_tables >> substation_functions - substation_functions >> hvmv_substation_extraction >> create_voronoi - substation_functions >> ehv_substation_extraction >> create_voronoi + substation_functions >> hvmv_substation_extraction + substation_functions >> ehv_substation_extraction vg250_clean_and_prepare >> hvmv_substation_extraction vg250_clean_and_prepare >> ehv_substation_extraction - # MV grid districts - define_mv_grid_districts = PythonOperator( - task_id="define_mv_grid_districts", - python_callable=mvgd.define_mv_grid_districts - ) - create_voronoi >> define_mv_grid_districts - # osmTGmod ehv/hv grid model generation run_osmtgmod = PythonOperator( task_id="run_osmtgmod", python_callable=osmtgmod.run_osmtgmod, ) - osmtgmod_pypsa = PythonOperator( task_id="osmtgmod_pypsa", python_callable=osmtgmod.osmtgmmod_to_pypsa, ) + osmtgmod_substation = PostgresOperator( + task_id="osmtgmod_substation", + sql=resources.read_text(osmtgmod, "substation_otg.sql"), + postgres_conn_id="egon_data", + autocommit=True, + ) ehv_substation_extraction >> run_osmtgmod hvmv_substation_extraction >> run_osmtgmod run_osmtgmod >> osmtgmod_pypsa + run_osmtgmod >> osmtgmod_substation + # MV grid districts + create_voronoi = PythonOperator( + task_id="create_voronoi", + python_callable=substation.create_voronoi + ) + osmtgmod_substation >> create_voronoi + + + define_mv_grid_districts = PythonOperator( + task_id="define_mv_grid_districts", + python_callable=mvgd.define_mv_grid_districts + ) + create_voronoi >> define_mv_grid_districts # Import potential areas for wind onshore and ground-mounted PV download_re_potential_areas = PythonOperator( diff --git a/src/egon/data/processing/mv_grid_districts.py b/src/egon/data/processing/mv_grid_districts.py index de7b31fc2..45c0ccf90 100644 --- a/src/egon/data/processing/mv_grid_districts.py +++ b/src/egon/data/processing/mv_grid_districts.py @@ -29,7 +29,8 @@ from egon.data import db from egon.data.db import session_scope -from egon.data.processing.substation import EgonHvmvSubstationVoronoi +from egon.data.processing.substation import (EgonHvmvSubstationVoronoi, + EgonHvmvSubstation) Base = declarative_base() metadata = Base.metadata @@ -54,28 +55,6 @@ class Vg250GemClean(Base): geometry = Column(Geometry("POLYGON", 3035), index=True) -class EgonHvmvSubstation(Base): - __tablename__ = "egon_hvmv_substation" - __table_args__ = {"schema": "grid"} - - subst_id = Column(Integer, primary_key=True) - lon = Column(Float) - lat = Column(Float) - point = Column(Geometry("POINT", 4326), index=True) - polygon = Column(Geometry, index=True) - voltage = Column(String) - power_type = Column(String) - substation = Column(String) - osm_id = Column(String) - osm_www = Column(String) - frequency = Column(String) - subst_name = Column(String) - ref = Column(String) - operator = Column(String) - dbahn = Column(String) - status = Column(Integer) - - class HvmvSubstPerMunicipality(Base): __tablename__ = "hvmv_subst_per_municipality" __table_args__ = {"schema": "grid"} @@ -261,7 +240,7 @@ def split_multi_substation_municipalities(): ), ) ).geom.label("geom"), - EgonHvmvSubstationVoronoi.subst_id, + EgonHvmvSubstationVoronoi.bus_id, EgonHvmvSubstationVoronoi.id.label("voronoi_id"), ) .filter(HvmvSubstPerMunicipality.subst_count > 1) @@ -290,7 +269,7 @@ def split_multi_substation_municipalities(): cuts_substation_subquery = ( session.query( VoronoiMunicipalityCuts.id, - EgonHvmvSubstation.subst_id, + EgonHvmvSubstation.bus_id, func.ST_Transform(EgonHvmvSubstation.point, 3035).label( "geom_sub" ), @@ -304,7 +283,7 @@ def split_multi_substation_municipalities(): ) .group_by( VoronoiMunicipalityCuts.id, - EgonHvmvSubstation.subst_id, + EgonHvmvSubstation.bus_id, EgonHvmvSubstation.point, ) .subquery() @@ -314,7 +293,7 @@ def split_multi_substation_municipalities(): ).update( { "subst_count": cuts_substation_subquery.c.subst_count, - "subst_id": cuts_substation_subquery.c.subst_id, + "subst_id": cuts_substation_subquery.c.bus_id, "geom_sub": cuts_substation_subquery.c.geom_sub, }, synchronize_session="fetch", @@ -567,7 +546,7 @@ def merge_polygons_to_grid_district(): # Step 2: Insert municipality polygons with exactly one substation one_substation = ( session.query( - EgonHvmvSubstation.subst_id, + EgonHvmvSubstation.bus_id, func.ST_Multi(HvmvSubstPerMunicipality.geometry).label("geom"), func.ST_Area( func.ST_Multi(HvmvSubstPerMunicipality.geometry) @@ -717,7 +696,7 @@ def nearest_polygon_with_substation( ), #with_substation.c.id func.ST_Distance( - func.ST_Centroid(without_substation.c.geom), + func.ST_Centroid(without_substation.c.geom), func.ST_Centroid(with_substation.c.geom) ) ) diff --git a/src/egon/data/processing/osmtgmod/__init__.py b/src/egon/data/processing/osmtgmod/__init__.py index 60a92e92c..3e08f89a8 100644 --- a/src/egon/data/processing/osmtgmod/__init__.py +++ b/src/egon/data/processing/osmtgmod/__init__.py @@ -429,7 +429,7 @@ def osmtgmod( CREATE TABLE transfer_busses_complete as SELECT DISTINCT ON (osm_id) * FROM (SELECT * FROM grid.egon_ehv_substation - UNION SELECT subst_id, lon, lat, point, polygon, voltage, + UNION SELECT bus_id, lon, lat, point, polygon, voltage, power_type, substation, osm_id, osm_www, frequency, subst_name, ref, operator, dbahn, status FROM grid.egon_hvmv_substation ORDER BY osm_id) as foo; diff --git a/src/egon/data/processing/osmtgmod/substation_otg.sql b/src/egon/data/processing/osmtgmod/substation_otg.sql new file mode 100644 index 000000000..e93690f63 --- /dev/null +++ b/src/egon/data/processing/osmtgmod/substation_otg.sql @@ -0,0 +1,41 @@ +ALTER TABLE grid.egon_hvmv_substation + ADD COLUMN otg_id bigint; + +-- fill table with bus_i from osmtgmod +UPDATE grid.egon_hvmv_substation + SET otg_id = osmtgmod_results.bus_data.bus_i + FROM osmtgmod_results.bus_data + WHERE osmtgmod_results.bus_data.base_kv <= 110 AND (SELECT TRIM(leading 'n' FROM TRIM(leading 'w' FROM grid.egon_hvmv_substation.osm_id))::BIGINT)=osmtgmod_results.bus_data.osm_substation_id; + +DELETE FROM grid.egon_hvmv_substation WHERE otg_id IS NULL; + +UPDATE grid.egon_hvmv_substation + SET bus_id = otg_id; + +ALTER TABLE grid.egon_hvmv_substation + DROP COLUMN otg_id; + + +-- do the same with model_draft.ego_grid_ehv_substation + +-- update model_draft.ego_grid_ehv_substation table with new column of respective osmtgmod bus_i +ALTER TABLE grid.egon_ehv_substation + ADD COLUMN otg_id bigint; + +-- fill table with bus_i from osmtgmod +UPDATE grid.egon_ehv_substation + SET otg_id = osmtgmod_results.bus_data.bus_i + FROM osmtgmod_results.bus_data + WHERE osmtgmod_results.bus_data.base_kv > 110 AND(SELECT TRIM(leading 'n' FROM TRIM(leading 'w' FROM TRIM(leading 'r' FROM grid.egon_ehv_substation.osm_id)))::BIGINT)=osmtgmod_results.bus_data.osm_substation_id; + +DELETE FROM grid.egon_ehv_substation WHERE otg_id IS NULL; + +UPDATE grid.egon_ehv_substation + SET bus_id = otg_id; + +ALTER TABLE grid.egon_ehv_substation + DROP COLUMN otg_id; + + + + diff --git a/src/egon/data/processing/substation/__init__.py b/src/egon/data/processing/substation/__init__.py index ffde1cddd..45a5577a5 100644 --- a/src/egon/data/processing/substation/__init__.py +++ b/src/egon/data/processing/substation/__init__.py @@ -13,10 +13,10 @@ class EgonEhvSubstation(Base): __tablename__ = 'egon_ehv_substation' __table_args__ = {'schema': 'grid'} - subst_id = Column(Integer, - Sequence('egon_ehv_substation_subst_id_seq', schema='grid'), + bus_id = Column(Integer, + Sequence('egon_ehv_substation_bus_id_seq', schema='grid'), server_default= - Sequence('egon_ehv_substation_subst_id_seq', schema='grid').next_value(), + Sequence('egon_ehv_substation_bus_id_seq', schema='grid').next_value(), primary_key=True) lon = Column(Float(53)) lat = Column(Float(53)) @@ -38,10 +38,10 @@ class EgonEhvSubstation(Base): class EgonHvmvSubstation(Base): __tablename__ = 'egon_hvmv_substation' __table_args__ = {'schema': 'grid'} - subst_id = Column(Integer, - Sequence('egon_hvmv_substation_subst_id_seq', schema='grid'), + bus_id = Column(Integer, + Sequence('egon_hvmv_substation_bus_id_seq', schema='grid'), server_default= - Sequence('egon_hvmv_substation_subst_id_seq', schema='grid').next_value(), + Sequence('egon_hvmv_substation_bus_id_seq', schema='grid').next_value(), primary_key=True) lon = Column(Float(53)) lat = Column(Float(53)) @@ -67,7 +67,7 @@ class EgonHvmvSubstationVoronoi(Base): server_default= Sequence('egon_hvmv_substation_voronoi_id_seq', schema='grid').next_value(), primary_key=True) - subst_id = Column(Integer) + bus_id = Column(Integer) geom = Column(Geometry('Multipolygon', 4326)) @@ -79,7 +79,7 @@ class EgonEhvSubstationVoronoi(Base): server_default= Sequence('egon_ehv_substation_voronoi_id_seq', schema='grid').next_value(), primary_key=True) - subst_id = Column(Integer) + bus_id = Column(Integer) geom = Column(Geometry('Multipolygon', 4326)) def create_tables(): @@ -136,13 +136,13 @@ def create_tables(): db.execute_sql( f"""DROP SEQUENCE IF EXISTS {cfg_hvmv['processed']['schema']}. - {cfg_hvmv['processed']['table']}_subst_id_seq CASCADE;""" + {cfg_hvmv['processed']['table']}_bus_id_seq CASCADE;""" ) db.execute_sql( f"""DROP SEQUENCE IF EXISTS {cfg_ehv['processed']['schema']}. - {cfg_ehv['processed']['table']}_subst_id_seq CASCADE;""" + {cfg_ehv['processed']['table']}_bus_id_seq CASCADE;""" ) engine = db.engine() @@ -289,14 +289,14 @@ def create_voronoi(): db.execute_sql( f""" UPDATE {schema}.{voronoi_table} AS t1 - SET subst_id = t2.subst_id + SET bus_id = t2.bus_id FROM (SELECT voi.id AS id, - sub.subst_id ::integer AS subst_id + sub.bus_id ::integer AS bus_id FROM {schema}.{voronoi_table} AS voi, {schema}.{substation_table} AS sub WHERE voi.geom && sub.point AND ST_CONTAINS(voi.geom,sub.point) - GROUP BY voi.id,sub.subst_id + GROUP BY voi.id,sub.bus_id )AS t2 WHERE t1.id = t2.id; """