Merge branch 'dev' into fixes/#349-add-egon-prefix-to-mv_grid_districts

openego · Sep 6, 2021 · c78e287 · c78e287
2 parents 156e555 + 06bad1a
commit c78e287
Show file tree

Hide file tree

Showing 9 changed files with 303 additions and 242 deletions.
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -128,8 +128,8 @@ Added
   `#321 <https://github.com/openego/eGon-data/issues/358>`_
 * Integrate existing CHP and extdended CHP > 10MW_el
   `#266 <https://github.com/openego/eGon-data/issues/266>`_
-
-.. _PR #159: https://github.com/openego/eGon-data/pull/159
+* Add random seed to CLI parameters
+  `#351 <https://github.com/openego/eGon-data/issues/351>`_
 
 
 Changed
@@ -239,6 +239,8 @@ Bug fixes
   `#387 <https://github.com/openego/eGon-data/issues/390>`_
 * Add xlrd and openpyxl to installation setup
   `#400 <https://github.com/openego/eGon-data/issues/400>`_
+* Store files of OSM, zensus and VG250 in working dir
+  `#341 <https://github.com/openego/eGon-data/issues/341>`_
 * Remove hard-coded slashes in file paths to ensure Windows compatibility
   `#398 <https://github.com/openego/eGon-data/issues/398>`_
 * Add missing dependency in pipeline.py

diff --git a/src/egon/data/airflow/dags/pipeline.py b/src/egon/data/airflow/dags/pipeline.py
@@ -3,6 +3,7 @@
 from airflow.operators.postgres_operator import PostgresOperator
 from airflow.operators.python_operator import PythonOperator
 from airflow.utils.dates import days_ago
+import airflow
 import importlib_resources as resources
 
 from egon.data.datasets import database
@@ -17,41 +18,39 @@
 from egon.data.datasets.electricity_demand_etrago import ElectricalLoadEtrago
 from egon.data.datasets.era5 import WeatherData
 from egon.data.datasets.etrago_setup import EtragoSetup
+from egon.data.datasets.gas_prod import GasProduction
 from egon.data.datasets.heat_demand import HeatDemandImport
 from egon.data.datasets.heat_etrago import HeatEtrago
 from egon.data.datasets.heat_supply import HeatSupply
-from egon.data.datasets.industry import IndustrialDemandCurves
 from egon.data.datasets.industrial_sites import MergeIndustrialSites
-from egon.data.datasets.renewable_feedin import RenewableFeedin
-from egon.data.datasets.osm import OpenStreetMap
-from egon.data.datasets.osmtgmod import Osmtgmod
+from egon.data.datasets.industry import IndustrialDemandCurves
 from egon.data.datasets.mastr import mastr_data_setup
-from egon.data.datasets.re_potential_areas import re_potential_area_setup
-from egon.data.datasets.society_prognosis import SocietyPrognosis
 from egon.data.datasets.mv_grid_districts import mv_grid_districts_setup
+from egon.data.datasets.osm import OpenStreetMap
+from egon.data.datasets.osmtgmod import Osmtgmod
 from egon.data.datasets.power_plants import PowerPlants
+from egon.data.datasets.re_potential_areas import re_potential_area_setup
+from egon.data.datasets.renewable_feedin import RenewableFeedin
 from egon.data.datasets.scenario_capacities import ScenarioCapacities
 from egon.data.datasets.scenario_parameters import ScenarioParameters
+from egon.data.datasets.society_prognosis import SocietyPrognosis
 from egon.data.datasets.vg250 import Vg250
 from egon.data.datasets.vg250_mv_grid_districts import Vg250MvGridDistricts
 from egon.data.datasets.zensus_mv_grid_districts import ZensusMvGridDistricts
 from egon.data.datasets.zensus_vg250 import ZensusVg250
+
 from egon.data.datasets.gas_prod import GasProduction
 from egon.data.datasets.industrial_gas_demand import IndustrialGasDemand
 import airflow
 
-import egon.data.importing.zensus as import_zs
+import egon.data.importing.zensus as import_
 import egon.data.importing.gas_grid as gas_grid
-import egon.data.processing.power2gas as power2gas
-import egon.data.processing.substation as substation
+import egon.data.importing.zensus as import_zs
+import egon.data.processing.calculate_dlr as dlr
 import egon.data.processing.gas_areas as gas_areas
 import egon.data.processing.loadarea as loadarea
-import egon.data.processing.calculate_dlr as dlr
-
-
-
-from egon.data import db
-
+import egon.data.processing.power2gas as power2gas
+import egon.data.processing.substation as substation
 
 with airflow.DAG(
     "egon-data-processing-pipeline",
@@ -123,21 +122,23 @@
     population_import >> zensus_misc_import
 
     # Combine Zensus and VG250 data
-    zensus_vg250 = ZensusVg250(
-        dependencies=[vg250, population_import])
+    zensus_vg250 = ZensusVg250(dependencies=[vg250, population_import])
 
     # DemandRegio data import
-    demandregio = DemandRegio(dependencies=[
-        setup, vg250, scenario_parameters, data_bundle])
-    demandregio_demand_cts_ind = tasks['demandregio.insert-cts-ind-demands']
-
+    demandregio = DemandRegio(
+        dependencies=[setup, vg250, scenario_parameters, data_bundle]
+    )
+    demandregio_demand_cts_ind = tasks["demandregio.insert-cts-ind-demands"]
 
     # Society prognosis
-    society_prognosis = SocietyPrognosis(dependencies=[
-        demandregio,
-        zensus_vg250,
-        population_import,
-        zensus_misc_import])
+    society_prognosis = SocietyPrognosis(
+        dependencies=[
+            demandregio,
+            zensus_vg250,
+            population_import,
+            zensus_misc_import,
+        ]
+    )
 
     # Distribute household electrical demands to zensus cells
     household_electricity_demand_annual = HouseholdElectricityDemand(
@@ -163,8 +164,7 @@
 
     # setting etrago input tables
 
-    setup_etrago = EtragoSetup(
-        dependencies=[setup])
+    setup_etrago = EtragoSetup(dependencies=[setup])
     etrago_input_data = tasks["etrago_setup.create-tables"]
 
     # Retrieve MaStR data
@@ -197,18 +197,21 @@
         autocommit=True,
     )
 
-
     osm_add_metadata >> substation_tables >> substation_functions
     substation_functions >> hvmv_substation_extraction
     substation_functions >> ehv_substation_extraction
     vg250_clean_and_prepare >> hvmv_substation_extraction
     vg250_clean_and_prepare >> ehv_substation_extraction
 
     # osmTGmod ehv/hv grid model generation
-    osmtgmod = Osmtgmod(dependencies=[osm_download,
-                                         ehv_substation_extraction,
-                                         hvmv_substation_extraction,
-                                         setup_etrago])
+    osmtgmod = Osmtgmod(
+        dependencies=[
+            osm_download,
+            ehv_substation_extraction,
+            hvmv_substation_extraction,
+            setup_etrago,
+        ]
+    )
     osmtgmod.insert_into(pipeline)
     osmtgmod_pypsa = tasks["osmtgmod.to-pypsa"]
     osmtgmod_substation = tasks["osmtgmod_substation"]
@@ -235,8 +238,8 @@
 
     # Future heat demand calculation based on Peta5_0_1 data
     heat_demand_Germany = HeatDemandImport(
-        dependencies=[vg250, scenario_parameters, zensus_vg250])
-
+        dependencies=[vg250, scenario_parameters, zensus_vg250]
+    )
 
     # Gas grid import
     gas_grid_insert_data = PythonOperator(
@@ -262,12 +265,13 @@
         python_callable=gas_areas.create_voronoi,
     )
 
-    gas_grid_insert_data  >> create_gas_polygons
+    gas_grid_insert_data >> create_gas_polygons
     vg250_clean_and_prepare >> create_gas_polygons
-    
+
     # Gas prod import
     gas_production_insert_data = GasProduction(
-        dependencies=[create_gas_polygons])
+        dependencies=[create_gas_polygons]
+    )
 
     # Insert industrial gas demand
     industrial_gas_demand = IndustrialGasDemand( 
@@ -276,7 +280,7 @@
     # Extract landuse areas from osm data set
     create_landuse_table = PythonOperator(
         task_id="create-landuse-table",
-        python_callable=loadarea.create_landuse_table
+        python_callable=loadarea.create_landuse_table,
     )
 
     landuse_extraction = PostgresOperator(
@@ -291,7 +295,9 @@
     vg250_clean_and_prepare >> landuse_extraction
 
     # Import weather data
-    weather_data = WeatherData(dependencies=[setup, scenario_parameters, vg250])
+    weather_data = WeatherData(
+        dependencies=[setup, scenario_parameters, vg250]
+    )
     download_weather_data = tasks["era5.download-era5"]
 
     renewable_feedin = RenewableFeedin(dependencies=[weather_data, vg250])
@@ -302,9 +308,9 @@
 
     # District heating areas demarcation
     district_heating_areas = DistrictHeatingAreas(
-        dependencies=[heat_demand_Germany, scenario_parameters])
-    import_district_heating_areas = tasks[
-        "district_heating_areas.demarcation"]
+        dependencies=[heat_demand_Germany, scenario_parameters]
+    )
+    import_district_heating_areas = tasks["district_heating_areas.demarcation"]
 
     zensus_misc_import >> import_district_heating_areas
 
@@ -323,27 +329,27 @@
     )
 
     map_zensus_grid_districts = tasks["zensus_mv_grid_districts.mapping"]
-
+
+    # Map federal states to mv_grid_districts
+    vg250_mv_grid_districts = Vg250MvGridDistricts(
+        dependencies=[vg250, mv_grid_districts]
+    )
+
     # Distribute electrical CTS demands to zensus grid
     cts_electricity_demand_annual = CtsElectricityDemand(
         dependencies=[
             demandregio,
             zensus_vg250,
+            zensus_mv_grid_districts,
             heat_demand_Germany,
             etrago_input_data,
             household_electricity_demand_annual,
-            zensus_mv_grid_districts
         ]
     )
 
     elec_cts_demands_zensus = tasks[
-        'electricity_demand.distribute-cts-demands']
-
-
-    # Map federal states to mv_grid_districts
-    vg250_mv_grid_districts = Vg250MvGridDistricts(
-        dependencies=[vg250, mv_grid_districts]
-    )
+        "electricity_demand.distribute-cts-demands"
+    ]
 
     # Power plants
     power_plants = PowerPlants(
@@ -375,9 +381,7 @@
     map_zensus_grid_districts >> solar_rooftop_etrago
 
     # CHP locations
-    chp = Chp(
-        dependencies=[mv_grid_districts,
-                      mastr_data])
+    chp = Chp(dependencies=[mv_grid_districts, mastr_data])
 
     chp_locations_nep = tasks["chp.insert-chp-egon2035"]
     chp_heat_bus = tasks["chp.assign-heat-bus"]
@@ -388,13 +392,20 @@
 
     # Heat supply
     heat_supply = HeatSupply(
-        dependencies=[data_bundle, zensus_mv_grid_districts,
-                      district_heating_areas, power_plants,
-                      zensus_mv_grid_districts, chp])
+        dependencies=[
+            data_bundle,
+            zensus_mv_grid_districts,
+            district_heating_areas,
+            power_plants,
+            zensus_mv_grid_districts,
+            chp,
+        ]
+    )
 
     # Heat to eTraGo
     heat_etrago = HeatEtrago(
-        dependencies=[heat_supply, mv_grid_districts, setup_etrago])
+        dependencies=[heat_supply, mv_grid_districts, setup_etrago]
+    )
 
     heat_etrago_buses = tasks["heat_etrago.buses"]
     heat_etrago_supply = tasks["heat_etrago.supply"]

diff --git a/src/egon/data/cli.py b/src/egon/data/cli.py
@@ -142,6 +142,19 @@
     show_default=True,
 )
 
+@click.option(
+    "--random-seed",
+    default=42,
+    metavar="RANDOM_SEED",
+    help=(
+        "Random seed used by some tasks in the pipeline to ensure "
+        " deterministic behaviour. All published results in the eGon project "
+        " will be created with the default value so keep it if you want to "
+        " make sure to get the same results."
+    ),
+    show_default=True,
+)
+
 @click.version_option(version=egon.data.__version__)
 @click.pass_context
 def egon_data(context, **kwargs):

diff --git a/src/egon/data/datasets.yml b/src/egon/data/datasets.yml
@@ -6,8 +6,8 @@ openstreetmap:
       stylefile: "oedb.style"
     target:
       table_prefix: "osm"
-      path: "germany-210101.osm.pbf"
-      path_testmode: "schleswig-holstein-210101.osm.pbf"
+      file: "germany-210101.osm.pbf"
+      file_testmode: "schleswig-holstein-210101.osm.pbf"
   processed:
     schema: "openstreetmap"
     tables:
@@ -23,7 +23,7 @@ vg250:
     source:
       url: "https://daten.gdz.bkg.bund.de/produkte/vg/vg250_ebenen_0101/2020/vg250_01-01.geo84.shape.ebenen.zip"
     target:
-      path: "vg250_01-01.geo84.shape.ebenen.zip"
+      file: "vg250_01-01.geo84.shape.ebenen.zip"
   processed:
     schema: "boundaries"
     file_table_map:
@@ -38,7 +38,7 @@ zensus_population:
     source:
       url: "https://www.zensus2011.de/SharedDocs/Downloads/DE/Pressemitteilung/DemografischeGrunddaten/csv_Bevoelkerung_100m_Gitter.zip?__blob=publicationFile&v=3"
     target:
-      path: "csv_Bevoelkerung_100m_Gitter.zip"
+      file: "csv_Bevoelkerung_100m_Gitter.zip"
   processed:
     schema: "society"
     table: "destatis_zensus_population_per_ha"
@@ -51,7 +51,7 @@ zensus_misc:
         - "https://www.zensus2011.de/SharedDocs/Downloads/DE/Pressemitteilung/DemografischeGrunddaten/csv_Wohnungen_100m_Gitter.zip?__blob=publicationFile&v=5"
   processed:
     schema: "society"
-    path_table_map:
+    file_table_map:
       "csv_Haushalte_100m_Gitter.zip": "egon_destatis_zensus_household_per_ha"
       "csv_Gebaeude_100m_Gitter.zip": "egon_destatis_zensus_building_per_ha"
       "csv_Wohnungen_100m_Gitter.zip": "egon_destatis_zensus_apartment_per_ha"