Allow time series for any attribute

carnegie · Mar 27, 2024 · 86398ad · 86398ad
1 parent 1fe604a
commit 86398ad
Show file tree

Hide file tree

Showing 5 changed files with 98 additions and 98 deletions.
diff --git a/run_pypsa.py b/run_pypsa.py
@@ -114,34 +114,31 @@ def dicts_to_pypsa(case_dict, component_list, component_attr):
     n = add_buses_to_network(n, component_list)
 
     for component_dict in component_list:
-        # for generators and loads, add time series to components
-        if component_dict["component"] == "Generator" or component_dict["component"] == "Load":
-            # Add time series to components
-            if "time_series_file" in component_dict:
-                ts_file = os.path.join(case_dict["input_path"],component_dict["time_series_file"])
+        for attribute in component_dict:
+            # Check if attribute is a string and csv file holding a time series
+            if isinstance(component_dict[attribute], str) and component_dict[attribute].endswith('.csv'):
+                logging.info(f"Processing time series file: {component_dict[attribute]}")
+                # Add time series to components
+                ts_file = os.path.join(case_dict["input_path"],component_dict[attribute])
                 try:
                     ts = process_time_series_file(ts_file, case_dict["datetime_start"], case_dict["datetime_end"])
+                    logging.info(f"Time series file: {component_dict[attribute]} processed successfully.")
+                    logging.info(ts)
                 except Exception:  # if time series not found in input path, use csv's in test directory
-                    logging.warning("Time series file not found for " + component_dict["name"] + ". Using time series files in test directory.")
-                    case_dict['input_path'] = "./test"
-                    ts_file = os.path.join(case_dict["input_path"],component_dict["time_series_file"])
-                    ts = process_time_series_file(ts_file, case_dict["datetime_start"], case_dict["datetime_end"])
+                    logging.error("Time series file not found for " + component_dict[attribute] + " of " + component_dict["name"] + ". Now exiting.")
+                    sys.exit(1)
 
                 if ts is not None:
                     # Include time series as snapshots taking every delta_t value
                     n.snapshots = ts.iloc[::case_dict['delta_t'], :].index if case_dict['delta_t'] else ts.index
                     # Add time series to component
-                    if component_dict["component"] == "Generator":
-                        component_dict["p_max_pu"] = ts.iloc[:, 0]
-                    elif component_dict["component"] == "Load":
-                        component_dict["p_set"] = ts.iloc[:, 0]
+                    component_dict[attribute] = ts.iloc[:, 0]
                     # Scale by numerics_scaling, this avoids rounding otherwise done in Gurobi for small numbers and normalize time series if needed
                     component_dict = scale_normalize_time_series(component_dict, case_dict["numerics_scaling"])
                     # Remove time_series_file from component_dict
-                    component_dict.pop("time_series_file")                    
                 else:
-                    logging.warning("Time series file not found for " + component_dict["name"] + ". Skipping component.")
-                    continue
+                    logging.warning("Time series not properly processed for " + component_dict[attribute] + " of " + component_dict["name"] + ". Now exiting.")
+                    sys.exit(1)
 
         # Without time series file, set snaphsots to number of time steps defined in the input file
         if len(n.snapshots) == 1 and case_dict["no_time_steps"] is not None:
@@ -233,7 +230,7 @@ def postprocess_results(n, case_dict):
     time_results_df = pd.concat([time_results_df, n.links_t["p0"].rename(columns=dict(
         zip(n.links_t["p0"].columns.to_list(),
             [name + " dispatch" for name in n.links_t["p0"].columns.to_list()])))], axis=1)
-
+    
     # Collect objective and system cost in one dataframe
     system_cost = (n.statistics()["Capital Expenditure"].sum() + n.statistics()[
         "Operational Expenditure"].sum()) / case_dict["total_hours"]

diff --git a/test/test_case.csv b/test/test_case.csv
@@ -1,78 +1,78 @@
-PyPSA case input file,,,,,,,,,,,,,,,,,,,
-,,,,,,,,,,,,,,,,,,,
-"Everything outside of  the <CASE_DATA> or <COMPONENT_DATA>  flag is for notes, etc.",,,,,,,,,,,,,,,,,,,
-,,,,,,,,,,,,,,,,,,,
-Note that demand has no decisions.,,,,,,,,,,,,,,,,,,,
-"Note that unmet demand is represented a source with a variable cost only, so unmet demand has an output decision.",,,,,,,,,,,,,,,,,,,
-Information about PyPSA components and their attributes can be found here: https://pypsa.readthedocs.io/en/latest/components.html,,,,,,,,,,,,,,,,,,,
-,,,,,,,,,,,,,,,,,,,
-REQUIRED KEYWORDS,,,,,,,,,,,,,,,,,,,
-component,PyPSA component type,,,,,,,,,,,,,,,,,,
-name,Unique name of the component,,,,,,,,,,,,,,,,,,
-bus,"Name of bus from which this technology would get or give its energy (or in the case of link, the giving bus)",,,,,,,,,,,,,,,,,,
-,,,,,,,,,,,,,,,,,,,
-OPTIONAL KEYWORDS,,,,,,,,,,,,,,,,,,,
-time_series_file,Name of time series file that will get loaded,,,,,,,,,,,,,,,,,,
-capital_cost,"Fixed cost, if not defined default is 0",,,,,,,,,,,,,,,,,,
-marginal_cost,"Marginal cost, if not defined default is 0",,,,,,,,,,,,,,,,,,
-max_hours,Hours at max capacity for StorageUnit ,,,,,,,,,,,,,,,,,,
-cyclic_state_of_charge,Assume cyclic state of charge for StorageUnit (Boolean),,,,,,,,,,,,,,,,,,
-efficiency,Efficiency of component,,,,,,,,,,,,,,,,,,
-standing_loss,Losses per hour to state of charge,,,,,,,,,,,,,,,,,,
-,,,,,,,,,,,,,,,,,,,
-CASE_DATA,,,,,,,,,,,,,,,,,,,
-,,,,,,,,,,,,,,,,,,,
-input_path,/carnegie/data/Shared/Labs/Caldeira Lab/Everyone/energy_demand_capacity_data/test_case_solar_wind_demand/,,,,,,,,,,,,,,,,,,
-costs_path,https://raw.githubusercontent.com/PyPSA/technology-data/master/outputs/costs_2020.csv,,,,,,,,,,,,,,,,,,
-output_path,output_data,,,,,,,,,,,,,,,,,,
-case_name,test_case,,,,,,,,,,,,,,,,,,
-filename_prefix,test_prefix,,,,,,,,,,,,,,,,,,
-datetime_start,1/1/2016 0:00,,Note: Dates must be formatted as text (not excel date format),,,,,,,,,,,,,,,,
-datetime_end,1/1/2017 0:00,,,,,,,,,,,,,,,,,,
-delta_t,1,,,,,,,,,,,,,,,,,,
-no_time_steps,8784,Note: this assumes time unit for dt is 'hour',,,,,,,,,,,,,,,,,
-total_hours,8784,,,,,,,,,,,,,,,,,,
-,,,,,,,,,,,,,,,,,,,
-solver,gurobi,,,,,,,,,,,,,,,,,,
-logging_level,warning,,"Note: Can be error, warning, info, or debug and specifies level of detail in terminal output",,,,,,,,,,,,,,,,
-numerics_scaling,1.00E+10,,Note: Factor to avoid rounding in Gurobi solver for small values,,,,,,,,,,,,,,,,
-,,,,,,,,,,,,,,,,,,,
-time_unit,h,,,,,,,,,,,,,,,,,,
-power unit,kW,,,,,,,,,,,,,,,,,,
-currency,$,,,,,,,,,,,,,,,,,,
-,,,,,Note: p_min_pu allow bidirectionality of link,,,,,,,,,,,,,,
-END_CASE_DATA,,,,,,,,Note: Capital costs are the product of hourly fixed costs and time_range,,,,,,,,,,,
-,,,"Note: For Link, bus is interpreted as bus0",,,,Note: p_nom is a factor multiplied to the given capacity,,,,,,,"Note: For StorageUnit, efficiency is interpreted as efficiency_store",,,,,
-MEM vocabulary,,,,,,,,,,,,,,,,,,,
-tech_type,tech_name,,node,,,,normalization,capacity,fixed_cost,,var_cost,,charging_time,,efficiency,,decay_rate,,
-COMPONENT_DATA,,,,,,,,,,,,,,,,,,,
-component,name,carrier,bus,bus1,p_min_pu,time_series_file,normalization,p_nom,capital_cost,,marginal_cost,,max_hours,cyclic_state_of_charge,efficiency,efficiency_dispatch,standing_loss,,
-Generator,solar,solar,bus,,,solar.csv,,,171.6544341,$/time range/kW,,$/kWh,,,,,,,
-Load,load,load,bus,,,demand.csv,,,,,,,,,,,,,
-Generator,natgas,natgas,bus,,,,,,104.0882472,$/time range/kW,0.039088111,$/kWh,,,,,,,
-StorageUnit,battery,battery,bus,,,,,,223.872126,$/time range/kW,,$/kWh,6.008,TRUE,0.9,,0.00000114,1/h,Note: PyPSA costs storage_unit by power cost; cost of energy capacity is effectively capital_cost/max_hours
-Generator,nuclear,nuclear,bus,,,,,,548.7837489,$/time range/kW,0.025047273,$/kWh,,,,,,,
-Generator,wind,wind,bus,,,wind.csv,,,181.4975656,$/time range/kW,,$/kWh,,,,,,,
-Link,electrolysis,electrolysis,bus,h2,,,,,43.92,$/time range/kW,,$/kWh,,,0.7,,,,
-Store,h2_storage,h2_storage,h2,,,,,,0.140544,$/time range/kWh,,$/kWh,,,,,,,
-Link,fuel_cell,fuel_cell,h2,bus,,,,,17.568,$/time range/kW,,$/kWh,,,0.5,,,,
-,,,,,,,,,,,,,,,,,,,
-END_COMPONENT_DATA,,,,,,,,,,,,,,,,,,,
-,,,,,,,,,,,,,,,,,,,
-"Note that any information that is in a column without an attribute header is consider a comment, and not used.",,,,,,,,,,,,,,,,,,,
-"Note that for MEM, storage is in energy units whereas for PyPSA it is in power units.",,,,,,,,,,,,,,,,,,,
-"Note that H46-H52  contain formulas, and our PyPSA front end will read this in as a value.",,,,,,,,,,,,,,,,,,,
-"Note: If there is a # in front of component (e.g. #Generator), this row will be ignored",,,,,,,,,,,,,,,,,,,
-,,,,,,,,,,,,,,,,,,,
-Cost calculations,,,,,,,,,,,,,,,,,,,
-,Discount rate,0.07,,,,,,,,,,,,,,,,,
-,name,Overnight cost [$/kW],Fixed O&M cost [$/kWyear],Capital recovery factor [%/year],Lifetime [years],Annual fixed costs [$/year],Variable O&M [$/kWh],Fuel cost [$/kWh],Efficiency,,Hourly fixed costs,,,,,,,,
-,solar,1851,22.02,0.080586404,30,171.1854329,,,,,0.019541716,$/h/kW,,,,,,,
-,natgas,982,11.11,0.094392926,20,103.8038531,0.00354,0.0191,0.5373,,0.011849755,$/h/kW,,,,,,,
-,battery,261,,0.142377503,10,37.16052821,,,,,0.004242069,$/h/kW,,,,,,,
-,nuclear,5946,101.28,0.075009139,40,547.2843397,0.00232,0.0075,0.33,,0.062475381,$/h/kW,,,,,,,
-,wind,1657,47.47,0.080586404,30,181.0016706,,,,,0.020662291,$/h/kW,,,,,,,
-,electrolysis,,,,,,,,,,0.005,$/h/kW,,,,,,,
-,h2_storage,,,,,,,,,,0.000016,$/h/kW,,,,,,,
-,fuel_cell,,,,,,,,,,0.002,$/h/kW,,,,,,,
-,"Note: This is a test case, the costs aren't meant to be very realistic but provide reproducibility in tests",,,,,,,,,,,,,,,,,,
+PyPSA case input file,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,
+"Everything outside of  the <CASE_DATA> or <COMPONENT_DATA>  flag is for notes, etc.",,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,
+Note that demand has no decisions.,,,,,,,,,,,,,,,,,
+"Note that unmet demand is represented a source with a variable cost only, so unmet demand has an output decision.",,,,,,,,,,,,,,,,,
+Information about PyPSA components and their attributes can be found here: https://pypsa.readthedocs.io/en/latest/components.html,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,
+REQUIRED KEYWORDS,,,,,,,,,,,,,,,,,
+component,PyPSA component type,,,,,,,,,,,,,,,,
+name,Unique name of the component,,,,,,,,,,,,,,,,
+bus,"Name of bus from which this technology would get or give its energy (or in the case of link, the giving bus)",,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,
+OPTIONAL KEYWORDS,,,,,,,,,,,,,,,,,
+time_series_file,Name of time series file that will get loaded,,,,,,,,,,,,,,,,
+capital_cost,"Fixed cost, if not defined default is 0",,,,,,,,,,,,,,,,
+marginal_cost,"Marginal cost, if not defined default is 0",,,,,,,,,,,,,,,,
+max_hours,Hours at max capacity for StorageUnit ,,,,,,,,,,,,,,,,
+cyclic_state_of_charge,Assume cyclic state of charge for StorageUnit (Boolean),,,,,,,,,,,,,,,,
+efficiency,Efficiency of component,,,,,,,,,,,,,,,,
+standing_loss,Losses per hour to state of charge,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,
+CASE_DATA,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,
+input_path,test/,,,,,,,,,,,,,,,,
+costs_path,https://raw.githubusercontent.com/PyPSA/technology-data/master/outputs/costs_2020.csv,,,,,,,,,,,,,,,,
+output_path,output_data,,,,,,,,,,,,,,,,
+case_name,test_case,,,,,,,,,,,,,,,,
+filename_prefix,test_prefix,,,,,,,,,,,,,,,,
+datetime_start,2016-01-01 00:00:00,,Note: Dates must be formatted as text (not excel date format),,,,,,,,,,,,,,
+datetime_end,2017-01-01 0:00:00,,,,,,,,,,,,,,,,
+delta_t,1,,,,,,,,,,,,,,,,
+no_time_steps,8784,Note: this assumes time unit for dt is 'hour',,,,,,,,,,,,,,,
+total_hours,8784,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,
+solver,gurobi,,,,,,,,,,,,,,,,
+logging_level,warning,,"Note: Can be error, warning, info, or debug and specifies level of detail in terminal output",,,,,,,,,,,,,,
+numerics_scaling,1.00E+00,,Note: Factor to avoid rounding in Gurobi solver for small values,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,
+time_unit,h,,,,,,,,,,,,,,,,
+power_unit,kW,,,,,,,,,,,,,,,,
+currency,$,,,,,,,,,,,,,,,,
+,,,,,Note: p_min_pu allow bidirectionality of link,,,,,,,,,,,,
+END_CASE_DATA,,,,,,,,Note: Capital costs are the product of hourly fixed costs and time_range,,,,,,,,,
+,,,"Note: For Link, bus is interpreted as bus0",,,,Note: p_nom is a factor multiplied to the given capacity,,,,,,,"Note: For StorageUnit, efficiency is interpreted as efficiency_store",,,
+,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,
+COMPONENT_DATA,,,,,,,,,,,,,,,,,
+component,name,carrier,bus,bus1,p_set,p_max_pu,capital_cost,,marginal_cost,,max_hours,cyclic_state_of_charge,efficiency,efficiency_dispatch,standing_loss,,
+Generator,solar,solar,bus,,,solar.csv,171.6544341,$/time range/kW,,$/kWh,,,,,,,
+Load,load,load,bus,,demand.csv,,,,,,,,,,,,
+Generator,natgas,natgas,bus,,,,104.0882472,$/time range/kW,0.039088111,$/kWh,,,,,,,
+StorageUnit,battery,battery,bus,,,,223.872126,$/time range/kW,0.01,$/kWh,6.008,TRUE,0.9,,0.00000114,1/h,Note: PyPSA costs storage_unit by power cost; cost of energy capacity is effectively capital_cost/max_hours
+Generator,nuclear,nuclear,bus,,,,548.7837489,$/time range/kW,0.025047273,$/kWh,,,,,,,
+Generator,wind,wind,bus,,,wind.csv,181.4975656,$/time range/kW,,$/kWh,,,,,,,
+Link,electrolysis,electrolysis,bus,h2,,,43.92,$/time range/kW,0.015,$/kWh,,,0.7,,,,
+Store,h2_storage,h2_storage,h2,,,,0.140544,$/time range/kWh,,$/kWh,,TRUE,,,4.00E-06,,
+Link,fuel_cell,fuel_cell,h2,bus,,,17.568,$/time range/kW,,$/kWh,,,0.5,,,,
+,,,,,,,,,,,,,,,,,
+END_COMPONENT_DATA,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,
+"Note that any information that is in a column without an attribute header is consider a comment, and not used.",,,,,,,,,,,,,,,,,
+"Note that for MEM, storage is in energy units whereas for PyPSA it is in power units.",,,,,,,,,,,,,,,,,
+"Note that H46-H52  contain formulas, and our PyPSA front end will read this in as a value.",,,,,,,,,,,,,,,,,
+"Note: If there is a # in front of component (e.g. #Generator), this row will be ignored",,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,
+Cost calculations,,,,,,,,,,,,,,,,,
+,Discount rate,0.07,,,,,,,,,,,,,,,
+,name,Overnight cost [$/kW],Fixed O&M cost [$/kWyear],Capital recovery factor [%/year],Lifetime [years],Annual fixed costs [$/year],Variable O&M [$/kWh],Fuel cost [$/kWh],Efficiency,,Hourly fixed costs,,,,,,
+,solar,1851,22.02,0.080586404,30,171.1854329,,,,,0.019541716,$/h/kW,,,,,
+,natgas,982,11.11,0.094392926,20,103.8038531,0.00354,0.0191,0.5373,,0.011849755,$/h/kW,,,,,
+,battery,261,,0.142377503,10,37.16052821,,,,,0.004242069,$/h/kW,,,,,
+,nuclear,5946,101.28,0.075009139,40,547.2843397,0.00232,0.0075,0.33,,0.062475381,$/h/kW,,,,,
+,wind,1657,47.47,0.080586404,30,181.0016706,,,,,0.020662291,$/h/kW,,,,,
+,electrolysis,,,,,,,,,,0.005,$/h/kW,,,,,
+,h2_storage,,,,,,,,,,0.000016,$/h/kW,,,,,
+,fuel_cell,,,,,,,,,,0.002,$/h/kW,,,,,
+,"Note: This is a test case, the costs aren't meant to be very realistic but provide reproducibility in tests",,,,,,,,,,,,,,,,
diff --git a/test/test_case.xlsx b/test/test_case.xlsx
diff --git a/test/test_case_db_values.xlsx b/test/test_case_db_values.xlsx
diff --git a/utilities/read_input.py b/utilities/read_input.py
@@ -131,7 +131,7 @@ def read_component_data(comp_dict, attr, val, technology, costs_df):
     if attr != None:
         read_attr = None
         # if "name", "bus", or "time_series_file" is in attr or value can be converted to a float, use that
-        if (val != None and (any(x in attr for x in ['name', 'bus', 'carrier', 'time_series_file']) or is_number(val) or '=' in val)):
+        if (val != None and (any(x in attr for x in ['name', 'bus', 'carrier']) or is_number(val) or '=' in val)):
             comp_dict[attr] = val
         # if otherwise value is a string, use database value if the string is just 'db'
         # if first two letters are db use the rest of the string as the attribute name
@@ -147,9 +147,12 @@ def read_component_data(comp_dict, attr, val, technology, costs_df):
                 else:
                     val = val.replace('db_','')
                     read_attr = val
+            elif val.endswith('.csv'):
+                comp_dict[attr] = val
             else:
-                logging.error('Tried to read in a string that is not a number, name, or contains "db" to indicate use a database value. Failed = '+val + ' for attribute ' + attr + ' for component ' + comp_dict["component"] + ' ' + comp_dict["name"])
-
+                logging.error('Failed to read in '+val + ' for attribute ' + attr + ' for component ' + comp_dict["component"] + ' ' + comp_dict["name"])
+                logging.error('Exiting now.')
+                exit()
 
         # if read_attr is defined, use it to get the value from the costs dataframe
         if read_attr != None: