RA Toolkit end-to-end pipeline enhancements

data_toolkit/manual_adjustments.py

@@ -24,6 +24,7 @@
 from data_toolkit.project.project_data_filters_common import (
     get_eia860_sql_filter_string,
     DISAGG_PROJECT_NAME_STR,
+    AGG_PROJECT_NAME_STR,
 )
 
 # Storage durations
@@ -128,7 +129,7 @@ def make_copy_files(
 
 def add_battery_durations(
     conn,
-    disagg_project_name_str,
+    agg_project_name_str,
     study_year,
     eia860_sql_filter_string,
     csv_location,
@@ -147,16 +148,17 @@ def add_battery_durations(
 
     for tech in tech_dur_dict.keys():
         sql = f"""
-            SELECT {disagg_project_name_str} AS project, 
+            SELECT {agg_project_name_str} AS project,
             {study_year} as period
             FROM raw_data_eia860_generators
             JOIN user_defined_eia_gridpath_key ON
-                    raw_data_eia860_generators.prime_mover_code = 
+                    raw_data_eia860_generators.prime_mover_code =
                     user_defined_eia_gridpath_key.prime_mover_code
                     AND energy_source_code_1 = energy_source_code
             WHERE 1 = 1
             AND {eia860_sql_filter_string}
             AND raw_data_eia860_generators.prime_mover_code = '{tech}'
+            GROUP BY project
             ;
         """
         relevant_projects_df = pd.read_sql(sql, conn)
@@ -218,7 +220,7 @@ def main(args=None):
 
     add_battery_durations(
         conn=conn,
-        disagg_project_name_str=DISAGG_PROJECT_NAME_STR,
+        agg_project_name_str=AGG_PROJECT_NAME_STR,
         study_year=parsed_args.study_year,
         eia860_sql_filter_string=get_eia860_sql_filter_string(
             study_year=parsed_args.study_year, region=parsed_args.region

data_toolkit/project/availability/eia860_to_project_availability_input_csvs.py

@@ -106,7 +106,7 @@ def get_project_availability(
     subscenario_name,
 ):
     sql = f"""
-    SELECT {disagg_project_name_str} AS project, 
+    SELECT {agg_project_name_str} AS project,
     'exogenous' AS availability_type,
     NULL AS exogenous_availability_independent_scenario_id,
     NULL AS exogenous_availability_weather_scenario_id,
@@ -115,28 +115,12 @@ def get_project_availability(
     NULL AS endogenous_availability_scenario_id
     FROM raw_data_eia860_generators
     JOIN user_defined_eia_gridpath_key ON
-            raw_data_eia860_generators.prime_mover_code = 
+            raw_data_eia860_generators.prime_mover_code =
             user_defined_eia_gridpath_key.prime_mover_code
             AND energy_source_code_1 = energy_source_code
      WHERE 1 = 1
      AND {eia860_sql_filter_string}
-     AND NOT {var_gen_filter_str}
-     AND NOT {hydro_filter_str}
-    UNION
-    -- Aggregated units include wind, offshore wind, solar, and hydro
-    SELECT {agg_project_name_str} AS project,
-        'exogenous' AS availability_type,
-    NULL AS exogenous_availability_independent_scenario_id,
-    NULL AS exogenous_availability_weather_scenario_id,
-    NULL AS exogenous_availability_independent_bt_hrz_scenario_id,
-    NULL AS exogenous_availability_weather_bt_hrz_scenario_id,
-    NULL AS endogenous_availability_scenario_id
-    FROM raw_data_eia860_generators
-    JOIN user_defined_eia_gridpath_key
-    USING (prime_mover_code)
-    WHERE 1 = 1
-    AND {eia860_sql_filter_string}
-    AND ({var_gen_filter_str} OR {hydro_filter_str})
+     GROUP BY project
     """
 
     df = pd.read_sql(sql, conn)

data_toolkit/project/capacity_specified/eia860_to_project_specified_capacity_input_csvs.py

@@ -109,51 +109,29 @@ def get_project_capacity(
     subscenario_name,
 ):
     sql = f"""
-    SELECT {disagg_project_name_str} AS project, 
+    SELECT {agg_project_name_str} AS project,
         {study_year} as period,
-        capacity_mw AS specified_capacity_mw,
+        SUM(capacity_mw) AS specified_capacity_mw,
         NULL AS specified_energy_mwh,
         NULL AS shaping_capacity_mw,
         NULL AS hyb_gen_specified_capacity_mw,
         NULL AS hyb_stor_specified_capacity_mw,
-        CASE 
-            WHEN raw_data_eia860_generators.prime_mover_code NOT IN ('BA', 
+        SUM(CASE
+            WHEN raw_data_eia860_generators.prime_mover_code NOT IN ('BA',
             'ES', 'FW', 'PS') THEN NULL
             ELSE energy_storage_capacity_mwh
-        END
-            AS specified_stor_capacity_mwh,
+        END) AS specified_stor_capacity_mwh,
         NULL AS fuel_production_capacity_fuelunitperhour,
         NULL AS fuel_release_capacity_fuelunitperhour,
         NULL AS fuel_storage_capacity_fuelunit
     FROM raw_data_eia860_generators
     JOIN user_defined_eia_gridpath_key ON
-            raw_data_eia860_generators.prime_mover_code = 
+            raw_data_eia860_generators.prime_mover_code =
             user_defined_eia_gridpath_key.prime_mover_code
             AND energy_source_code_1 = energy_source_code
      WHERE 1 = 1
      AND {eia860_sql_filter_string}
-     AND NOT {var_gen_filter_str}
-     AND NOT {hydro_filter_str}
-    UNION
-    -- Aggregated units include wind, offshore wind, solar, and hydro
-    SELECT {agg_project_name_str} AS project,
-        {study_year} as period,
-        SUM(capacity_mw) AS specified_capacity_mw,
-        NULL AS specified_energy_mwh,
-        NULL AS shaping_capacity_mw,
-        NULL AS hyb_gen_specified_capacity_mw,
-        NULL AS hyb_stor_specified_capacity_mw,
-        SUM(energy_storage_capacity_mwh) AS specified_stor_capacity_mwh,
-        NULL AS fuel_production_capacity_fuelunitperhour,
-        NULL AS fuel_release_capacity_fuelunitperhour,
-        NULL AS fuel_storage_capacity_fuelunit
-    FROM raw_data_eia860_generators
-    JOIN user_defined_eia_gridpath_key
-    USING (prime_mover_code)
-    WHERE 1 = 1
-    AND {eia860_sql_filter_string}
-    AND ({var_gen_filter_str} OR {hydro_filter_str})
-    GROUP BY project
+     GROUP BY project
     ;
     """
 

data_toolkit/project/fixed_cost/eia860_to_project_fixed_cost_input_csvs.py

@@ -107,54 +107,29 @@ def get_project_fixed_cost(
     subscenario_name,
 ):
     sql = f"""
-    SELECT {disagg_project_name_str} AS project,
+    SELECT {agg_project_name_str} AS project,
         {study_year} as period,
         0 AS fixed_cost_per_mw_yr,
         0 AS fixed_cost_per_energy_mwh_yr,
         0 AS fixed_cost_per_shaping_mw_yr,
         NULL AS hyb_gen_fixed_cost_per_mw_yr,
         NULL AS hyb_stor_fixed_cost_per_mw_yr,
-        CASE WHEN raw_data_eia860_generators.prime_mover_code NOT IN ('BA', 
-        'ES', 'FW', 'PS') 
-            THEN NULL
-            ELSE 0
+        CASE WHEN gridpath_operational_type = 'stor'
+            THEN 0
+            ELSE NULL
         END
             AS fixed_cost_per_stor_mwh_yr,
         NULL AS fuel_production_capacity_fixed_cost_per_fuelunitperhour_yr,
         NULL AS fuel_release_capacity_fixed_cost_per_fuelunitperhour_yr,
         NULL AS fuel_storage_capacity_fixed_cost_per_fuelunit_yr
     FROM raw_data_eia860_generators
-   JOIN user_defined_eia_gridpath_key ON
-            raw_data_eia860_generators.prime_mover_code = 
+    JOIN user_defined_eia_gridpath_key ON
+            raw_data_eia860_generators.prime_mover_code =
             user_defined_eia_gridpath_key.prime_mover_code
             AND energy_source_code_1 = energy_source_code
      WHERE 1 = 1
      AND {eia860_sql_filter_string}
-     AND NOT {var_gen_filter_str}
-     AND NOT {hydro_filter_str}
-    UNION
-    -- Aggregated units include wind, offshore wind, solar, and hydro
-    SELECT {agg_project_name_str} AS project,
-        {study_year} as period,
-        0 AS specified_fixed_cost_mw,
-        0 AS fixed_cost_per_energy_mwh_yr,
-        0 AS fixed_cost_per_shaping_mw_yr,
-        NULL AS hyb_gen_specified_fixed_cost_mw,
-        NULL AS hyb_stor_specified_fixed_cost_mw,
-        CASE
-            WHEN energy_storage_capacity_mwh IS NULL THEN NULL
-            ELSE 0
-            END 
-            AS fixed_cost_per_stor_mwh_yr,
-        NULL AS fuel_production_fixed_cost_fuelunitperhour,
-        NULL AS fuel_release_fixed_cost_fuelunitperhour,
-        NULL AS fuel_storage_fixed_cost_fuelunit
-    FROM raw_data_eia860_generators
-    JOIN user_defined_eia_gridpath_key
-    USING (prime_mover_code)
-    WHERE 1 = 1
-    AND {eia860_sql_filter_string}
-    AND ({var_gen_filter_str} OR {hydro_filter_str})
+     GROUP BY project
     ;
     """
 

data_toolkit/project/load_zones/eia860_to_project_load_zone_input_csvs.py

@@ -106,26 +106,16 @@ def get_project_load_zones(
     subscenario_name,
 ):
     sql = f"""
-    SELECT {disagg_project_name_str} AS project, balancing_authority_code_eia AS load_zone
+    SELECT {agg_project_name_str} AS project,
+        balancing_authority_code_eia AS load_zone
     FROM raw_data_eia860_generators
     JOIN user_defined_eia_gridpath_key ON
-            raw_data_eia860_generators.prime_mover_code = 
+            raw_data_eia860_generators.prime_mover_code =
             user_defined_eia_gridpath_key.prime_mover_code
             AND energy_source_code_1 = energy_source_code
     WHERE 1 = 1
     AND {eia860_sql_filter_string}
-    AND NOT {var_gen_filter_str}
-    AND NOT {hydro_filter_str}
-    -- Aggregated units include wind, offshore wind, solar, and hydro
-    UNION
-    SELECT {agg_project_name_str} AS project,
-        balancing_authority_code_eia AS load_zone
-    FROM raw_data_eia860_generators
-    JOIN user_defined_eia_gridpath_key
-    USING (prime_mover_code)
-    WHERE 1 = 1
-    AND {eia860_sql_filter_string}
-    AND ({var_gen_filter_str} OR {hydro_filter_str})
+    GROUP BY project
     ;
     """
 

data_toolkit/project/opchar/eia860_to_project_opchar_input_csvs.py

@@ -107,6 +107,14 @@ def parse_arguments(args):
     )
     parser.add_argument("-hy_id", "--hydro_operational_chars_scenario_id", default=1)
 
+    parser.add_argument(
+        "-hydro_bt",
+        "--hydro_balancing_type",
+        default=None,
+        help="Override balancing_type_project for hydro projects. "
+        "If not specified, uses gridpath_balancing_type from the key table.",
+    )
+
     parser.add_argument("-q", "--quiet", default=False, action="store_true")
 
     parsed_arguments = parser.parse_known_args(args=args)[0]
@@ -131,6 +139,7 @@ def get_project_opchar(
     hr_id,
     var_id,
     hy_id,
+    hydro_balancing_type=None,
 ):
     # Wind, offshore wind, and PV are aggregated, so treated separately since
     # they are aggregated, so here we make a UNION between tables filtering
@@ -160,33 +169,39 @@ def get_project_opchar(
         variable_generator_profile_scenario_id=f"{var_id}",
     )
 
+    hydro_bt_expr = (
+        f"'{hydro_balancing_type}'"
+        if hydro_balancing_type
+        else "gridpath_balancing_type"
+    )
     hydro_opchars_str = make_opchar_sql_str(
         technology="gridpath_technology",
         operational_type="gridpath_operational_type",
-        balancing_type_project="gridpath_balancing_type",
+        balancing_type_project=hydro_bt_expr,
         variable_om_cost_per_mwh="default_variable_om_cost_per_mwh",
         hydro_operational_chars_scenario_id=f"{hy_id}",
     )
 
     sql = f"""
-     SELECT {disagg_project_name_str} AS project,
+     SELECT {agg_project_name_str} AS project,
          {non_var_opchars_str}
      FROM raw_data_eia860_generators
      JOIN user_defined_eia_gridpath_key ON
-            raw_data_eia860_generators.prime_mover_code = 
+            raw_data_eia860_generators.prime_mover_code =
             user_defined_eia_gridpath_key.prime_mover_code
             AND energy_source_code_1 = energy_source_code
      WHERE 1 = 1
      AND {eia860_sql_filter_string}
      AND NOT {var_gen_filter_str}
      AND NOT {hydro_filter_str}
+     GROUP BY project
      -- Variable gen
      UNION
      SELECT {agg_project_name_str} AS project,
          {var_opchars_str}
      FROM raw_data_eia860_generators
      JOIN user_defined_eia_gridpath_key ON
-            raw_data_eia860_generators.prime_mover_code = 
+            raw_data_eia860_generators.prime_mover_code =
             user_defined_eia_gridpath_key.prime_mover_code
             AND energy_source_code_1 = energy_source_code
      WHERE 1 = 1
@@ -199,7 +214,7 @@ def get_project_opchar(
          {hydro_opchars_str}
      FROM raw_data_eia860_generators
      JOIN user_defined_eia_gridpath_key ON
-            raw_data_eia860_generators.prime_mover_code = 
+            raw_data_eia860_generators.prime_mover_code =
             user_defined_eia_gridpath_key.prime_mover_code
             AND energy_source_code_1 = energy_source_code
      WHERE 1 = 1
@@ -455,6 +470,7 @@ def main(args=None):
         hr_id=parsed_args.heat_rate_curves_scenario_id,
         var_id=parsed_args.variable_generator_profile_scenario_id,
         hy_id=parsed_args.hydro_operational_chars_scenario_id,
+        hydro_balancing_type=parsed_args.hydro_balancing_type,
     )
 
     conn.close()

data_toolkit/project/opchar/hydro/create_hydro_iteration_input_csvs.py

@@ -118,6 +118,15 @@ def parse_arguments(args):
         help="Overwrite existing files.",
     )
 
+    parser.add_argument(
+        "-hydro_bt",
+        "--hydro_balancing_type",
+        default=None,
+        help="Filter to a specific balancing type (e.g., 'day', 'week', 'month'). "
+        "If not specified, all balancing types from "
+        "user_defined_balancing_type_horizons are included.",
+    )
+
     parser.add_argument(
         "-parallel",
         "--n_parallel_projects",
@@ -140,6 +149,7 @@ def calculate_from_project_year_month_data(
     hydro_operational_chars_scenario_name,
     output_directory,
     overwrite,
+    hydro_balancing_type=None,
 ):
     """
     Create hydro project CSVs for a temporal subscenario and a balancing
@@ -153,10 +163,13 @@ def calculate_from_project_year_month_data(
     """).fetchall()]
 
     bt_horizons = [bt_h for bt_h in c.execute("""
-            SELECT DISTINCT balancing_type, horizon 
+            SELECT DISTINCT balancing_type, horizon
             FROM user_defined_balancing_type_horizons;
             """).fetchall()]
 
+    if hydro_balancing_type is not None:
+        bt_horizons = [bt_h for bt_h in bt_horizons if bt_h[0] == hydro_balancing_type]
+
     if overwrite:
         filename = get_filename(
             output_directory,
@@ -266,6 +279,7 @@ def calculate_from_project_year_month_data_pool(pool_datum):
         output_directory,
         overwrite,
         stage_id,
+        hydro_balancing_type,
     ) = pool_datum
 
     calculate_from_project_year_month_data(
@@ -276,6 +290,7 @@ def calculate_from_project_year_month_data_pool(pool_datum):
         output_directory=output_directory,
         overwrite=overwrite,
         stage_id=stage_id,
+        hydro_balancing_type=hydro_balancing_type,
     )
 
 
@@ -341,6 +356,7 @@ def main(args=None):
                 parsed_args.output_directory,
                 parsed_args.overwrite,
                 parsed_args.stage_id,
+                parsed_args.hydro_balancing_type,
             ]
             for prj in projects
         ]