diff --git a/devtools/debug-ferc1-etl.ipynb b/devtools/debug-ferc1-etl.ipynb index d2b0c97a20..49263f4b80 100644 --- a/devtools/debug-ferc1-etl.ipynb +++ b/devtools/debug-ferc1-etl.ipynb @@ -113,11 +113,11 @@ "source": [ "from dagster import build_op_context\n", "\n", - "from pudl.extract.ferc1 import raw_xbrl_metadata_json\n", - "from pudl.transform.ferc1 import clean_xbrl_metadata_json\n", + "from pudl.extract.ferc1 import raw_ferc1_xbrl__metadata_json\n", + "from pudl.transform.ferc1 import _core_ferc1_xbrl__metadata_json\n", "\n", "context = build_op_context()\n", - "xbrl_metadata_json_dict = clean_xbrl_metadata_json(raw_xbrl_metadata_json(context))" + "xbrl_metadata_json_dict = _core_ferc1_xbrl__metadata_json(raw_ferc1_xbrl__metadata_json(context))" ] }, { diff --git a/docs/dev/naming_conventions.rst b/docs/dev/naming_conventions.rst index 4993fb77ec..1ce2e5872d 100644 --- a/docs/dev/naming_conventions.rst +++ b/docs/dev/naming_conventions.rst @@ -98,7 +98,7 @@ Naming convention: ``core_{source}__{asset_type}_{asset_name}`` typically contain measurements of processes like net generation or co2 emissions. Examples: - * ``core_ferc714__hourly_demand_pa``, + * ``out_ferc714__hourly_planning_area_demand``, * ``core_ferc1__yearly_plant_in_service``. Output layer diff --git a/docs/release_notes.rst b/docs/release_notes.rst index 76199e8021..ab28568bd9 100644 --- a/docs/release_notes.rst +++ b/docs/release_notes.rst @@ -263,7 +263,8 @@ Data Coverage The newly accessible tables include: * :ref:`core_ferc714__respondent_id` (linking FERC-714 respondents to EIA utilities) - * :ref:`core_ferc714__hourly_demand_pa` (hourly electricity demand by planning area) + * :ref:`out_ferc714__hourly_planning_area_demand` (hourly electricity demand by + planning area) * :ref:`out_ferc714__respondents_with_fips` (annual respondents with county FIPS IDs) * :ref:`out_ferc714__summarized_demand` (annual demand for FERC-714 respondents) @@ -369,7 +370,7 @@ Analysis (:ref:`out_eia861__compiled_geometry_balancing_authorities` and :ref:`out_eia861__compiled_geometry_utilities`), and the estimated total hourly electricity demand for each US state in - :ref:`out_ferc714__hourly_predicted_state_demand`. See :issue:`1973` + :ref:`out_ferc714__hourly_estimated_state_demand`. See :issue:`1973` and :pr:`2550`. Deprecations diff --git a/migrations/versions/2e5b623ab40b_create_intermediate_steam_plants_table_.py b/migrations/versions/2e5b623ab40b_create_intermediate_steam_plants_table_.py deleted file mode 100644 index 4cfc3a559a..0000000000 --- a/migrations/versions/2e5b623ab40b_create_intermediate_steam_plants_table_.py +++ /dev/null @@ -1,31 +0,0 @@ -"""create intermediate steam plants table with plant ids - -Revision ID: 2e5b623ab40b -Revises: 4b08158ae952 -Create Date: 2023-12-19 17:37:33.476337 - -""" -import sqlalchemy as sa -from alembic import op - -# revision identifiers, used by Alembic. -revision = '2e5b623ab40b' -down_revision = '4b08158ae952' -branch_labels = None -depends_on = None - - -def upgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - with op.batch_alter_table('core_ferc1__yearly_steam_plants_sched402', schema=None) as batch_op: - batch_op.drop_column('plant_id_ferc1') - - # ### end Alembic commands ### - - -def downgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - with op.batch_alter_table('core_ferc1__yearly_steam_plants_sched402', schema=None) as batch_op: - batch_op.add_column(sa.Column('plant_id_ferc1', sa.INTEGER(), nullable=True)) - - # ### end Alembic commands ### diff --git a/migrations/versions/955e745a9fbd_add_epacamd_eia_back_to_datasources_enum.py b/migrations/versions/955e745a9fbd_add_epacamd_eia_back_to_datasources_enum.py deleted file mode 100644 index 18b997ac26..0000000000 --- a/migrations/versions/955e745a9fbd_add_epacamd_eia_back_to_datasources_enum.py +++ /dev/null @@ -1,37 +0,0 @@ -"""Add epacamd_eia back to datasources enum - -Revision ID: 955e745a9fbd -Revises: 997d2c7bc7de -Create Date: 2024-01-25 14:15:03.711635 - -""" -import sqlalchemy as sa -from alembic import op - -# revision identifiers, used by Alembic. -revision = '955e745a9fbd' -down_revision = '997d2c7bc7de' -branch_labels = None -depends_on = None - - -def upgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - with op.batch_alter_table('core_pudl__codes_datasources', schema=None) as batch_op: - batch_op.alter_column('datasource', - existing_type=sa.VARCHAR(length=26), - type_=sa.Enum('censusdp1tract', 'eia176', 'eia860', 'eia860m', 'eia861', 'eia923', 'eia_bulk_elec', 'eiawater', 'epacems', 'epacamd_eia', 'ferc1', 'ferc2', 'ferc6', 'ferc60', 'ferc714', 'ferceqr', 'mshamines', 'phmsagas', 'pudl'), - existing_nullable=False) - - # ### end Alembic commands ### - - -def downgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - with op.batch_alter_table('core_pudl__codes_datasources', schema=None) as batch_op: - batch_op.alter_column('datasource', - existing_type=sa.Enum('censusdp1tract', 'eia176', 'eia860', 'eia860m', 'eia861', 'eia923', 'eia_bulk_elec', 'eiawater', 'epacems', 'epacamd_eia', 'ferc1', 'ferc2', 'ferc6', 'ferc60', 'ferc714', 'ferceqr', 'mshamines', 'phmsagas', 'pudl'), - type_=sa.VARCHAR(length=26), - existing_nullable=False) - - # ### end Alembic commands ### diff --git a/migrations/versions/997d2c7bc7de_add_utility_name_to_plant_parts_list.py b/migrations/versions/997d2c7bc7de_add_utility_name_to_plant_parts_list.py deleted file mode 100644 index b267f3285c..0000000000 --- a/migrations/versions/997d2c7bc7de_add_utility_name_to_plant_parts_list.py +++ /dev/null @@ -1,31 +0,0 @@ -"""add utility name to plant parts list - -Revision ID: 997d2c7bc7de -Revises: 2e5b623ab40b -Create Date: 2024-01-10 16:18:35.723544 - -""" -import sqlalchemy as sa -from alembic import op - -# revision identifiers, used by Alembic. -revision = '997d2c7bc7de' -down_revision = '2e5b623ab40b' -branch_labels = None -depends_on = None - - -def upgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - with op.batch_alter_table('out_eia__yearly_plant_parts', schema=None) as batch_op: - batch_op.add_column(sa.Column('utility_name_eia', sa.Text(), nullable=True, comment='The name of the utility.')) - - # ### end Alembic commands ### - - -def downgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - with op.batch_alter_table('out_eia__yearly_plant_parts', schema=None) as batch_op: - batch_op.drop_column('utility_name_eia') - - # ### end Alembic commands ### diff --git a/migrations/versions/4b08158ae952_apply_naming_convention_to_all_assets.py b/migrations/versions/c1e3439ef338_apply_new_naming_convention_to_.py similarity index 99% rename from migrations/versions/4b08158ae952_apply_naming_convention_to_all_assets.py rename to migrations/versions/c1e3439ef338_apply_new_naming_convention_to_.py index e623f2a241..19ffd1dbcd 100644 --- a/migrations/versions/4b08158ae952_apply_naming_convention_to_all_assets.py +++ b/migrations/versions/c1e3439ef338_apply_new_naming_convention_to_.py @@ -1,8 +1,8 @@ -"""Apply naming convention to all assets +"""Apply new naming convention to remaining assets -Revision ID: 4b08158ae952 +Revision ID: c1e3439ef338 Revises: -Create Date: 2023-12-13 11:50:58.385316 +Create Date: 2024-01-30 10:21:25.661903 """ import sqlalchemy as sa @@ -10,7 +10,7 @@ from sqlalchemy.dialects import sqlite # revision identifiers, used by Alembic. -revision = '4b08158ae952' +revision = 'c1e3439ef338' down_revision = None branch_labels = None depends_on = None @@ -222,12 +222,12 @@ def upgrade() -> None: op.create_table('core_eia__entity_plants', sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), sa.Column('plant_name_eia', sa.Text(), nullable=True, comment='Plant name.'), - sa.Column('city', sa.Text(), nullable=True), + sa.Column('city', sa.Text(), nullable=True, comment='Name of the city.'), sa.Column('county', sa.Text(), nullable=True, comment='County name.'), sa.Column('latitude', sa.Float(), nullable=True, comment="Latitude of the plant's location, in degrees."), sa.Column('longitude', sa.Float(), nullable=True, comment="Longitude of the plant's location, in degrees."), sa.Column('state', sa.Text(), nullable=True, comment='Two letter US state abbreviation.'), - sa.Column('street_address', sa.Text(), nullable=True), + sa.Column('street_address', sa.Text(), nullable=True, comment='Physical street address.'), sa.Column('zip_code', sa.Text(), nullable=True, comment='Five digit US Zip Code.'), sa.Column('timezone', sa.Enum('Africa/Abidjan', 'Africa/Accra', 'Africa/Addis_Ababa', 'Africa/Algiers', 'Africa/Asmara', 'Africa/Asmera', 'Africa/Bamako', 'Africa/Bangui', 'Africa/Banjul', 'Africa/Bissau', 'Africa/Blantyre', 'Africa/Brazzaville', 'Africa/Bujumbura', 'Africa/Cairo', 'Africa/Casablanca', 'Africa/Ceuta', 'Africa/Conakry', 'Africa/Dakar', 'Africa/Dar_es_Salaam', 'Africa/Djibouti', 'Africa/Douala', 'Africa/El_Aaiun', 'Africa/Freetown', 'Africa/Gaborone', 'Africa/Harare', 'Africa/Johannesburg', 'Africa/Juba', 'Africa/Kampala', 'Africa/Khartoum', 'Africa/Kigali', 'Africa/Kinshasa', 'Africa/Lagos', 'Africa/Libreville', 'Africa/Lome', 'Africa/Luanda', 'Africa/Lubumbashi', 'Africa/Lusaka', 'Africa/Malabo', 'Africa/Maputo', 'Africa/Maseru', 'Africa/Mbabane', 'Africa/Mogadishu', 'Africa/Monrovia', 'Africa/Nairobi', 'Africa/Ndjamena', 'Africa/Niamey', 'Africa/Nouakchott', 'Africa/Ouagadougou', 'Africa/Porto-Novo', 'Africa/Sao_Tome', 'Africa/Timbuktu', 'Africa/Tripoli', 'Africa/Tunis', 'Africa/Windhoek', 'America/Adak', 'America/Anchorage', 'America/Anguilla', 'America/Antigua', 'America/Araguaina', 'America/Argentina/Buenos_Aires', 'America/Argentina/Catamarca', 'America/Argentina/ComodRivadavia', 'America/Argentina/Cordoba', 'America/Argentina/Jujuy', 'America/Argentina/La_Rioja', 'America/Argentina/Mendoza', 'America/Argentina/Rio_Gallegos', 'America/Argentina/Salta', 'America/Argentina/San_Juan', 'America/Argentina/San_Luis', 'America/Argentina/Tucuman', 'America/Argentina/Ushuaia', 'America/Aruba', 'America/Asuncion', 'America/Atikokan', 'America/Atka', 'America/Bahia', 'America/Bahia_Banderas', 'America/Barbados', 'America/Belem', 'America/Belize', 'America/Blanc-Sablon', 'America/Boa_Vista', 'America/Bogota', 'America/Boise', 'America/Buenos_Aires', 'America/Cambridge_Bay', 'America/Campo_Grande', 'America/Cancun', 'America/Caracas', 'America/Catamarca', 'America/Cayenne', 'America/Cayman', 'America/Chicago', 'America/Chihuahua', 'America/Ciudad_Juarez', 'America/Coral_Harbour', 'America/Cordoba', 'America/Costa_Rica', 'America/Creston', 'America/Cuiaba', 'America/Curacao', 'America/Danmarkshavn', 'America/Dawson', 'America/Dawson_Creek', 'America/Denver', 'America/Detroit', 'America/Dominica', 'America/Edmonton', 'America/Eirunepe', 'America/El_Salvador', 'America/Ensenada', 'America/Fort_Nelson', 'America/Fort_Wayne', 'America/Fortaleza', 'America/Glace_Bay', 'America/Godthab', 'America/Goose_Bay', 'America/Grand_Turk', 'America/Grenada', 'America/Guadeloupe', 'America/Guatemala', 'America/Guayaquil', 'America/Guyana', 'America/Halifax', 'America/Havana', 'America/Hermosillo', 'America/Indiana/Indianapolis', 'America/Indiana/Knox', 'America/Indiana/Marengo', 'America/Indiana/Petersburg', 'America/Indiana/Tell_City', 'America/Indiana/Vevay', 'America/Indiana/Vincennes', 'America/Indiana/Winamac', 'America/Indianapolis', 'America/Inuvik', 'America/Iqaluit', 'America/Jamaica', 'America/Jujuy', 'America/Juneau', 'America/Kentucky/Louisville', 'America/Kentucky/Monticello', 'America/Knox_IN', 'America/Kralendijk', 'America/La_Paz', 'America/Lima', 'America/Los_Angeles', 'America/Louisville', 'America/Lower_Princes', 'America/Maceio', 'America/Managua', 'America/Manaus', 'America/Marigot', 'America/Martinique', 'America/Matamoros', 'America/Mazatlan', 'America/Mendoza', 'America/Menominee', 'America/Merida', 'America/Metlakatla', 'America/Mexico_City', 'America/Miquelon', 'America/Moncton', 'America/Monterrey', 'America/Montevideo', 'America/Montreal', 'America/Montserrat', 'America/Nassau', 'America/New_York', 'America/Nipigon', 'America/Nome', 'America/Noronha', 'America/North_Dakota/Beulah', 'America/North_Dakota/Center', 'America/North_Dakota/New_Salem', 'America/Nuuk', 'America/Ojinaga', 'America/Panama', 'America/Pangnirtung', 'America/Paramaribo', 'America/Phoenix', 'America/Port-au-Prince', 'America/Port_of_Spain', 'America/Porto_Acre', 'America/Porto_Velho', 'America/Puerto_Rico', 'America/Punta_Arenas', 'America/Rainy_River', 'America/Rankin_Inlet', 'America/Recife', 'America/Regina', 'America/Resolute', 'America/Rio_Branco', 'America/Rosario', 'America/Santa_Isabel', 'America/Santarem', 'America/Santiago', 'America/Santo_Domingo', 'America/Sao_Paulo', 'America/Scoresbysund', 'America/Shiprock', 'America/Sitka', 'America/St_Barthelemy', 'America/St_Johns', 'America/St_Kitts', 'America/St_Lucia', 'America/St_Thomas', 'America/St_Vincent', 'America/Swift_Current', 'America/Tegucigalpa', 'America/Thule', 'America/Thunder_Bay', 'America/Tijuana', 'America/Toronto', 'America/Tortola', 'America/Vancouver', 'America/Virgin', 'America/Whitehorse', 'America/Winnipeg', 'America/Yakutat', 'America/Yellowknife', 'Antarctica/Casey', 'Antarctica/Davis', 'Antarctica/DumontDUrville', 'Antarctica/Macquarie', 'Antarctica/Mawson', 'Antarctica/McMurdo', 'Antarctica/Palmer', 'Antarctica/Rothera', 'Antarctica/South_Pole', 'Antarctica/Syowa', 'Antarctica/Troll', 'Antarctica/Vostok', 'Arctic/Longyearbyen', 'Asia/Aden', 'Asia/Almaty', 'Asia/Amman', 'Asia/Anadyr', 'Asia/Aqtau', 'Asia/Aqtobe', 'Asia/Ashgabat', 'Asia/Ashkhabad', 'Asia/Atyrau', 'Asia/Baghdad', 'Asia/Bahrain', 'Asia/Baku', 'Asia/Bangkok', 'Asia/Barnaul', 'Asia/Beirut', 'Asia/Bishkek', 'Asia/Brunei', 'Asia/Calcutta', 'Asia/Chita', 'Asia/Choibalsan', 'Asia/Chongqing', 'Asia/Chungking', 'Asia/Colombo', 'Asia/Dacca', 'Asia/Damascus', 'Asia/Dhaka', 'Asia/Dili', 'Asia/Dubai', 'Asia/Dushanbe', 'Asia/Famagusta', 'Asia/Gaza', 'Asia/Harbin', 'Asia/Hebron', 'Asia/Ho_Chi_Minh', 'Asia/Hong_Kong', 'Asia/Hovd', 'Asia/Irkutsk', 'Asia/Istanbul', 'Asia/Jakarta', 'Asia/Jayapura', 'Asia/Jerusalem', 'Asia/Kabul', 'Asia/Kamchatka', 'Asia/Karachi', 'Asia/Kashgar', 'Asia/Kathmandu', 'Asia/Katmandu', 'Asia/Khandyga', 'Asia/Kolkata', 'Asia/Krasnoyarsk', 'Asia/Kuala_Lumpur', 'Asia/Kuching', 'Asia/Kuwait', 'Asia/Macao', 'Asia/Macau', 'Asia/Magadan', 'Asia/Makassar', 'Asia/Manila', 'Asia/Muscat', 'Asia/Nicosia', 'Asia/Novokuznetsk', 'Asia/Novosibirsk', 'Asia/Omsk', 'Asia/Oral', 'Asia/Phnom_Penh', 'Asia/Pontianak', 'Asia/Pyongyang', 'Asia/Qatar', 'Asia/Qostanay', 'Asia/Qyzylorda', 'Asia/Rangoon', 'Asia/Riyadh', 'Asia/Saigon', 'Asia/Sakhalin', 'Asia/Samarkand', 'Asia/Seoul', 'Asia/Shanghai', 'Asia/Singapore', 'Asia/Srednekolymsk', 'Asia/Taipei', 'Asia/Tashkent', 'Asia/Tbilisi', 'Asia/Tehran', 'Asia/Tel_Aviv', 'Asia/Thimbu', 'Asia/Thimphu', 'Asia/Tokyo', 'Asia/Tomsk', 'Asia/Ujung_Pandang', 'Asia/Ulaanbaatar', 'Asia/Ulan_Bator', 'Asia/Urumqi', 'Asia/Ust-Nera', 'Asia/Vientiane', 'Asia/Vladivostok', 'Asia/Yakutsk', 'Asia/Yangon', 'Asia/Yekaterinburg', 'Asia/Yerevan', 'Atlantic/Azores', 'Atlantic/Bermuda', 'Atlantic/Canary', 'Atlantic/Cape_Verde', 'Atlantic/Faeroe', 'Atlantic/Faroe', 'Atlantic/Jan_Mayen', 'Atlantic/Madeira', 'Atlantic/Reykjavik', 'Atlantic/South_Georgia', 'Atlantic/St_Helena', 'Atlantic/Stanley', 'Australia/ACT', 'Australia/Adelaide', 'Australia/Brisbane', 'Australia/Broken_Hill', 'Australia/Canberra', 'Australia/Currie', 'Australia/Darwin', 'Australia/Eucla', 'Australia/Hobart', 'Australia/LHI', 'Australia/Lindeman', 'Australia/Lord_Howe', 'Australia/Melbourne', 'Australia/NSW', 'Australia/North', 'Australia/Perth', 'Australia/Queensland', 'Australia/South', 'Australia/Sydney', 'Australia/Tasmania', 'Australia/Victoria', 'Australia/West', 'Australia/Yancowinna', 'Brazil/Acre', 'Brazil/DeNoronha', 'Brazil/East', 'Brazil/West', 'CET', 'CST6CDT', 'Canada/Atlantic', 'Canada/Central', 'Canada/Eastern', 'Canada/Mountain', 'Canada/Newfoundland', 'Canada/Pacific', 'Canada/Saskatchewan', 'Canada/Yukon', 'Chile/Continental', 'Chile/EasterIsland', 'Cuba', 'EET', 'EST', 'EST5EDT', 'Egypt', 'Eire', 'Etc/GMT', 'Etc/GMT+0', 'Etc/GMT+1', 'Etc/GMT+10', 'Etc/GMT+11', 'Etc/GMT+12', 'Etc/GMT+2', 'Etc/GMT+3', 'Etc/GMT+4', 'Etc/GMT+5', 'Etc/GMT+6', 'Etc/GMT+7', 'Etc/GMT+8', 'Etc/GMT+9', 'Etc/GMT-0', 'Etc/GMT-1', 'Etc/GMT-10', 'Etc/GMT-11', 'Etc/GMT-12', 'Etc/GMT-13', 'Etc/GMT-14', 'Etc/GMT-2', 'Etc/GMT-3', 'Etc/GMT-4', 'Etc/GMT-5', 'Etc/GMT-6', 'Etc/GMT-7', 'Etc/GMT-8', 'Etc/GMT-9', 'Etc/GMT0', 'Etc/Greenwich', 'Etc/UCT', 'Etc/UTC', 'Etc/Universal', 'Etc/Zulu', 'Europe/Amsterdam', 'Europe/Andorra', 'Europe/Astrakhan', 'Europe/Athens', 'Europe/Belfast', 'Europe/Belgrade', 'Europe/Berlin', 'Europe/Bratislava', 'Europe/Brussels', 'Europe/Bucharest', 'Europe/Budapest', 'Europe/Busingen', 'Europe/Chisinau', 'Europe/Copenhagen', 'Europe/Dublin', 'Europe/Gibraltar', 'Europe/Guernsey', 'Europe/Helsinki', 'Europe/Isle_of_Man', 'Europe/Istanbul', 'Europe/Jersey', 'Europe/Kaliningrad', 'Europe/Kiev', 'Europe/Kirov', 'Europe/Kyiv', 'Europe/Lisbon', 'Europe/Ljubljana', 'Europe/London', 'Europe/Luxembourg', 'Europe/Madrid', 'Europe/Malta', 'Europe/Mariehamn', 'Europe/Minsk', 'Europe/Monaco', 'Europe/Moscow', 'Europe/Nicosia', 'Europe/Oslo', 'Europe/Paris', 'Europe/Podgorica', 'Europe/Prague', 'Europe/Riga', 'Europe/Rome', 'Europe/Samara', 'Europe/San_Marino', 'Europe/Sarajevo', 'Europe/Saratov', 'Europe/Simferopol', 'Europe/Skopje', 'Europe/Sofia', 'Europe/Stockholm', 'Europe/Tallinn', 'Europe/Tirane', 'Europe/Tiraspol', 'Europe/Ulyanovsk', 'Europe/Uzhgorod', 'Europe/Vaduz', 'Europe/Vatican', 'Europe/Vienna', 'Europe/Vilnius', 'Europe/Volgograd', 'Europe/Warsaw', 'Europe/Zagreb', 'Europe/Zaporozhye', 'Europe/Zurich', 'GB', 'GB-Eire', 'GMT', 'GMT+0', 'GMT-0', 'GMT0', 'Greenwich', 'HST', 'Hongkong', 'Iceland', 'Indian/Antananarivo', 'Indian/Chagos', 'Indian/Christmas', 'Indian/Cocos', 'Indian/Comoro', 'Indian/Kerguelen', 'Indian/Mahe', 'Indian/Maldives', 'Indian/Mauritius', 'Indian/Mayotte', 'Indian/Reunion', 'Iran', 'Israel', 'Jamaica', 'Japan', 'Kwajalein', 'Libya', 'MET', 'MST', 'MST7MDT', 'Mexico/BajaNorte', 'Mexico/BajaSur', 'Mexico/General', 'NZ', 'NZ-CHAT', 'Navajo', 'PRC', 'PST8PDT', 'Pacific/Apia', 'Pacific/Auckland', 'Pacific/Bougainville', 'Pacific/Chatham', 'Pacific/Chuuk', 'Pacific/Easter', 'Pacific/Efate', 'Pacific/Enderbury', 'Pacific/Fakaofo', 'Pacific/Fiji', 'Pacific/Funafuti', 'Pacific/Galapagos', 'Pacific/Gambier', 'Pacific/Guadalcanal', 'Pacific/Guam', 'Pacific/Honolulu', 'Pacific/Johnston', 'Pacific/Kanton', 'Pacific/Kiritimati', 'Pacific/Kosrae', 'Pacific/Kwajalein', 'Pacific/Majuro', 'Pacific/Marquesas', 'Pacific/Midway', 'Pacific/Nauru', 'Pacific/Niue', 'Pacific/Norfolk', 'Pacific/Noumea', 'Pacific/Pago_Pago', 'Pacific/Palau', 'Pacific/Pitcairn', 'Pacific/Pohnpei', 'Pacific/Ponape', 'Pacific/Port_Moresby', 'Pacific/Rarotonga', 'Pacific/Saipan', 'Pacific/Samoa', 'Pacific/Tahiti', 'Pacific/Tarawa', 'Pacific/Tongatapu', 'Pacific/Truk', 'Pacific/Wake', 'Pacific/Wallis', 'Pacific/Yap', 'Poland', 'Portugal', 'ROC', 'ROK', 'Singapore', 'Turkey', 'UCT', 'US/Alaska', 'US/Aleutian', 'US/Arizona', 'US/Central', 'US/East-Indiana', 'US/Eastern', 'US/Hawaii', 'US/Indiana-Starke', 'US/Michigan', 'US/Mountain', 'US/Pacific', 'US/Samoa', 'UTC', 'Universal', 'W-SU', 'WET', 'Zulu'), nullable=True, comment='IANA timezone name'), sa.PrimaryKeyConstraint('plant_id_eia', name=op.f('pk_core_eia__entity_plants')) @@ -278,7 +278,7 @@ def upgrade() -> None: sa.PrimaryKeyConstraint('code', name=op.f('pk_core_pudl__codes_data_maturities')) ) op.create_table('core_pudl__codes_datasources', - sa.Column('datasource', sa.Enum('censusdp1tract', 'eia176', 'eia860', 'eia860m', 'eia861', 'eia923', 'eia_bulk_elec', 'eiawater', 'epacems', 'core_epa__assn_eia_epacamd', 'ferc1', 'ferc2', 'ferc6', 'ferc60', 'ferc714', 'ferceqr', 'mshamines', 'phmsagas', 'pudl'), nullable=False, comment='Code identifying a dataset available within PUDL.'), + sa.Column('datasource', sa.Enum('censusdp1tract', 'eia176', 'eia191', 'eia757a', 'eia860', 'eia860m', 'eia861', 'eia923', 'eia_bulk_elec', 'eiawater', 'epacems', 'epacamd_eia', 'ferc1', 'ferc2', 'ferc6', 'ferc60', 'ferc714', 'ferceqr', 'mshamines', 'phmsagas', 'pudl'), nullable=False, comment='Code identifying a dataset available within PUDL.'), sa.Column('partitions', sa.Text(), nullable=True, comment='The data parititions used to generate this instance of the database.'), sa.Column('doi', sa.Text(), nullable=True, comment='Unique digitial object identifier of Zenodo archive.'), sa.Column('pudl_version', sa.Text(), nullable=True, comment='The version of PUDL used to generate this database.'), @@ -287,13 +287,13 @@ def upgrade() -> None: op.create_table('core_pudl__codes_subdivisions', sa.Column('country_code', sa.Enum('USA', 'CAN'), nullable=False, comment='Three letter ISO-3166 country code (e.g. USA or CAN).'), sa.Column('country_name', sa.Text(), nullable=True, comment='Full country name (e.g. United States of America).'), - sa.Column('subdivision_code', sa.Enum('NJ', 'KY', 'PE', 'NT', 'DE', 'MP', 'ON', 'MB', 'OK', 'SD', 'GU', 'CA', 'CT', 'NE', 'AS', 'UT', 'KS', 'MO', 'NC', 'QC', 'NB', 'AL', 'AR', 'BC', 'NU', 'OR', 'VT', 'NM', 'PR', 'WI', 'WA', 'PA', 'RI', 'TX', 'GA', 'MA', 'AB', 'AZ', 'YT', 'ME', 'VI', 'SC', 'IN', 'WV', 'FL', 'NY', 'MS', 'ID', 'DC', 'IL', 'MI', 'CO', 'LA', 'NL', 'MD', 'OH', 'NS', 'AK', 'SK', 'VA', 'MT', 'ND', 'NH', 'MN', 'TN', 'IA', 'HI', 'WY', 'NV'), nullable=False, comment='Two-letter ISO-3166 political subdivision code (e.g. US state or Canadian provice abbreviations like CA or AB).'), + sa.Column('subdivision_code', sa.Enum('BC', 'IN', 'IA', 'SK', 'NV', 'HI', 'VA', 'PE', 'ON', 'GA', 'MD', 'PR', 'WY', 'WI', 'AS', 'MT', 'KS', 'NS', 'CO', 'ND', 'OK', 'MI', 'IL', 'NU', 'AK', 'LA', 'AL', 'ID', 'MA', 'MP', 'NB', 'PA', 'ME', 'RI', 'WV', 'KY', 'OR', 'NY', 'NJ', 'TN', 'DC', 'QC', 'TX', 'DE', 'OH', 'AZ', 'AR', 'CT', 'NT', 'CA', 'WA', 'MS', 'GU', 'NE', 'NM', 'SD', 'FL', 'SC', 'MN', 'MO', 'MB', 'YT', 'NC', 'UT', 'NL', 'AB', 'NH', 'VI', 'VT'), nullable=False, comment='Two-letter ISO-3166 political subdivision code (e.g. US state or Canadian provice abbreviations like CA or AB).'), sa.Column('subdivision_name', sa.Text(), nullable=True, comment='Full name of political subdivision (e.g. US state or Canadian province names like California or Alberta.'), sa.Column('subdivision_type', sa.Text(), nullable=True, comment='ISO-3166 political subdivision type. E.g. state, province, outlying_area.'), sa.Column('timezone_approx', sa.Enum('Africa/Abidjan', 'Africa/Accra', 'Africa/Addis_Ababa', 'Africa/Algiers', 'Africa/Asmara', 'Africa/Asmera', 'Africa/Bamako', 'Africa/Bangui', 'Africa/Banjul', 'Africa/Bissau', 'Africa/Blantyre', 'Africa/Brazzaville', 'Africa/Bujumbura', 'Africa/Cairo', 'Africa/Casablanca', 'Africa/Ceuta', 'Africa/Conakry', 'Africa/Dakar', 'Africa/Dar_es_Salaam', 'Africa/Djibouti', 'Africa/Douala', 'Africa/El_Aaiun', 'Africa/Freetown', 'Africa/Gaborone', 'Africa/Harare', 'Africa/Johannesburg', 'Africa/Juba', 'Africa/Kampala', 'Africa/Khartoum', 'Africa/Kigali', 'Africa/Kinshasa', 'Africa/Lagos', 'Africa/Libreville', 'Africa/Lome', 'Africa/Luanda', 'Africa/Lubumbashi', 'Africa/Lusaka', 'Africa/Malabo', 'Africa/Maputo', 'Africa/Maseru', 'Africa/Mbabane', 'Africa/Mogadishu', 'Africa/Monrovia', 'Africa/Nairobi', 'Africa/Ndjamena', 'Africa/Niamey', 'Africa/Nouakchott', 'Africa/Ouagadougou', 'Africa/Porto-Novo', 'Africa/Sao_Tome', 'Africa/Timbuktu', 'Africa/Tripoli', 'Africa/Tunis', 'Africa/Windhoek', 'America/Adak', 'America/Anchorage', 'America/Anguilla', 'America/Antigua', 'America/Araguaina', 'America/Argentina/Buenos_Aires', 'America/Argentina/Catamarca', 'America/Argentina/ComodRivadavia', 'America/Argentina/Cordoba', 'America/Argentina/Jujuy', 'America/Argentina/La_Rioja', 'America/Argentina/Mendoza', 'America/Argentina/Rio_Gallegos', 'America/Argentina/Salta', 'America/Argentina/San_Juan', 'America/Argentina/San_Luis', 'America/Argentina/Tucuman', 'America/Argentina/Ushuaia', 'America/Aruba', 'America/Asuncion', 'America/Atikokan', 'America/Atka', 'America/Bahia', 'America/Bahia_Banderas', 'America/Barbados', 'America/Belem', 'America/Belize', 'America/Blanc-Sablon', 'America/Boa_Vista', 'America/Bogota', 'America/Boise', 'America/Buenos_Aires', 'America/Cambridge_Bay', 'America/Campo_Grande', 'America/Cancun', 'America/Caracas', 'America/Catamarca', 'America/Cayenne', 'America/Cayman', 'America/Chicago', 'America/Chihuahua', 'America/Ciudad_Juarez', 'America/Coral_Harbour', 'America/Cordoba', 'America/Costa_Rica', 'America/Creston', 'America/Cuiaba', 'America/Curacao', 'America/Danmarkshavn', 'America/Dawson', 'America/Dawson_Creek', 'America/Denver', 'America/Detroit', 'America/Dominica', 'America/Edmonton', 'America/Eirunepe', 'America/El_Salvador', 'America/Ensenada', 'America/Fort_Nelson', 'America/Fort_Wayne', 'America/Fortaleza', 'America/Glace_Bay', 'America/Godthab', 'America/Goose_Bay', 'America/Grand_Turk', 'America/Grenada', 'America/Guadeloupe', 'America/Guatemala', 'America/Guayaquil', 'America/Guyana', 'America/Halifax', 'America/Havana', 'America/Hermosillo', 'America/Indiana/Indianapolis', 'America/Indiana/Knox', 'America/Indiana/Marengo', 'America/Indiana/Petersburg', 'America/Indiana/Tell_City', 'America/Indiana/Vevay', 'America/Indiana/Vincennes', 'America/Indiana/Winamac', 'America/Indianapolis', 'America/Inuvik', 'America/Iqaluit', 'America/Jamaica', 'America/Jujuy', 'America/Juneau', 'America/Kentucky/Louisville', 'America/Kentucky/Monticello', 'America/Knox_IN', 'America/Kralendijk', 'America/La_Paz', 'America/Lima', 'America/Los_Angeles', 'America/Louisville', 'America/Lower_Princes', 'America/Maceio', 'America/Managua', 'America/Manaus', 'America/Marigot', 'America/Martinique', 'America/Matamoros', 'America/Mazatlan', 'America/Mendoza', 'America/Menominee', 'America/Merida', 'America/Metlakatla', 'America/Mexico_City', 'America/Miquelon', 'America/Moncton', 'America/Monterrey', 'America/Montevideo', 'America/Montreal', 'America/Montserrat', 'America/Nassau', 'America/New_York', 'America/Nipigon', 'America/Nome', 'America/Noronha', 'America/North_Dakota/Beulah', 'America/North_Dakota/Center', 'America/North_Dakota/New_Salem', 'America/Nuuk', 'America/Ojinaga', 'America/Panama', 'America/Pangnirtung', 'America/Paramaribo', 'America/Phoenix', 'America/Port-au-Prince', 'America/Port_of_Spain', 'America/Porto_Acre', 'America/Porto_Velho', 'America/Puerto_Rico', 'America/Punta_Arenas', 'America/Rainy_River', 'America/Rankin_Inlet', 'America/Recife', 'America/Regina', 'America/Resolute', 'America/Rio_Branco', 'America/Rosario', 'America/Santa_Isabel', 'America/Santarem', 'America/Santiago', 'America/Santo_Domingo', 'America/Sao_Paulo', 'America/Scoresbysund', 'America/Shiprock', 'America/Sitka', 'America/St_Barthelemy', 'America/St_Johns', 'America/St_Kitts', 'America/St_Lucia', 'America/St_Thomas', 'America/St_Vincent', 'America/Swift_Current', 'America/Tegucigalpa', 'America/Thule', 'America/Thunder_Bay', 'America/Tijuana', 'America/Toronto', 'America/Tortola', 'America/Vancouver', 'America/Virgin', 'America/Whitehorse', 'America/Winnipeg', 'America/Yakutat', 'America/Yellowknife', 'Antarctica/Casey', 'Antarctica/Davis', 'Antarctica/DumontDUrville', 'Antarctica/Macquarie', 'Antarctica/Mawson', 'Antarctica/McMurdo', 'Antarctica/Palmer', 'Antarctica/Rothera', 'Antarctica/South_Pole', 'Antarctica/Syowa', 'Antarctica/Troll', 'Antarctica/Vostok', 'Arctic/Longyearbyen', 'Asia/Aden', 'Asia/Almaty', 'Asia/Amman', 'Asia/Anadyr', 'Asia/Aqtau', 'Asia/Aqtobe', 'Asia/Ashgabat', 'Asia/Ashkhabad', 'Asia/Atyrau', 'Asia/Baghdad', 'Asia/Bahrain', 'Asia/Baku', 'Asia/Bangkok', 'Asia/Barnaul', 'Asia/Beirut', 'Asia/Bishkek', 'Asia/Brunei', 'Asia/Calcutta', 'Asia/Chita', 'Asia/Choibalsan', 'Asia/Chongqing', 'Asia/Chungking', 'Asia/Colombo', 'Asia/Dacca', 'Asia/Damascus', 'Asia/Dhaka', 'Asia/Dili', 'Asia/Dubai', 'Asia/Dushanbe', 'Asia/Famagusta', 'Asia/Gaza', 'Asia/Harbin', 'Asia/Hebron', 'Asia/Ho_Chi_Minh', 'Asia/Hong_Kong', 'Asia/Hovd', 'Asia/Irkutsk', 'Asia/Istanbul', 'Asia/Jakarta', 'Asia/Jayapura', 'Asia/Jerusalem', 'Asia/Kabul', 'Asia/Kamchatka', 'Asia/Karachi', 'Asia/Kashgar', 'Asia/Kathmandu', 'Asia/Katmandu', 'Asia/Khandyga', 'Asia/Kolkata', 'Asia/Krasnoyarsk', 'Asia/Kuala_Lumpur', 'Asia/Kuching', 'Asia/Kuwait', 'Asia/Macao', 'Asia/Macau', 'Asia/Magadan', 'Asia/Makassar', 'Asia/Manila', 'Asia/Muscat', 'Asia/Nicosia', 'Asia/Novokuznetsk', 'Asia/Novosibirsk', 'Asia/Omsk', 'Asia/Oral', 'Asia/Phnom_Penh', 'Asia/Pontianak', 'Asia/Pyongyang', 'Asia/Qatar', 'Asia/Qostanay', 'Asia/Qyzylorda', 'Asia/Rangoon', 'Asia/Riyadh', 'Asia/Saigon', 'Asia/Sakhalin', 'Asia/Samarkand', 'Asia/Seoul', 'Asia/Shanghai', 'Asia/Singapore', 'Asia/Srednekolymsk', 'Asia/Taipei', 'Asia/Tashkent', 'Asia/Tbilisi', 'Asia/Tehran', 'Asia/Tel_Aviv', 'Asia/Thimbu', 'Asia/Thimphu', 'Asia/Tokyo', 'Asia/Tomsk', 'Asia/Ujung_Pandang', 'Asia/Ulaanbaatar', 'Asia/Ulan_Bator', 'Asia/Urumqi', 'Asia/Ust-Nera', 'Asia/Vientiane', 'Asia/Vladivostok', 'Asia/Yakutsk', 'Asia/Yangon', 'Asia/Yekaterinburg', 'Asia/Yerevan', 'Atlantic/Azores', 'Atlantic/Bermuda', 'Atlantic/Canary', 'Atlantic/Cape_Verde', 'Atlantic/Faeroe', 'Atlantic/Faroe', 'Atlantic/Jan_Mayen', 'Atlantic/Madeira', 'Atlantic/Reykjavik', 'Atlantic/South_Georgia', 'Atlantic/St_Helena', 'Atlantic/Stanley', 'Australia/ACT', 'Australia/Adelaide', 'Australia/Brisbane', 'Australia/Broken_Hill', 'Australia/Canberra', 'Australia/Currie', 'Australia/Darwin', 'Australia/Eucla', 'Australia/Hobart', 'Australia/LHI', 'Australia/Lindeman', 'Australia/Lord_Howe', 'Australia/Melbourne', 'Australia/NSW', 'Australia/North', 'Australia/Perth', 'Australia/Queensland', 'Australia/South', 'Australia/Sydney', 'Australia/Tasmania', 'Australia/Victoria', 'Australia/West', 'Australia/Yancowinna', 'Brazil/Acre', 'Brazil/DeNoronha', 'Brazil/East', 'Brazil/West', 'CET', 'CST6CDT', 'Canada/Atlantic', 'Canada/Central', 'Canada/Eastern', 'Canada/Mountain', 'Canada/Newfoundland', 'Canada/Pacific', 'Canada/Saskatchewan', 'Canada/Yukon', 'Chile/Continental', 'Chile/EasterIsland', 'Cuba', 'EET', 'EST', 'EST5EDT', 'Egypt', 'Eire', 'Etc/GMT', 'Etc/GMT+0', 'Etc/GMT+1', 'Etc/GMT+10', 'Etc/GMT+11', 'Etc/GMT+12', 'Etc/GMT+2', 'Etc/GMT+3', 'Etc/GMT+4', 'Etc/GMT+5', 'Etc/GMT+6', 'Etc/GMT+7', 'Etc/GMT+8', 'Etc/GMT+9', 'Etc/GMT-0', 'Etc/GMT-1', 'Etc/GMT-10', 'Etc/GMT-11', 'Etc/GMT-12', 'Etc/GMT-13', 'Etc/GMT-14', 'Etc/GMT-2', 'Etc/GMT-3', 'Etc/GMT-4', 'Etc/GMT-5', 'Etc/GMT-6', 'Etc/GMT-7', 'Etc/GMT-8', 'Etc/GMT-9', 'Etc/GMT0', 'Etc/Greenwich', 'Etc/UCT', 'Etc/UTC', 'Etc/Universal', 'Etc/Zulu', 'Europe/Amsterdam', 'Europe/Andorra', 'Europe/Astrakhan', 'Europe/Athens', 'Europe/Belfast', 'Europe/Belgrade', 'Europe/Berlin', 'Europe/Bratislava', 'Europe/Brussels', 'Europe/Bucharest', 'Europe/Budapest', 'Europe/Busingen', 'Europe/Chisinau', 'Europe/Copenhagen', 'Europe/Dublin', 'Europe/Gibraltar', 'Europe/Guernsey', 'Europe/Helsinki', 'Europe/Isle_of_Man', 'Europe/Istanbul', 'Europe/Jersey', 'Europe/Kaliningrad', 'Europe/Kiev', 'Europe/Kirov', 'Europe/Kyiv', 'Europe/Lisbon', 'Europe/Ljubljana', 'Europe/London', 'Europe/Luxembourg', 'Europe/Madrid', 'Europe/Malta', 'Europe/Mariehamn', 'Europe/Minsk', 'Europe/Monaco', 'Europe/Moscow', 'Europe/Nicosia', 'Europe/Oslo', 'Europe/Paris', 'Europe/Podgorica', 'Europe/Prague', 'Europe/Riga', 'Europe/Rome', 'Europe/Samara', 'Europe/San_Marino', 'Europe/Sarajevo', 'Europe/Saratov', 'Europe/Simferopol', 'Europe/Skopje', 'Europe/Sofia', 'Europe/Stockholm', 'Europe/Tallinn', 'Europe/Tirane', 'Europe/Tiraspol', 'Europe/Ulyanovsk', 'Europe/Uzhgorod', 'Europe/Vaduz', 'Europe/Vatican', 'Europe/Vienna', 'Europe/Vilnius', 'Europe/Volgograd', 'Europe/Warsaw', 'Europe/Zagreb', 'Europe/Zaporozhye', 'Europe/Zurich', 'GB', 'GB-Eire', 'GMT', 'GMT+0', 'GMT-0', 'GMT0', 'Greenwich', 'HST', 'Hongkong', 'Iceland', 'Indian/Antananarivo', 'Indian/Chagos', 'Indian/Christmas', 'Indian/Cocos', 'Indian/Comoro', 'Indian/Kerguelen', 'Indian/Mahe', 'Indian/Maldives', 'Indian/Mauritius', 'Indian/Mayotte', 'Indian/Reunion', 'Iran', 'Israel', 'Jamaica', 'Japan', 'Kwajalein', 'Libya', 'MET', 'MST', 'MST7MDT', 'Mexico/BajaNorte', 'Mexico/BajaSur', 'Mexico/General', 'NZ', 'NZ-CHAT', 'Navajo', 'PRC', 'PST8PDT', 'Pacific/Apia', 'Pacific/Auckland', 'Pacific/Bougainville', 'Pacific/Chatham', 'Pacific/Chuuk', 'Pacific/Easter', 'Pacific/Efate', 'Pacific/Enderbury', 'Pacific/Fakaofo', 'Pacific/Fiji', 'Pacific/Funafuti', 'Pacific/Galapagos', 'Pacific/Gambier', 'Pacific/Guadalcanal', 'Pacific/Guam', 'Pacific/Honolulu', 'Pacific/Johnston', 'Pacific/Kanton', 'Pacific/Kiritimati', 'Pacific/Kosrae', 'Pacific/Kwajalein', 'Pacific/Majuro', 'Pacific/Marquesas', 'Pacific/Midway', 'Pacific/Nauru', 'Pacific/Niue', 'Pacific/Norfolk', 'Pacific/Noumea', 'Pacific/Pago_Pago', 'Pacific/Palau', 'Pacific/Pitcairn', 'Pacific/Pohnpei', 'Pacific/Ponape', 'Pacific/Port_Moresby', 'Pacific/Rarotonga', 'Pacific/Saipan', 'Pacific/Samoa', 'Pacific/Tahiti', 'Pacific/Tarawa', 'Pacific/Tongatapu', 'Pacific/Truk', 'Pacific/Wake', 'Pacific/Wallis', 'Pacific/Yap', 'Poland', 'Portugal', 'ROC', 'ROK', 'Singapore', 'Turkey', 'UCT', 'US/Alaska', 'US/Aleutian', 'US/Arizona', 'US/Central', 'US/East-Indiana', 'US/Eastern', 'US/Hawaii', 'US/Indiana-Starke', 'US/Michigan', 'US/Mountain', 'US/Pacific', 'US/Samoa', 'UTC', 'Universal', 'W-SU', 'WET', 'Zulu'), nullable=True, comment='IANA timezone name of the timezone which encompasses the largest portion of the population in the associated geographic area.'), sa.Column('state_id_fips', sa.Text(), nullable=True, comment='Two digit state FIPS code.'), sa.Column('division_name_us_census', sa.Text(), nullable=True, comment='Longer human readable name describing the US Census division.'), - sa.Column('division_code_us_census', sa.Enum('SAT', 'PCC', 'WNC', 'WSC', 'MTN', 'NEW', 'ENC', 'MAT', 'ESC', 'PCN'), nullable=True, comment='Three-letter US Census division code as it appears in the bulk electricity data published by the EIA. Note that EIA splits the Pacific division into distinct contiguous (CA, OR, WA) and non-contiguous (AK, HI) states. For reference see this US Census region and division map: https://www2.census.gov/geo/pdfs/maps-data/maps/reference/us_regdiv.pdf'), + sa.Column('division_code_us_census', sa.Enum('WSC', 'PCC', 'MAT', 'SAT', 'MTN', 'PCN', 'ESC', 'ENC', 'WNC', 'NEW'), nullable=True, comment='Three-letter US Census division code as it appears in the bulk electricity data published by the EIA. Note that EIA splits the Pacific division into distinct contiguous (CA, OR, WA) and non-contiguous (AK, HI) states. For reference see this US Census region and division map: https://www2.census.gov/geo/pdfs/maps-data/maps/reference/us_regdiv.pdf'), sa.Column('region_name_us_census', sa.Text(), nullable=True, comment='Human-readable name of a US Census region.'), sa.Column('is_epacems_state', sa.Boolean(), nullable=True, comment="Indicates whether the associated state reports data within the EPA's Continuous Emissions Monitoring System."), sa.PrimaryKeyConstraint('country_code', 'subdivision_code', name=op.f('pk_core_pudl__codes_subdivisions')) @@ -332,12 +332,12 @@ def upgrade() -> None: sa.Column('state_id_fips', sa.Text(), nullable=True, comment='Two digit state FIPS code.'), sa.PrimaryKeyConstraint('utility_id_eia', 'report_date', 'county_id_fips', name=op.f('pk_out_eia861__compiled_geometry_utilities')) ) - op.create_table('out_ferc714__hourly_predicted_state_demand', + op.create_table('out_ferc714__hourly_estimated_state_demand', sa.Column('state_id_fips', sa.Text(), nullable=False, comment='Two digit state FIPS code.'), sa.Column('utc_datetime', sqlite.DATETIME(), nullable=False), sa.Column('demand_mwh', sa.Float(), nullable=True), sa.Column('scaled_demand_mwh', sa.Float(), nullable=True, comment='Estimated electricity demand scaled by the total sales within a state.'), - sa.PrimaryKeyConstraint('state_id_fips', 'utc_datetime', name=op.f('pk_out_ferc714__hourly_predicted_state_demand')) + sa.PrimaryKeyConstraint('state_id_fips', 'utc_datetime', name=op.f('pk_out_ferc714__hourly_estimated_state_demand')) ) op.create_table('_out_eia__monthly_heat_rate_by_unit', sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), @@ -373,8 +373,8 @@ def upgrade() -> None: op.create_table('core_eia860__scd_utilities', sa.Column('utility_id_eia', sa.Integer(), nullable=False, comment='The EIA Utility Identification number.'), sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), - sa.Column('street_address', sa.Text(), nullable=True), - sa.Column('city', sa.Text(), nullable=True), + sa.Column('street_address', sa.Text(), nullable=True, comment='Physical street address.'), + sa.Column('city', sa.Text(), nullable=True, comment='Name of the city.'), sa.Column('state', sa.Text(), nullable=True, comment='Two letter US state abbreviation.'), sa.Column('zip_code', sa.Text(), nullable=True, comment='Five digit US Zip Code.'), sa.Column('plants_reported_owner', sa.Boolean(), nullable=True, comment='Is the reporting entity an owner of power plants reported on Schedule 2 of the form?'), @@ -891,15 +891,6 @@ def upgrade() -> None: sa.ForeignKeyConstraint(['plant_id_eia'], ['core_eia__entity_plants.plant_id_eia'], name=op.f('fk_core_eia__entity_generators_plant_id_eia_core_eia__entity_plants')), sa.PrimaryKeyConstraint('plant_id_eia', 'generator_id', name=op.f('pk_core_eia__entity_generators')) ) - op.create_table('core_ferc714__hourly_demand_pa', - sa.Column('respondent_id_ferc714', sa.Integer(), nullable=False, comment='FERC Form 714 respondent ID. Note that this ID does not correspond to FERC respondent IDs from other forms.'), - sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), - sa.Column('utc_datetime', sqlite.DATETIME(), nullable=False), - sa.Column('timezone', sa.Enum('America/New_York', 'America/Chicago', 'America/Denver', 'America/Los_Angeles', 'America/Anchorage', 'Pacific/Honolulu'), nullable=True, comment='IANA timezone name'), - sa.Column('demand_mwh', sa.Float(), nullable=True), - sa.ForeignKeyConstraint(['respondent_id_ferc714'], ['core_ferc714__respondent_id.respondent_id_ferc714'], name=op.f('fk_core_ferc714__hourly_demand_pa_respondent_id_ferc714_core_ferc714__respondent_id')), - sa.PrimaryKeyConstraint('respondent_id_ferc714', 'utc_datetime', name=op.f('pk_core_ferc714__hourly_demand_pa')) - ) op.create_table('core_pudl__assn_eia_pudl_plants', sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), sa.Column('plant_name_eia', sa.Text(), nullable=True, comment='Plant name.'), @@ -1090,6 +1081,15 @@ def upgrade() -> None: sa.ForeignKeyConstraint(['utility_id_pudl'], ['core_pudl__entity_utilities_pudl.utility_id_pudl'], name=op.f('fk_out_eia923__monthly_generation_fuel_combined_utility_id_pudl_core_pudl__entity_utilities_pudl')), sa.PrimaryKeyConstraint('plant_id_eia', 'report_date', 'prime_mover_code', 'energy_source_code', name=op.f('pk_out_eia923__monthly_generation_fuel_combined')) ) + op.create_table('out_ferc714__hourly_planning_area_demand', + sa.Column('respondent_id_ferc714', sa.Integer(), nullable=False, comment='FERC Form 714 respondent ID. Note that this ID does not correspond to FERC respondent IDs from other forms.'), + sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), + sa.Column('utc_datetime', sqlite.DATETIME(), nullable=False), + sa.Column('timezone', sa.Enum('America/New_York', 'America/Chicago', 'America/Denver', 'America/Los_Angeles', 'America/Anchorage', 'Pacific/Honolulu'), nullable=True, comment='IANA timezone name'), + sa.Column('demand_mwh', sa.Float(), nullable=True), + sa.ForeignKeyConstraint(['respondent_id_ferc714'], ['core_ferc714__respondent_id.respondent_id_ferc714'], name=op.f('fk_out_ferc714__hourly_planning_area_demand_respondent_id_ferc714_core_ferc714__respondent_id')), + sa.PrimaryKeyConstraint('respondent_id_ferc714', 'utc_datetime', name=op.f('pk_out_ferc714__hourly_planning_area_demand')) + ) op.create_table('out_ferc714__respondents_with_fips', sa.Column('eia_code', sa.Integer(), nullable=True, comment='EIA utility or balancing area authority ID associated with this FERC Form 714 respondent. Note that many utilities are also balancing authorities and in many cases EIA uses the same integer ID to identify a utility in its role as a balancing authority AND as a utility, but there is no requirement that these IDs be the same, and in a number of cases they are different.'), sa.Column('respondent_type', sa.Enum('utility', 'balancing_authority'), nullable=True), @@ -1739,7 +1739,7 @@ def upgrade() -> None: sa.Column('bga_source', sa.Text(), nullable=True, comment='The source from where the unit_id_pudl is compiled. The unit_id_pudl comes directly from EIA 860, or string association (which looks at all the boilers and generators that are not associated with a unit and tries to find a matching string in the respective collection of boilers or generator), or from a unit connection (where the unit_id_eia is employed to find additional boiler generator connections).'), sa.Column('bypass_heat_recovery', sa.Boolean(), nullable=True, comment='Can this generator operate while bypassing the heat recovery steam generator?'), sa.Column('carbon_capture', sa.Boolean(), nullable=True, comment='Indicates whether the generator uses carbon capture technology.'), - sa.Column('city', sa.Text(), nullable=True), + sa.Column('city', sa.Text(), nullable=True, comment='Name of the city.'), sa.Column('cofire_fuels', sa.Boolean(), nullable=True, comment='Can the generator co-fire fuels?.'), sa.Column('county', sa.Text(), nullable=True, comment='County name.'), sa.Column('current_planned_generator_operating_date', sa.Date(), nullable=True, comment='The most recently updated effective date on which the generator is scheduled to start operation'), @@ -1799,7 +1799,7 @@ def upgrade() -> None: sa.Column('startup_source_code_4', sa.Text(), nullable=True, comment='The code representing the first, second, third or fourth start-up and flame stabilization energy source used by the combustion unit(s) associated with this generator.'), sa.Column('state', sa.Text(), nullable=True, comment='Two letter US state abbreviation.'), sa.Column('stoker_tech', sa.Boolean(), nullable=True, comment='Indicates whether the generator uses stoker technology'), - sa.Column('street_address', sa.Text(), nullable=True), + sa.Column('street_address', sa.Text(), nullable=True, comment='Physical street address.'), sa.Column('subcritical_tech', sa.Boolean(), nullable=True, comment='Indicates whether the generator uses subcritical technology'), sa.Column('summer_capacity_estimate', sa.Boolean(), nullable=True, comment='Whether the summer capacity value was an estimate'), sa.Column('summer_capacity_mw', sa.Float(), nullable=True, comment='The net summer capacity.'), @@ -1851,8 +1851,8 @@ def upgrade() -> None: sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), sa.Column('utility_name_eia', sa.Text(), nullable=True, comment='The name of the utility.'), sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), - sa.Column('street_address', sa.Text(), nullable=True), - sa.Column('city', sa.Text(), nullable=True), + sa.Column('street_address', sa.Text(), nullable=True, comment='Physical street address.'), + sa.Column('city', sa.Text(), nullable=True, comment='Name of the city.'), sa.Column('state', sa.Text(), nullable=True, comment='Two letter US state abbreviation.'), sa.Column('zip_code', sa.Text(), nullable=True, comment='Five digit US Zip Code.'), sa.Column('plants_reported_owner', sa.Boolean(), nullable=True, comment='Is the reporting entity an owner of power plants reported on Schedule 2 of the form?'), @@ -2658,7 +2658,7 @@ def upgrade() -> None: sa.Column('report_year', sa.Integer(), nullable=True, comment='Four-digit year in which the data was reported.'), sa.Column('plant_name_ferc1', sa.Text(), nullable=True, comment='Name of the plant, as reported to FERC. This is a freeform string, not guaranteed to be consistent across references to the same plant.'), sa.Column('project_num', sa.Integer(), nullable=True, comment='FERC Licensed Project Number.'), - sa.Column('plant_type', sa.Enum('run_of_river', 'na_category', 'run_of_river_with_storage', 'storage', 'hydro'), nullable=True), + sa.Column('plant_type', sa.Enum('na_category', 'run_of_river', 'run_of_river_with_storage', 'storage', 'hydro'), nullable=True), sa.Column('construction_type', sa.Enum('conventional', 'outdoor', 'semioutdoor'), nullable=True, comment="Type of plant construction ('outdoor', 'semioutdoor', or 'conventional'). Categorized by PUDL based on our best guess of intended value in FERC1 freeform strings."), sa.Column('construction_year', sa.Integer(), nullable=True, comment="Year the plant's oldest still operational unit was built."), sa.Column('installation_year', sa.Integer(), nullable=True, comment="Year the plant's most recently built unit was installed."), @@ -2774,9 +2774,8 @@ def upgrade() -> None: sa.Column('record_id', sa.Text(), nullable=True, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), sa.Column('utility_id_ferc1', sa.Integer(), nullable=True, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), sa.Column('report_year', sa.Integer(), nullable=True, comment='Four-digit year in which the data was reported.'), - sa.Column('plant_id_ferc1', sa.Integer(), nullable=True, comment='Algorithmically assigned PUDL FERC Plant ID. WARNING: NOT STABLE BETWEEN PUDL DB INITIALIZATIONS.'), sa.Column('plant_name_ferc1', sa.Text(), nullable=True, comment='Name of the plant, as reported to FERC. This is a freeform string, not guaranteed to be consistent across references to the same plant.'), - sa.Column('plant_type', sa.Enum('solar_thermal', 'internal_combustion', 'wind', 'combustion_turbine', 'combined_cycle', 'steam', 'na_category', 'photovoltaic', 'nuclear', 'geothermal'), nullable=True), + sa.Column('plant_type', sa.Enum('na_category', 'wind', 'solar_thermal', 'geothermal', 'steam', 'photovoltaic', 'combined_cycle', 'nuclear', 'combustion_turbine', 'internal_combustion'), nullable=True), sa.Column('construction_type', sa.Enum('conventional', 'outdoor', 'semioutdoor'), nullable=True, comment="Type of plant construction ('outdoor', 'semioutdoor', or 'conventional'). Categorized by PUDL based on our best guess of intended value in FERC1 freeform strings."), sa.Column('construction_year', sa.Integer(), nullable=True, comment="Year the plant's oldest still operational unit was built."), sa.Column('installation_year', sa.Integer(), nullable=True, comment="Year the plant's most recently built unit was installed."), @@ -2871,12 +2870,12 @@ def upgrade() -> None: op.create_table('out_eia__yearly_plants', sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), sa.Column('plant_name_eia', sa.Text(), nullable=True, comment='Plant name.'), - sa.Column('city', sa.Text(), nullable=True), + sa.Column('city', sa.Text(), nullable=True, comment='Name of the city.'), sa.Column('county', sa.Text(), nullable=True, comment='County name.'), sa.Column('latitude', sa.Float(), nullable=True, comment="Latitude of the plant's location, in degrees."), sa.Column('longitude', sa.Float(), nullable=True, comment="Longitude of the plant's location, in degrees."), sa.Column('state', sa.Text(), nullable=True, comment='Two letter US state abbreviation.'), - sa.Column('street_address', sa.Text(), nullable=True), + sa.Column('street_address', sa.Text(), nullable=True, comment='Physical street address.'), sa.Column('zip_code', sa.Text(), nullable=True, comment='Five digit US Zip Code.'), sa.Column('timezone', sa.Enum('Africa/Abidjan', 'Africa/Accra', 'Africa/Addis_Ababa', 'Africa/Algiers', 'Africa/Asmara', 'Africa/Asmera', 'Africa/Bamako', 'Africa/Bangui', 'Africa/Banjul', 'Africa/Bissau', 'Africa/Blantyre', 'Africa/Brazzaville', 'Africa/Bujumbura', 'Africa/Cairo', 'Africa/Casablanca', 'Africa/Ceuta', 'Africa/Conakry', 'Africa/Dakar', 'Africa/Dar_es_Salaam', 'Africa/Djibouti', 'Africa/Douala', 'Africa/El_Aaiun', 'Africa/Freetown', 'Africa/Gaborone', 'Africa/Harare', 'Africa/Johannesburg', 'Africa/Juba', 'Africa/Kampala', 'Africa/Khartoum', 'Africa/Kigali', 'Africa/Kinshasa', 'Africa/Lagos', 'Africa/Libreville', 'Africa/Lome', 'Africa/Luanda', 'Africa/Lubumbashi', 'Africa/Lusaka', 'Africa/Malabo', 'Africa/Maputo', 'Africa/Maseru', 'Africa/Mbabane', 'Africa/Mogadishu', 'Africa/Monrovia', 'Africa/Nairobi', 'Africa/Ndjamena', 'Africa/Niamey', 'Africa/Nouakchott', 'Africa/Ouagadougou', 'Africa/Porto-Novo', 'Africa/Sao_Tome', 'Africa/Timbuktu', 'Africa/Tripoli', 'Africa/Tunis', 'Africa/Windhoek', 'America/Adak', 'America/Anchorage', 'America/Anguilla', 'America/Antigua', 'America/Araguaina', 'America/Argentina/Buenos_Aires', 'America/Argentina/Catamarca', 'America/Argentina/ComodRivadavia', 'America/Argentina/Cordoba', 'America/Argentina/Jujuy', 'America/Argentina/La_Rioja', 'America/Argentina/Mendoza', 'America/Argentina/Rio_Gallegos', 'America/Argentina/Salta', 'America/Argentina/San_Juan', 'America/Argentina/San_Luis', 'America/Argentina/Tucuman', 'America/Argentina/Ushuaia', 'America/Aruba', 'America/Asuncion', 'America/Atikokan', 'America/Atka', 'America/Bahia', 'America/Bahia_Banderas', 'America/Barbados', 'America/Belem', 'America/Belize', 'America/Blanc-Sablon', 'America/Boa_Vista', 'America/Bogota', 'America/Boise', 'America/Buenos_Aires', 'America/Cambridge_Bay', 'America/Campo_Grande', 'America/Cancun', 'America/Caracas', 'America/Catamarca', 'America/Cayenne', 'America/Cayman', 'America/Chicago', 'America/Chihuahua', 'America/Ciudad_Juarez', 'America/Coral_Harbour', 'America/Cordoba', 'America/Costa_Rica', 'America/Creston', 'America/Cuiaba', 'America/Curacao', 'America/Danmarkshavn', 'America/Dawson', 'America/Dawson_Creek', 'America/Denver', 'America/Detroit', 'America/Dominica', 'America/Edmonton', 'America/Eirunepe', 'America/El_Salvador', 'America/Ensenada', 'America/Fort_Nelson', 'America/Fort_Wayne', 'America/Fortaleza', 'America/Glace_Bay', 'America/Godthab', 'America/Goose_Bay', 'America/Grand_Turk', 'America/Grenada', 'America/Guadeloupe', 'America/Guatemala', 'America/Guayaquil', 'America/Guyana', 'America/Halifax', 'America/Havana', 'America/Hermosillo', 'America/Indiana/Indianapolis', 'America/Indiana/Knox', 'America/Indiana/Marengo', 'America/Indiana/Petersburg', 'America/Indiana/Tell_City', 'America/Indiana/Vevay', 'America/Indiana/Vincennes', 'America/Indiana/Winamac', 'America/Indianapolis', 'America/Inuvik', 'America/Iqaluit', 'America/Jamaica', 'America/Jujuy', 'America/Juneau', 'America/Kentucky/Louisville', 'America/Kentucky/Monticello', 'America/Knox_IN', 'America/Kralendijk', 'America/La_Paz', 'America/Lima', 'America/Los_Angeles', 'America/Louisville', 'America/Lower_Princes', 'America/Maceio', 'America/Managua', 'America/Manaus', 'America/Marigot', 'America/Martinique', 'America/Matamoros', 'America/Mazatlan', 'America/Mendoza', 'America/Menominee', 'America/Merida', 'America/Metlakatla', 'America/Mexico_City', 'America/Miquelon', 'America/Moncton', 'America/Monterrey', 'America/Montevideo', 'America/Montreal', 'America/Montserrat', 'America/Nassau', 'America/New_York', 'America/Nipigon', 'America/Nome', 'America/Noronha', 'America/North_Dakota/Beulah', 'America/North_Dakota/Center', 'America/North_Dakota/New_Salem', 'America/Nuuk', 'America/Ojinaga', 'America/Panama', 'America/Pangnirtung', 'America/Paramaribo', 'America/Phoenix', 'America/Port-au-Prince', 'America/Port_of_Spain', 'America/Porto_Acre', 'America/Porto_Velho', 'America/Puerto_Rico', 'America/Punta_Arenas', 'America/Rainy_River', 'America/Rankin_Inlet', 'America/Recife', 'America/Regina', 'America/Resolute', 'America/Rio_Branco', 'America/Rosario', 'America/Santa_Isabel', 'America/Santarem', 'America/Santiago', 'America/Santo_Domingo', 'America/Sao_Paulo', 'America/Scoresbysund', 'America/Shiprock', 'America/Sitka', 'America/St_Barthelemy', 'America/St_Johns', 'America/St_Kitts', 'America/St_Lucia', 'America/St_Thomas', 'America/St_Vincent', 'America/Swift_Current', 'America/Tegucigalpa', 'America/Thule', 'America/Thunder_Bay', 'America/Tijuana', 'America/Toronto', 'America/Tortola', 'America/Vancouver', 'America/Virgin', 'America/Whitehorse', 'America/Winnipeg', 'America/Yakutat', 'America/Yellowknife', 'Antarctica/Casey', 'Antarctica/Davis', 'Antarctica/DumontDUrville', 'Antarctica/Macquarie', 'Antarctica/Mawson', 'Antarctica/McMurdo', 'Antarctica/Palmer', 'Antarctica/Rothera', 'Antarctica/South_Pole', 'Antarctica/Syowa', 'Antarctica/Troll', 'Antarctica/Vostok', 'Arctic/Longyearbyen', 'Asia/Aden', 'Asia/Almaty', 'Asia/Amman', 'Asia/Anadyr', 'Asia/Aqtau', 'Asia/Aqtobe', 'Asia/Ashgabat', 'Asia/Ashkhabad', 'Asia/Atyrau', 'Asia/Baghdad', 'Asia/Bahrain', 'Asia/Baku', 'Asia/Bangkok', 'Asia/Barnaul', 'Asia/Beirut', 'Asia/Bishkek', 'Asia/Brunei', 'Asia/Calcutta', 'Asia/Chita', 'Asia/Choibalsan', 'Asia/Chongqing', 'Asia/Chungking', 'Asia/Colombo', 'Asia/Dacca', 'Asia/Damascus', 'Asia/Dhaka', 'Asia/Dili', 'Asia/Dubai', 'Asia/Dushanbe', 'Asia/Famagusta', 'Asia/Gaza', 'Asia/Harbin', 'Asia/Hebron', 'Asia/Ho_Chi_Minh', 'Asia/Hong_Kong', 'Asia/Hovd', 'Asia/Irkutsk', 'Asia/Istanbul', 'Asia/Jakarta', 'Asia/Jayapura', 'Asia/Jerusalem', 'Asia/Kabul', 'Asia/Kamchatka', 'Asia/Karachi', 'Asia/Kashgar', 'Asia/Kathmandu', 'Asia/Katmandu', 'Asia/Khandyga', 'Asia/Kolkata', 'Asia/Krasnoyarsk', 'Asia/Kuala_Lumpur', 'Asia/Kuching', 'Asia/Kuwait', 'Asia/Macao', 'Asia/Macau', 'Asia/Magadan', 'Asia/Makassar', 'Asia/Manila', 'Asia/Muscat', 'Asia/Nicosia', 'Asia/Novokuznetsk', 'Asia/Novosibirsk', 'Asia/Omsk', 'Asia/Oral', 'Asia/Phnom_Penh', 'Asia/Pontianak', 'Asia/Pyongyang', 'Asia/Qatar', 'Asia/Qostanay', 'Asia/Qyzylorda', 'Asia/Rangoon', 'Asia/Riyadh', 'Asia/Saigon', 'Asia/Sakhalin', 'Asia/Samarkand', 'Asia/Seoul', 'Asia/Shanghai', 'Asia/Singapore', 'Asia/Srednekolymsk', 'Asia/Taipei', 'Asia/Tashkent', 'Asia/Tbilisi', 'Asia/Tehran', 'Asia/Tel_Aviv', 'Asia/Thimbu', 'Asia/Thimphu', 'Asia/Tokyo', 'Asia/Tomsk', 'Asia/Ujung_Pandang', 'Asia/Ulaanbaatar', 'Asia/Ulan_Bator', 'Asia/Urumqi', 'Asia/Ust-Nera', 'Asia/Vientiane', 'Asia/Vladivostok', 'Asia/Yakutsk', 'Asia/Yangon', 'Asia/Yekaterinburg', 'Asia/Yerevan', 'Atlantic/Azores', 'Atlantic/Bermuda', 'Atlantic/Canary', 'Atlantic/Cape_Verde', 'Atlantic/Faeroe', 'Atlantic/Faroe', 'Atlantic/Jan_Mayen', 'Atlantic/Madeira', 'Atlantic/Reykjavik', 'Atlantic/South_Georgia', 'Atlantic/St_Helena', 'Atlantic/Stanley', 'Australia/ACT', 'Australia/Adelaide', 'Australia/Brisbane', 'Australia/Broken_Hill', 'Australia/Canberra', 'Australia/Currie', 'Australia/Darwin', 'Australia/Eucla', 'Australia/Hobart', 'Australia/LHI', 'Australia/Lindeman', 'Australia/Lord_Howe', 'Australia/Melbourne', 'Australia/NSW', 'Australia/North', 'Australia/Perth', 'Australia/Queensland', 'Australia/South', 'Australia/Sydney', 'Australia/Tasmania', 'Australia/Victoria', 'Australia/West', 'Australia/Yancowinna', 'Brazil/Acre', 'Brazil/DeNoronha', 'Brazil/East', 'Brazil/West', 'CET', 'CST6CDT', 'Canada/Atlantic', 'Canada/Central', 'Canada/Eastern', 'Canada/Mountain', 'Canada/Newfoundland', 'Canada/Pacific', 'Canada/Saskatchewan', 'Canada/Yukon', 'Chile/Continental', 'Chile/EasterIsland', 'Cuba', 'EET', 'EST', 'EST5EDT', 'Egypt', 'Eire', 'Etc/GMT', 'Etc/GMT+0', 'Etc/GMT+1', 'Etc/GMT+10', 'Etc/GMT+11', 'Etc/GMT+12', 'Etc/GMT+2', 'Etc/GMT+3', 'Etc/GMT+4', 'Etc/GMT+5', 'Etc/GMT+6', 'Etc/GMT+7', 'Etc/GMT+8', 'Etc/GMT+9', 'Etc/GMT-0', 'Etc/GMT-1', 'Etc/GMT-10', 'Etc/GMT-11', 'Etc/GMT-12', 'Etc/GMT-13', 'Etc/GMT-14', 'Etc/GMT-2', 'Etc/GMT-3', 'Etc/GMT-4', 'Etc/GMT-5', 'Etc/GMT-6', 'Etc/GMT-7', 'Etc/GMT-8', 'Etc/GMT-9', 'Etc/GMT0', 'Etc/Greenwich', 'Etc/UCT', 'Etc/UTC', 'Etc/Universal', 'Etc/Zulu', 'Europe/Amsterdam', 'Europe/Andorra', 'Europe/Astrakhan', 'Europe/Athens', 'Europe/Belfast', 'Europe/Belgrade', 'Europe/Berlin', 'Europe/Bratislava', 'Europe/Brussels', 'Europe/Bucharest', 'Europe/Budapest', 'Europe/Busingen', 'Europe/Chisinau', 'Europe/Copenhagen', 'Europe/Dublin', 'Europe/Gibraltar', 'Europe/Guernsey', 'Europe/Helsinki', 'Europe/Isle_of_Man', 'Europe/Istanbul', 'Europe/Jersey', 'Europe/Kaliningrad', 'Europe/Kiev', 'Europe/Kirov', 'Europe/Kyiv', 'Europe/Lisbon', 'Europe/Ljubljana', 'Europe/London', 'Europe/Luxembourg', 'Europe/Madrid', 'Europe/Malta', 'Europe/Mariehamn', 'Europe/Minsk', 'Europe/Monaco', 'Europe/Moscow', 'Europe/Nicosia', 'Europe/Oslo', 'Europe/Paris', 'Europe/Podgorica', 'Europe/Prague', 'Europe/Riga', 'Europe/Rome', 'Europe/Samara', 'Europe/San_Marino', 'Europe/Sarajevo', 'Europe/Saratov', 'Europe/Simferopol', 'Europe/Skopje', 'Europe/Sofia', 'Europe/Stockholm', 'Europe/Tallinn', 'Europe/Tirane', 'Europe/Tiraspol', 'Europe/Ulyanovsk', 'Europe/Uzhgorod', 'Europe/Vaduz', 'Europe/Vatican', 'Europe/Vienna', 'Europe/Vilnius', 'Europe/Volgograd', 'Europe/Warsaw', 'Europe/Zagreb', 'Europe/Zaporozhye', 'Europe/Zurich', 'GB', 'GB-Eire', 'GMT', 'GMT+0', 'GMT-0', 'GMT0', 'Greenwich', 'HST', 'Hongkong', 'Iceland', 'Indian/Antananarivo', 'Indian/Chagos', 'Indian/Christmas', 'Indian/Cocos', 'Indian/Comoro', 'Indian/Kerguelen', 'Indian/Mahe', 'Indian/Maldives', 'Indian/Mauritius', 'Indian/Mayotte', 'Indian/Reunion', 'Iran', 'Israel', 'Jamaica', 'Japan', 'Kwajalein', 'Libya', 'MET', 'MST', 'MST7MDT', 'Mexico/BajaNorte', 'Mexico/BajaSur', 'Mexico/General', 'NZ', 'NZ-CHAT', 'Navajo', 'PRC', 'PST8PDT', 'Pacific/Apia', 'Pacific/Auckland', 'Pacific/Bougainville', 'Pacific/Chatham', 'Pacific/Chuuk', 'Pacific/Easter', 'Pacific/Efate', 'Pacific/Enderbury', 'Pacific/Fakaofo', 'Pacific/Fiji', 'Pacific/Funafuti', 'Pacific/Galapagos', 'Pacific/Gambier', 'Pacific/Guadalcanal', 'Pacific/Guam', 'Pacific/Honolulu', 'Pacific/Johnston', 'Pacific/Kanton', 'Pacific/Kiritimati', 'Pacific/Kosrae', 'Pacific/Kwajalein', 'Pacific/Majuro', 'Pacific/Marquesas', 'Pacific/Midway', 'Pacific/Nauru', 'Pacific/Niue', 'Pacific/Norfolk', 'Pacific/Noumea', 'Pacific/Pago_Pago', 'Pacific/Palau', 'Pacific/Pitcairn', 'Pacific/Pohnpei', 'Pacific/Ponape', 'Pacific/Port_Moresby', 'Pacific/Rarotonga', 'Pacific/Saipan', 'Pacific/Samoa', 'Pacific/Tahiti', 'Pacific/Tarawa', 'Pacific/Tongatapu', 'Pacific/Truk', 'Pacific/Wake', 'Pacific/Wallis', 'Pacific/Yap', 'Poland', 'Portugal', 'ROC', 'ROK', 'Singapore', 'Turkey', 'UCT', 'US/Alaska', 'US/Aleutian', 'US/Arizona', 'US/Central', 'US/East-Indiana', 'US/Eastern', 'US/Hawaii', 'US/Indiana-Starke', 'US/Michigan', 'US/Mountain', 'US/Pacific', 'US/Samoa', 'UTC', 'Universal', 'W-SU', 'WET', 'Zulu'), nullable=True, comment='IANA timezone name'), sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), @@ -3127,7 +3126,7 @@ def upgrade() -> None: sa.Column('bypass_heat_recovery', sa.Boolean(), nullable=True, comment='Can this generator operate while bypassing the heat recovery steam generator?'), sa.Column('capacity_mw', sa.Float(), nullable=True, comment='Total installed (nameplate) capacity, in megawatts.'), sa.Column('carbon_capture', sa.Boolean(), nullable=True, comment='Indicates whether the generator uses carbon capture technology.'), - sa.Column('city', sa.Text(), nullable=True), + sa.Column('city', sa.Text(), nullable=True, comment='Name of the city.'), sa.Column('cofire_fuels', sa.Boolean(), nullable=True, comment='Can the generator co-fire fuels?.'), sa.Column('county', sa.Text(), nullable=True, comment='County name.'), sa.Column('current_planned_generator_operating_date', sa.Date(), nullable=True, comment='The most recently updated effective date on which the generator is scheduled to start operation'), @@ -3194,7 +3193,7 @@ def upgrade() -> None: sa.Column('startup_source_code_4', sa.Text(), nullable=True, comment='The code representing the first, second, third or fourth start-up and flame stabilization energy source used by the combustion unit(s) associated with this generator.'), sa.Column('state', sa.Text(), nullable=True, comment='Two letter US state abbreviation.'), sa.Column('stoker_tech', sa.Boolean(), nullable=True, comment='Indicates whether the generator uses stoker technology'), - sa.Column('street_address', sa.Text(), nullable=True), + sa.Column('street_address', sa.Text(), nullable=True, comment='Physical street address.'), sa.Column('subcritical_tech', sa.Boolean(), nullable=True, comment='Indicates whether the generator uses subcritical technology'), sa.Column('summer_capacity_estimate', sa.Boolean(), nullable=True, comment='Whether the summer capacity value was an estimate'), sa.Column('summer_capacity_mw', sa.Float(), nullable=True, comment='The net summer capacity.'), @@ -3312,7 +3311,7 @@ def upgrade() -> None: sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), sa.Column('generator_id', sa.Text(), nullable=False, comment='Generator ID is usually numeric, but sometimes includes letters. Make sure you treat it as a string!'), sa.Column('owner_utility_name_eia', sa.Text(), nullable=True, comment='The name of the EIA owner utility.'), - sa.Column('owner_state', sa.Enum('NJ', 'KY', 'PE', 'NT', 'DE', 'MP', 'ON', 'MB', 'OK', 'SD', 'GU', 'CA', 'CT', 'NE', 'AS', 'UT', 'KS', 'MO', 'NC', 'QC', 'NB', 'AL', 'AR', 'BC', 'NU', 'OR', 'VT', 'NM', 'PR', 'WI', 'WA', 'PA', 'RI', 'TX', 'GA', 'MA', 'AB', 'AZ', 'YT', 'ME', 'VI', 'SC', 'IN', 'WV', 'FL', 'NY', 'MS', 'ID', 'DC', 'IL', 'MI', 'CO', 'LA', 'NL', 'MD', 'OH', 'NS', 'AK', 'SK', 'VA', 'MT', 'ND', 'NH', 'MN', 'TN', 'IA', 'HI', 'WY', 'NV'), nullable=True, comment='Two letter ISO-3166 political subdivision code.'), + sa.Column('owner_state', sa.Enum('BC', 'IN', 'IA', 'SK', 'NV', 'HI', 'VA', 'PE', 'ON', 'GA', 'MD', 'PR', 'WY', 'WI', 'AS', 'MT', 'KS', 'NS', 'CO', 'ND', 'OK', 'MI', 'IL', 'NU', 'AK', 'LA', 'AL', 'ID', 'MA', 'MP', 'NB', 'PA', 'ME', 'RI', 'WV', 'KY', 'OR', 'NY', 'NJ', 'TN', 'DC', 'QC', 'TX', 'DE', 'OH', 'AZ', 'AR', 'CT', 'NT', 'CA', 'WA', 'MS', 'GU', 'NE', 'NM', 'SD', 'FL', 'SC', 'MN', 'MO', 'MB', 'YT', 'NC', 'UT', 'NL', 'AB', 'NH', 'VI', 'VT'), nullable=True, comment='Two letter ISO-3166 political subdivision code.'), sa.Column('owner_city', sa.Text(), nullable=True, comment='City of owner.'), sa.Column('owner_country', sa.Enum('USA', 'CAN'), nullable=True, comment='Three letter ISO-3166 country code.'), sa.Column('owner_street_address', sa.Text(), nullable=True, comment='Steet address of owner.'), @@ -3333,7 +3332,7 @@ def upgrade() -> None: sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), sa.Column('owner_utility_name_eia', sa.Text(), nullable=True, comment='The name of the EIA owner utility.'), sa.Column('generator_id', sa.Text(), nullable=False, comment='Generator ID is usually numeric, but sometimes includes letters. Make sure you treat it as a string!'), - sa.Column('owner_state', sa.Enum('NJ', 'KY', 'PE', 'NT', 'DE', 'MP', 'ON', 'MB', 'OK', 'SD', 'GU', 'CA', 'CT', 'NE', 'AS', 'UT', 'KS', 'MO', 'NC', 'QC', 'NB', 'AL', 'AR', 'BC', 'NU', 'OR', 'VT', 'NM', 'PR', 'WI', 'WA', 'PA', 'RI', 'TX', 'GA', 'MA', 'AB', 'AZ', 'YT', 'ME', 'VI', 'SC', 'IN', 'WV', 'FL', 'NY', 'MS', 'ID', 'DC', 'IL', 'MI', 'CO', 'LA', 'NL', 'MD', 'OH', 'NS', 'AK', 'SK', 'VA', 'MT', 'ND', 'NH', 'MN', 'TN', 'IA', 'HI', 'WY', 'NV'), nullable=True, comment='Two letter ISO-3166 political subdivision code.'), + sa.Column('owner_state', sa.Enum('BC', 'IN', 'IA', 'SK', 'NV', 'HI', 'VA', 'PE', 'ON', 'GA', 'MD', 'PR', 'WY', 'WI', 'AS', 'MT', 'KS', 'NS', 'CO', 'ND', 'OK', 'MI', 'IL', 'NU', 'AK', 'LA', 'AL', 'ID', 'MA', 'MP', 'NB', 'PA', 'ME', 'RI', 'WV', 'KY', 'OR', 'NY', 'NJ', 'TN', 'DC', 'QC', 'TX', 'DE', 'OH', 'AZ', 'AR', 'CT', 'NT', 'CA', 'WA', 'MS', 'GU', 'NE', 'NM', 'SD', 'FL', 'SC', 'MN', 'MO', 'MB', 'YT', 'NC', 'UT', 'NL', 'AB', 'NH', 'VI', 'VT'), nullable=True, comment='Two letter ISO-3166 political subdivision code.'), sa.Column('owner_city', sa.Text(), nullable=True, comment='City of owner.'), sa.Column('owner_country', sa.Enum('USA', 'CAN'), nullable=True, comment='Three letter ISO-3166 country code.'), sa.Column('owner_street_address', sa.Text(), nullable=True, comment='Steet address of owner.'), @@ -3467,7 +3466,7 @@ def upgrade() -> None: sa.Column('boiler_retirement_date', sa.Date(), nullable=True, comment='Date of the scheduled or effected retirement of the boiler.'), sa.Column('boiler_status', sa.Text(), nullable=True, comment='EIA short code identifying boiler operational status.'), sa.Column('boiler_type', sa.Text(), nullable=True, comment='EIA short code indicating the standards under which the boiler is operating as described in the U.S. EPA regulation under 40 CFR.'), - sa.Column('city', sa.Text(), nullable=True), + sa.Column('city', sa.Text(), nullable=True, comment='Name of the city.'), sa.Column('compliance_year_mercury', sa.Integer(), nullable=True, comment='Year boiler was or is expected to be in compliance with federal, state and/or local regulations for mercury emissions.'), sa.Column('compliance_year_nox', sa.Integer(), nullable=True, comment='Year boiler was or is expected to be in compliance with federal, state and/or local regulations for nitrogen oxide emissions.'), sa.Column('compliance_year_particulate', sa.Integer(), nullable=True, comment='Year boiler was or is expected to be in compliance with federal, state and/or local regulations for particulate matter emissions.'), @@ -3545,7 +3544,7 @@ def upgrade() -> None: sa.Column('standard_so2_percent_scrubbed', sa.Float(), nullable=True, comment='The percent of sulfur dioxide to be scrubbed specified by the most stringent sulfur dioxide regulation.'), sa.Column('standard_so2_rate', sa.Float(), nullable=True, comment='Numeric value for the unit of measurement specified for sulfur dioxide.'), sa.Column('state', sa.Text(), nullable=True, comment='Two letter US state abbreviation.'), - sa.Column('street_address', sa.Text(), nullable=True), + sa.Column('street_address', sa.Text(), nullable=True, comment='Physical street address.'), sa.Column('timezone', sa.Enum('Africa/Abidjan', 'Africa/Accra', 'Africa/Addis_Ababa', 'Africa/Algiers', 'Africa/Asmara', 'Africa/Asmera', 'Africa/Bamako', 'Africa/Bangui', 'Africa/Banjul', 'Africa/Bissau', 'Africa/Blantyre', 'Africa/Brazzaville', 'Africa/Bujumbura', 'Africa/Cairo', 'Africa/Casablanca', 'Africa/Ceuta', 'Africa/Conakry', 'Africa/Dakar', 'Africa/Dar_es_Salaam', 'Africa/Djibouti', 'Africa/Douala', 'Africa/El_Aaiun', 'Africa/Freetown', 'Africa/Gaborone', 'Africa/Harare', 'Africa/Johannesburg', 'Africa/Juba', 'Africa/Kampala', 'Africa/Khartoum', 'Africa/Kigali', 'Africa/Kinshasa', 'Africa/Lagos', 'Africa/Libreville', 'Africa/Lome', 'Africa/Luanda', 'Africa/Lubumbashi', 'Africa/Lusaka', 'Africa/Malabo', 'Africa/Maputo', 'Africa/Maseru', 'Africa/Mbabane', 'Africa/Mogadishu', 'Africa/Monrovia', 'Africa/Nairobi', 'Africa/Ndjamena', 'Africa/Niamey', 'Africa/Nouakchott', 'Africa/Ouagadougou', 'Africa/Porto-Novo', 'Africa/Sao_Tome', 'Africa/Timbuktu', 'Africa/Tripoli', 'Africa/Tunis', 'Africa/Windhoek', 'America/Adak', 'America/Anchorage', 'America/Anguilla', 'America/Antigua', 'America/Araguaina', 'America/Argentina/Buenos_Aires', 'America/Argentina/Catamarca', 'America/Argentina/ComodRivadavia', 'America/Argentina/Cordoba', 'America/Argentina/Jujuy', 'America/Argentina/La_Rioja', 'America/Argentina/Mendoza', 'America/Argentina/Rio_Gallegos', 'America/Argentina/Salta', 'America/Argentina/San_Juan', 'America/Argentina/San_Luis', 'America/Argentina/Tucuman', 'America/Argentina/Ushuaia', 'America/Aruba', 'America/Asuncion', 'America/Atikokan', 'America/Atka', 'America/Bahia', 'America/Bahia_Banderas', 'America/Barbados', 'America/Belem', 'America/Belize', 'America/Blanc-Sablon', 'America/Boa_Vista', 'America/Bogota', 'America/Boise', 'America/Buenos_Aires', 'America/Cambridge_Bay', 'America/Campo_Grande', 'America/Cancun', 'America/Caracas', 'America/Catamarca', 'America/Cayenne', 'America/Cayman', 'America/Chicago', 'America/Chihuahua', 'America/Ciudad_Juarez', 'America/Coral_Harbour', 'America/Cordoba', 'America/Costa_Rica', 'America/Creston', 'America/Cuiaba', 'America/Curacao', 'America/Danmarkshavn', 'America/Dawson', 'America/Dawson_Creek', 'America/Denver', 'America/Detroit', 'America/Dominica', 'America/Edmonton', 'America/Eirunepe', 'America/El_Salvador', 'America/Ensenada', 'America/Fort_Nelson', 'America/Fort_Wayne', 'America/Fortaleza', 'America/Glace_Bay', 'America/Godthab', 'America/Goose_Bay', 'America/Grand_Turk', 'America/Grenada', 'America/Guadeloupe', 'America/Guatemala', 'America/Guayaquil', 'America/Guyana', 'America/Halifax', 'America/Havana', 'America/Hermosillo', 'America/Indiana/Indianapolis', 'America/Indiana/Knox', 'America/Indiana/Marengo', 'America/Indiana/Petersburg', 'America/Indiana/Tell_City', 'America/Indiana/Vevay', 'America/Indiana/Vincennes', 'America/Indiana/Winamac', 'America/Indianapolis', 'America/Inuvik', 'America/Iqaluit', 'America/Jamaica', 'America/Jujuy', 'America/Juneau', 'America/Kentucky/Louisville', 'America/Kentucky/Monticello', 'America/Knox_IN', 'America/Kralendijk', 'America/La_Paz', 'America/Lima', 'America/Los_Angeles', 'America/Louisville', 'America/Lower_Princes', 'America/Maceio', 'America/Managua', 'America/Manaus', 'America/Marigot', 'America/Martinique', 'America/Matamoros', 'America/Mazatlan', 'America/Mendoza', 'America/Menominee', 'America/Merida', 'America/Metlakatla', 'America/Mexico_City', 'America/Miquelon', 'America/Moncton', 'America/Monterrey', 'America/Montevideo', 'America/Montreal', 'America/Montserrat', 'America/Nassau', 'America/New_York', 'America/Nipigon', 'America/Nome', 'America/Noronha', 'America/North_Dakota/Beulah', 'America/North_Dakota/Center', 'America/North_Dakota/New_Salem', 'America/Nuuk', 'America/Ojinaga', 'America/Panama', 'America/Pangnirtung', 'America/Paramaribo', 'America/Phoenix', 'America/Port-au-Prince', 'America/Port_of_Spain', 'America/Porto_Acre', 'America/Porto_Velho', 'America/Puerto_Rico', 'America/Punta_Arenas', 'America/Rainy_River', 'America/Rankin_Inlet', 'America/Recife', 'America/Regina', 'America/Resolute', 'America/Rio_Branco', 'America/Rosario', 'America/Santa_Isabel', 'America/Santarem', 'America/Santiago', 'America/Santo_Domingo', 'America/Sao_Paulo', 'America/Scoresbysund', 'America/Shiprock', 'America/Sitka', 'America/St_Barthelemy', 'America/St_Johns', 'America/St_Kitts', 'America/St_Lucia', 'America/St_Thomas', 'America/St_Vincent', 'America/Swift_Current', 'America/Tegucigalpa', 'America/Thule', 'America/Thunder_Bay', 'America/Tijuana', 'America/Toronto', 'America/Tortola', 'America/Vancouver', 'America/Virgin', 'America/Whitehorse', 'America/Winnipeg', 'America/Yakutat', 'America/Yellowknife', 'Antarctica/Casey', 'Antarctica/Davis', 'Antarctica/DumontDUrville', 'Antarctica/Macquarie', 'Antarctica/Mawson', 'Antarctica/McMurdo', 'Antarctica/Palmer', 'Antarctica/Rothera', 'Antarctica/South_Pole', 'Antarctica/Syowa', 'Antarctica/Troll', 'Antarctica/Vostok', 'Arctic/Longyearbyen', 'Asia/Aden', 'Asia/Almaty', 'Asia/Amman', 'Asia/Anadyr', 'Asia/Aqtau', 'Asia/Aqtobe', 'Asia/Ashgabat', 'Asia/Ashkhabad', 'Asia/Atyrau', 'Asia/Baghdad', 'Asia/Bahrain', 'Asia/Baku', 'Asia/Bangkok', 'Asia/Barnaul', 'Asia/Beirut', 'Asia/Bishkek', 'Asia/Brunei', 'Asia/Calcutta', 'Asia/Chita', 'Asia/Choibalsan', 'Asia/Chongqing', 'Asia/Chungking', 'Asia/Colombo', 'Asia/Dacca', 'Asia/Damascus', 'Asia/Dhaka', 'Asia/Dili', 'Asia/Dubai', 'Asia/Dushanbe', 'Asia/Famagusta', 'Asia/Gaza', 'Asia/Harbin', 'Asia/Hebron', 'Asia/Ho_Chi_Minh', 'Asia/Hong_Kong', 'Asia/Hovd', 'Asia/Irkutsk', 'Asia/Istanbul', 'Asia/Jakarta', 'Asia/Jayapura', 'Asia/Jerusalem', 'Asia/Kabul', 'Asia/Kamchatka', 'Asia/Karachi', 'Asia/Kashgar', 'Asia/Kathmandu', 'Asia/Katmandu', 'Asia/Khandyga', 'Asia/Kolkata', 'Asia/Krasnoyarsk', 'Asia/Kuala_Lumpur', 'Asia/Kuching', 'Asia/Kuwait', 'Asia/Macao', 'Asia/Macau', 'Asia/Magadan', 'Asia/Makassar', 'Asia/Manila', 'Asia/Muscat', 'Asia/Nicosia', 'Asia/Novokuznetsk', 'Asia/Novosibirsk', 'Asia/Omsk', 'Asia/Oral', 'Asia/Phnom_Penh', 'Asia/Pontianak', 'Asia/Pyongyang', 'Asia/Qatar', 'Asia/Qostanay', 'Asia/Qyzylorda', 'Asia/Rangoon', 'Asia/Riyadh', 'Asia/Saigon', 'Asia/Sakhalin', 'Asia/Samarkand', 'Asia/Seoul', 'Asia/Shanghai', 'Asia/Singapore', 'Asia/Srednekolymsk', 'Asia/Taipei', 'Asia/Tashkent', 'Asia/Tbilisi', 'Asia/Tehran', 'Asia/Tel_Aviv', 'Asia/Thimbu', 'Asia/Thimphu', 'Asia/Tokyo', 'Asia/Tomsk', 'Asia/Ujung_Pandang', 'Asia/Ulaanbaatar', 'Asia/Ulan_Bator', 'Asia/Urumqi', 'Asia/Ust-Nera', 'Asia/Vientiane', 'Asia/Vladivostok', 'Asia/Yakutsk', 'Asia/Yangon', 'Asia/Yekaterinburg', 'Asia/Yerevan', 'Atlantic/Azores', 'Atlantic/Bermuda', 'Atlantic/Canary', 'Atlantic/Cape_Verde', 'Atlantic/Faeroe', 'Atlantic/Faroe', 'Atlantic/Jan_Mayen', 'Atlantic/Madeira', 'Atlantic/Reykjavik', 'Atlantic/South_Georgia', 'Atlantic/St_Helena', 'Atlantic/Stanley', 'Australia/ACT', 'Australia/Adelaide', 'Australia/Brisbane', 'Australia/Broken_Hill', 'Australia/Canberra', 'Australia/Currie', 'Australia/Darwin', 'Australia/Eucla', 'Australia/Hobart', 'Australia/LHI', 'Australia/Lindeman', 'Australia/Lord_Howe', 'Australia/Melbourne', 'Australia/NSW', 'Australia/North', 'Australia/Perth', 'Australia/Queensland', 'Australia/South', 'Australia/Sydney', 'Australia/Tasmania', 'Australia/Victoria', 'Australia/West', 'Australia/Yancowinna', 'Brazil/Acre', 'Brazil/DeNoronha', 'Brazil/East', 'Brazil/West', 'CET', 'CST6CDT', 'Canada/Atlantic', 'Canada/Central', 'Canada/Eastern', 'Canada/Mountain', 'Canada/Newfoundland', 'Canada/Pacific', 'Canada/Saskatchewan', 'Canada/Yukon', 'Chile/Continental', 'Chile/EasterIsland', 'Cuba', 'EET', 'EST', 'EST5EDT', 'Egypt', 'Eire', 'Etc/GMT', 'Etc/GMT+0', 'Etc/GMT+1', 'Etc/GMT+10', 'Etc/GMT+11', 'Etc/GMT+12', 'Etc/GMT+2', 'Etc/GMT+3', 'Etc/GMT+4', 'Etc/GMT+5', 'Etc/GMT+6', 'Etc/GMT+7', 'Etc/GMT+8', 'Etc/GMT+9', 'Etc/GMT-0', 'Etc/GMT-1', 'Etc/GMT-10', 'Etc/GMT-11', 'Etc/GMT-12', 'Etc/GMT-13', 'Etc/GMT-14', 'Etc/GMT-2', 'Etc/GMT-3', 'Etc/GMT-4', 'Etc/GMT-5', 'Etc/GMT-6', 'Etc/GMT-7', 'Etc/GMT-8', 'Etc/GMT-9', 'Etc/GMT0', 'Etc/Greenwich', 'Etc/UCT', 'Etc/UTC', 'Etc/Universal', 'Etc/Zulu', 'Europe/Amsterdam', 'Europe/Andorra', 'Europe/Astrakhan', 'Europe/Athens', 'Europe/Belfast', 'Europe/Belgrade', 'Europe/Berlin', 'Europe/Bratislava', 'Europe/Brussels', 'Europe/Bucharest', 'Europe/Budapest', 'Europe/Busingen', 'Europe/Chisinau', 'Europe/Copenhagen', 'Europe/Dublin', 'Europe/Gibraltar', 'Europe/Guernsey', 'Europe/Helsinki', 'Europe/Isle_of_Man', 'Europe/Istanbul', 'Europe/Jersey', 'Europe/Kaliningrad', 'Europe/Kiev', 'Europe/Kirov', 'Europe/Kyiv', 'Europe/Lisbon', 'Europe/Ljubljana', 'Europe/London', 'Europe/Luxembourg', 'Europe/Madrid', 'Europe/Malta', 'Europe/Mariehamn', 'Europe/Minsk', 'Europe/Monaco', 'Europe/Moscow', 'Europe/Nicosia', 'Europe/Oslo', 'Europe/Paris', 'Europe/Podgorica', 'Europe/Prague', 'Europe/Riga', 'Europe/Rome', 'Europe/Samara', 'Europe/San_Marino', 'Europe/Sarajevo', 'Europe/Saratov', 'Europe/Simferopol', 'Europe/Skopje', 'Europe/Sofia', 'Europe/Stockholm', 'Europe/Tallinn', 'Europe/Tirane', 'Europe/Tiraspol', 'Europe/Ulyanovsk', 'Europe/Uzhgorod', 'Europe/Vaduz', 'Europe/Vatican', 'Europe/Vienna', 'Europe/Vilnius', 'Europe/Volgograd', 'Europe/Warsaw', 'Europe/Zagreb', 'Europe/Zaporozhye', 'Europe/Zurich', 'GB', 'GB-Eire', 'GMT', 'GMT+0', 'GMT-0', 'GMT0', 'Greenwich', 'HST', 'Hongkong', 'Iceland', 'Indian/Antananarivo', 'Indian/Chagos', 'Indian/Christmas', 'Indian/Cocos', 'Indian/Comoro', 'Indian/Kerguelen', 'Indian/Mahe', 'Indian/Maldives', 'Indian/Mauritius', 'Indian/Mayotte', 'Indian/Reunion', 'Iran', 'Israel', 'Jamaica', 'Japan', 'Kwajalein', 'Libya', 'MET', 'MST', 'MST7MDT', 'Mexico/BajaNorte', 'Mexico/BajaSur', 'Mexico/General', 'NZ', 'NZ-CHAT', 'Navajo', 'PRC', 'PST8PDT', 'Pacific/Apia', 'Pacific/Auckland', 'Pacific/Bougainville', 'Pacific/Chatham', 'Pacific/Chuuk', 'Pacific/Easter', 'Pacific/Efate', 'Pacific/Enderbury', 'Pacific/Fakaofo', 'Pacific/Fiji', 'Pacific/Funafuti', 'Pacific/Galapagos', 'Pacific/Gambier', 'Pacific/Guadalcanal', 'Pacific/Guam', 'Pacific/Honolulu', 'Pacific/Johnston', 'Pacific/Kanton', 'Pacific/Kiritimati', 'Pacific/Kosrae', 'Pacific/Kwajalein', 'Pacific/Majuro', 'Pacific/Marquesas', 'Pacific/Midway', 'Pacific/Nauru', 'Pacific/Niue', 'Pacific/Norfolk', 'Pacific/Noumea', 'Pacific/Pago_Pago', 'Pacific/Palau', 'Pacific/Pitcairn', 'Pacific/Pohnpei', 'Pacific/Ponape', 'Pacific/Port_Moresby', 'Pacific/Rarotonga', 'Pacific/Saipan', 'Pacific/Samoa', 'Pacific/Tahiti', 'Pacific/Tarawa', 'Pacific/Tongatapu', 'Pacific/Truk', 'Pacific/Wake', 'Pacific/Wallis', 'Pacific/Yap', 'Poland', 'Portugal', 'ROC', 'ROK', 'Singapore', 'Turkey', 'UCT', 'US/Alaska', 'US/Aleutian', 'US/Arizona', 'US/Central', 'US/East-Indiana', 'US/Eastern', 'US/Hawaii', 'US/Indiana-Starke', 'US/Michigan', 'US/Mountain', 'US/Pacific', 'US/Samoa', 'UTC', 'Universal', 'W-SU', 'WET', 'Zulu'), nullable=True, comment='IANA timezone name'), sa.Column('turndown_ratio', sa.Float(), nullable=True, comment='The turndown ratio for the boiler.'), sa.Column('unit_id_pudl', sa.Integer(), nullable=True, comment='Dynamically assigned PUDL unit id. WARNING: This ID is not guaranteed to be static long term as the input data and algorithm may evolve over time.'), @@ -3643,7 +3642,7 @@ def upgrade() -> None: sa.Column('bga_source', sa.Text(), nullable=True, comment='The source from where the unit_id_pudl is compiled. The unit_id_pudl comes directly from EIA 860, or string association (which looks at all the boilers and generators that are not associated with a unit and tries to find a matching string in the respective collection of boilers or generator), or from a unit connection (where the unit_id_eia is employed to find additional boiler generator connections).'), sa.Column('bypass_heat_recovery', sa.Boolean(), nullable=True, comment='Can this generator operate while bypassing the heat recovery steam generator?'), sa.Column('carbon_capture', sa.Boolean(), nullable=True, comment='Indicates whether the generator uses carbon capture technology.'), - sa.Column('city', sa.Text(), nullable=True), + sa.Column('city', sa.Text(), nullable=True, comment='Name of the city.'), sa.Column('cofire_fuels', sa.Boolean(), nullable=True, comment='Can the generator co-fire fuels?.'), sa.Column('county', sa.Text(), nullable=True, comment='County name.'), sa.Column('current_planned_generator_operating_date', sa.Date(), nullable=True, comment='The most recently updated effective date on which the generator is scheduled to start operation'), @@ -3703,7 +3702,7 @@ def upgrade() -> None: sa.Column('startup_source_code_4', sa.Text(), nullable=True, comment='The code representing the first, second, third or fourth start-up and flame stabilization energy source used by the combustion unit(s) associated with this generator.'), sa.Column('state', sa.Text(), nullable=True, comment='Two letter US state abbreviation.'), sa.Column('stoker_tech', sa.Boolean(), nullable=True, comment='Indicates whether the generator uses stoker technology'), - sa.Column('street_address', sa.Text(), nullable=True), + sa.Column('street_address', sa.Text(), nullable=True, comment='Physical street address.'), sa.Column('subcritical_tech', sa.Boolean(), nullable=True, comment='Indicates whether the generator uses subcritical technology'), sa.Column('summer_capacity_estimate', sa.Boolean(), nullable=True, comment='Whether the summer capacity value was an estimate'), sa.Column('summer_capacity_mw', sa.Float(), nullable=True, comment='The net summer capacity.'), @@ -3794,7 +3793,7 @@ def upgrade() -> None: sa.Column('record_id_eia', sa.Text(), nullable=False, comment='Identifier for EIA plant parts analysis records.'), sa.Column('plant_id_eia', sa.Integer(), nullable=True, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), sa.Column('report_date', sa.Date(), nullable=True, comment='Date reported.'), - sa.Column('plant_part', sa.Enum('plant_unit', 'plant_gen', 'plant', 'plant_operating_year', 'plant_prime_fuel', 'plant_prime_mover', 'plant_technology', 'plant_ferc_acct', 'plant_match_ferc1'), nullable=True, comment='The part of the plant a record corresponds to.'), + sa.Column('plant_part', sa.Enum('plant_gen', 'plant_technology', 'plant_unit', 'plant_ferc_acct', 'plant', 'plant_prime_fuel', 'plant_match_ferc1', 'plant_prime_mover', 'plant_operating_year'), nullable=True, comment='The part of the plant a record corresponds to.'), sa.Column('generator_id', sa.Text(), nullable=True, comment='Generator ID is usually numeric, but sometimes includes letters. Make sure you treat it as a string!'), sa.Column('unit_id_pudl', sa.Integer(), nullable=True, comment='Dynamically assigned PUDL unit id. WARNING: This ID is not guaranteed to be static long term as the input data and algorithm may evolve over time.'), sa.Column('prime_mover_code', sa.Text(), nullable=True, comment='Code for the type of prime mover (e.g. CT, CG)'), @@ -3803,7 +3802,7 @@ def upgrade() -> None: sa.Column('ferc_acct_name', sa.Enum('Hydraulic', 'Nuclear', 'Steam', 'Other'), nullable=True, comment='Name of FERC account, derived from technology description and prime mover code.'), sa.Column('utility_id_eia', sa.Integer(), nullable=True, comment='The EIA Utility Identification number.'), sa.Column('true_gran', sa.Boolean(), nullable=True, comment='Indicates whether a plant part list record is associated with the highest priority plant part for all identical records.'), - sa.Column('appro_part_label', sa.Enum('plant_unit', 'plant_gen', 'plant', 'plant_operating_year', 'plant_prime_fuel', 'plant_prime_mover', 'plant_technology', 'plant_ferc_acct', 'plant_match_ferc1'), nullable=True, comment='Plant part of the associated true granularity record.'), + sa.Column('appro_part_label', sa.Enum('plant_gen', 'plant_technology', 'plant_unit', 'plant_ferc_acct', 'plant', 'plant_prime_fuel', 'plant_match_ferc1', 'plant_prime_mover', 'plant_operating_year'), nullable=True, comment='Plant part of the associated true granularity record.'), sa.Column('appro_record_id_eia', sa.Text(), nullable=True, comment='EIA record ID of the associated true granularity record.'), sa.Column('ferc1_generator_agg_id', sa.Integer(), nullable=True, comment='ID dynamically assigned by PUDL to EIA records with multiple matches to a single FERC ID in the FERC-EIA manual matching process.'), sa.Column('capacity_eoy_mw', sa.Float(), nullable=True, comment='Total end of year installed (nameplate) capacity for a plant part, in megawatts.'), @@ -3832,6 +3831,7 @@ def upgrade() -> None: sa.Column('total_fuel_cost', sa.Float(), nullable=True, comment='Total annual reported fuel costs for the plant part. Includes costs from all fuels.'), sa.Column('total_mmbtu', sa.Float(), nullable=True, comment='Total annual heat content of fuel consumed by a plant part record in the plant parts list.'), sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), + sa.Column('utility_name_eia', sa.Text(), nullable=True, comment='The name of the utility.'), sa.Column('report_year', sa.Integer(), nullable=True, comment='Four-digit year in which the data was reported.'), sa.Column('plant_id_report_year', sa.Text(), nullable=True, comment='PUDL plant ID and report year of the record.'), sa.ForeignKeyConstraint(['energy_source_code_1'], ['core_eia__codes_energy_sources.code'], name=op.f('fk_out_eia__yearly_plant_parts_energy_source_code_1_core_eia__codes_energy_sources')), @@ -3847,7 +3847,7 @@ def upgrade() -> None: sa.Column('record_id_eia', sa.Text(), nullable=True, comment='Identifier for EIA plant parts analysis records.'), sa.Column('match_type', sa.Text(), nullable=True, comment='Indicates the source and validation of the match between EIA and FERC. Match types include matches was generated from the model, verified by the training data, overridden by the training data, etc.'), sa.Column('plant_name_ppe', sa.Text(), nullable=True, comment='Derived plant name that includes EIA plant name and other strings associated with ID and PK columns of the plant part.'), - sa.Column('plant_part', sa.Enum('plant_unit', 'plant_gen', 'plant', 'plant_operating_year', 'plant_prime_fuel', 'plant_prime_mover', 'plant_technology', 'plant_ferc_acct', 'plant_match_ferc1'), nullable=True, comment='The part of the plant a record corresponds to.'), + sa.Column('plant_part', sa.Enum('plant_gen', 'plant_technology', 'plant_unit', 'plant_ferc_acct', 'plant', 'plant_prime_fuel', 'plant_match_ferc1', 'plant_prime_mover', 'plant_operating_year'), nullable=True, comment='The part of the plant a record corresponds to.'), sa.Column('report_year', sa.Integer(), nullable=True, comment='Four-digit year in which the data was reported.'), sa.Column('report_date', sa.Date(), nullable=True, comment='Date reported.'), sa.Column('ownership_record_type', sa.Enum('owned', 'total'), nullable=True, comment='Whether each generator record is for one owner or represents a total of all ownerships.'), @@ -3863,7 +3863,7 @@ def upgrade() -> None: sa.Column('utility_id_eia', sa.Integer(), nullable=True, comment='The EIA Utility Identification number.'), sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), sa.Column('true_gran', sa.Boolean(), nullable=True, comment='Indicates whether a plant part list record is associated with the highest priority plant part for all identical records.'), - sa.Column('appro_part_label', sa.Enum('plant_unit', 'plant_gen', 'plant', 'plant_operating_year', 'plant_prime_fuel', 'plant_prime_mover', 'plant_technology', 'plant_ferc_acct', 'plant_match_ferc1'), nullable=True, comment='Plant part of the associated true granularity record.'), + sa.Column('appro_part_label', sa.Enum('plant_gen', 'plant_technology', 'plant_unit', 'plant_ferc_acct', 'plant', 'plant_prime_fuel', 'plant_match_ferc1', 'plant_prime_mover', 'plant_operating_year'), nullable=True, comment='Plant part of the associated true granularity record.'), sa.Column('appro_record_id_eia', sa.Text(), nullable=True, comment='EIA record ID of the associated true granularity record.'), sa.Column('record_count', sa.Integer(), nullable=True, comment='Number of distinct generator IDs that partcipated in the aggregation for a plant part list record.'), sa.Column('fraction_owned', sa.Float(), nullable=True, comment='Proportion of generator ownership attributable to this utility.'), @@ -4074,6 +4074,7 @@ def downgrade() -> None: op.drop_table('_out_eia__monthly_capacity_factor_by_generator') op.drop_table('out_ferc714__summarized_demand') op.drop_table('out_ferc714__respondents_with_fips') + op.drop_table('out_ferc714__hourly_planning_area_demand') op.drop_table('out_eia923__monthly_generation_fuel_combined') op.drop_table('out_eia923__monthly_fuel_receipts_costs') op.drop_table('out_eia923__generation_fuel_combined') @@ -4083,7 +4084,6 @@ def downgrade() -> None: op.drop_table('core_pudl__assn_ferc1_pudl_utilities') op.drop_table('core_pudl__assn_eia_pudl_utilities') op.drop_table('core_pudl__assn_eia_pudl_plants') - op.drop_table('core_ferc714__hourly_demand_pa') op.drop_table('core_eia__entity_generators') op.drop_table('core_eia__entity_boilers') op.drop_table('core_eia923__monthly_generation_fuel_nuclear') @@ -4118,7 +4118,7 @@ def downgrade() -> None: op.drop_table('core_eia860__scd_utilities') op.drop_table('core_eia860__scd_emissions_control_equipment') op.drop_table('_out_eia__monthly_heat_rate_by_unit') - op.drop_table('out_ferc714__hourly_predicted_state_demand') + op.drop_table('out_ferc714__hourly_estimated_state_demand') op.drop_table('out_eia861__compiled_geometry_utilities') op.drop_table('out_eia861__compiled_geometry_balancing_authorities') op.drop_table('core_pudl__entity_utilities_pudl') diff --git a/src/pudl/analysis/plant_parts_eia.py b/src/pudl/analysis/plant_parts_eia.py index d6225902a7..dc3269c997 100644 --- a/src/pudl/analysis/plant_parts_eia.py +++ b/src/pudl/analysis/plant_parts_eia.py @@ -367,6 +367,7 @@ @asset( + io_manager_key="pudl_sqlite_io_manager", compute_kind="Python", ) def out_eia__yearly_generators_by_ownership( diff --git a/src/pudl/analysis/service_territory.py b/src/pudl/analysis/service_territory.py index 70e0ba54ef..fcb642ccdd 100644 --- a/src/pudl/analysis/service_territory.py +++ b/src/pudl/analysis/service_territory.py @@ -424,7 +424,7 @@ def dagster_compile_geoms( out_eia__yearly_utilities: pd.DataFrame, core_eia861__yearly_service_territory: pd.DataFrame, core_eia861__assn_utility: pd.DataFrame, - core_censusdp1__entity_county: pd.DataFrame, + _core_censusdp1tract__counties: pd.DataFrame, ): """Compile all available utility or balancing authority geometries. @@ -442,7 +442,7 @@ def dagster_compile_geoms( out_eia__yearly_utilities=out_eia__yearly_utilities, core_eia861__yearly_service_territory=core_eia861__yearly_service_territory, core_eia861__assn_utility=core_eia861__assn_utility, - census_counties=core_censusdp1__entity_county, + census_counties=_core_censusdp1tract__counties, entity_type=entity_type, dissolve=dissolve, limit_by_state=limit_by_state, diff --git a/src/pudl/analysis/state_demand.py b/src/pudl/analysis/state_demand.py index 9e74cdc9cc..803cd690d5 100644 --- a/src/pudl/analysis/state_demand.py +++ b/src/pudl/analysis/state_demand.py @@ -264,12 +264,12 @@ def load_ventyx_hourly_state_demand(path: str) -> pd.DataFrame: }, ) def load_hourly_demand_matrix_ferc714( - core_ferc714__hourly_demand_pa: pd.DataFrame, + out_ferc714__hourly_planning_area_demand: pd.DataFrame, ) -> tuple[pd.DataFrame, pd.DataFrame]: """Read and format FERC 714 hourly demand into matrix form. Args: - core_ferc714__hourly_demand_pa: FERC 714 hourly demand time series by planning area. + out_ferc714__hourly_planning_area_demand: FERC 714 hourly demand time series by planning area. Returns: Hourly demand as a matrix with a `datetime` row index @@ -280,22 +280,22 @@ def load_hourly_demand_matrix_ferc714( of each `respondent_id_ferc714` and reporting `year` (int). """ # Convert UTC to local time (ignoring daylight savings) - core_ferc714__hourly_demand_pa["utc_offset"] = core_ferc714__hourly_demand_pa[ - "timezone" - ].map(STANDARD_UTC_OFFSETS) - core_ferc714__hourly_demand_pa["datetime"] = utc_to_local( - core_ferc714__hourly_demand_pa["utc_datetime"], - core_ferc714__hourly_demand_pa["utc_offset"], + out_ferc714__hourly_planning_area_demand[ + "utc_offset" + ] = out_ferc714__hourly_planning_area_demand["timezone"].map(STANDARD_UTC_OFFSETS) + out_ferc714__hourly_planning_area_demand["datetime"] = utc_to_local( + out_ferc714__hourly_planning_area_demand["utc_datetime"], + out_ferc714__hourly_planning_area_demand["utc_offset"], ) # Pivot to demand matrix: timestamps x respondents - matrix = core_ferc714__hourly_demand_pa.pivot( + matrix = out_ferc714__hourly_planning_area_demand.pivot( index="datetime", columns="respondent_id_ferc714", values="demand_mwh" ) # List timezone by year for each respondent - core_ferc714__hourly_demand_pa["year"] = core_ferc714__hourly_demand_pa[ - "report_date" - ].dt.year - utc_offset = core_ferc714__hourly_demand_pa.groupby( + out_ferc714__hourly_planning_area_demand[ + "year" + ] = out_ferc714__hourly_planning_area_demand["report_date"].dt.year + utc_offset = out_ferc714__hourly_planning_area_demand.groupby( ["respondent_id_ferc714", "year"], as_index=False )["utc_offset"].first() return matrix, utc_offset @@ -521,7 +521,7 @@ def county_assignments_ferc714( def census_counties( - core_censusdp1__entity_county: gpd.GeoDataFrame, + _core_censusdp1tract__counties: gpd.GeoDataFrame, ) -> gpd.GeoDataFrame: """Load county attributes. @@ -531,7 +531,7 @@ def census_counties( Returns: Dataframe with columns `county_id_fips` and `population`. """ - return core_censusdp1__entity_county[["geoid10", "dp0010001"]].rename( + return _core_censusdp1tract__counties[["geoid10", "dp0010001"]].rename( columns={"geoid10": "county_id_fips", "dp0010001": "population"} ) @@ -579,10 +579,10 @@ def total_state_sales_eia861( ), }, ) -def out_ferc714__hourly_predicted_state_demand( +def out_ferc714__hourly_estimated_state_demand( context, _out_ferc714__hourly_imputed_demand: pd.DataFrame, - core_censusdp1__entity_county: pd.DataFrame, + _core_censusdp1tract__counties: pd.DataFrame, out_ferc714__respondents_with_fips: pd.DataFrame, core_eia861__yearly_sales: pd.DataFrame = None, ) -> pd.DataFrame: @@ -591,7 +591,7 @@ def out_ferc714__hourly_predicted_state_demand( Args: _out_ferc714__hourly_imputed_demand: Hourly demand timeseries, with columns `respondent_id_ferc714`, report `year`, `utc_datetime`, and `demand_mwh`. - core_censusdp1__entity_county: The county layer of the Census DP1 shapefile. + _core_censusdp1tract__counties: The county layer of the Census DP1 shapefile. out_ferc714__respondents_with_fips: Annual respondents with the county FIPS IDs for their service territories. core_eia861__yearly_sales: EIA 861 sales data. If provided, the predicted hourly demand is @@ -608,7 +608,7 @@ def out_ferc714__hourly_predicted_state_demand( count_assign_ferc714 = county_assignments_ferc714( out_ferc714__respondents_with_fips ) - counties = census_counties(core_censusdp1__entity_county) + counties = census_counties(_core_censusdp1tract__counties) total_sales_eia861 = total_state_sales_eia861(core_eia861__yearly_sales) # Pre-compute list of respondent-years with demand diff --git a/src/pudl/convert/censusdp1tract_to_sqlite.py b/src/pudl/convert/censusdp1tract_to_sqlite.py index 12ae282386..375a6ba1d1 100644 --- a/src/pudl/convert/censusdp1tract_to_sqlite.py +++ b/src/pudl/convert/censusdp1tract_to_sqlite.py @@ -34,7 +34,7 @@ }, required_resource_keys={"datastore"}, ) -def censusdp1tract_to_sqlite(context): +def raw_censusdp1tract__all_tables(context): """Use GDAL's ogr2ogr utility to convert the Census DP1 GeoDB to an SQLite DB. The Census DP1 GeoDB is read from the datastore, where it is stored as a diff --git a/src/pudl/etl/__init__.py b/src/pudl/etl/__init__.py index 6118a55c9b..62c254d7fa 100644 --- a/src/pudl/etl/__init__.py +++ b/src/pudl/etl/__init__.py @@ -54,7 +54,7 @@ ), *load_assets_from_modules( [pudl.convert.censusdp1tract_to_sqlite, pudl.output.censusdp1tract], - group_name="core_censusdp1", + group_name="core_censusdp1tract", ), *load_assets_from_modules([glue_assets], group_name="core_assn"), *load_assets_from_modules([static_assets], group_name="core_codes"), diff --git a/src/pudl/etl/epacems_assets.py b/src/pudl/etl/epacems_assets.py index f71e7d47d2..fe13d9aa3e 100644 --- a/src/pudl/etl/epacems_assets.py +++ b/src/pudl/etl/epacems_assets.py @@ -163,7 +163,7 @@ def core_epacems__hourly_emissions( ), } ) -def emissions_unit_ids_epacems( +def _core_epacems__emissions_unit_ids( core_epacems__hourly_emissions: dd.DataFrame, ) -> pd.DataFrame: """Make unique annual plant_id_eia and emissions_unit_id_epa. diff --git a/src/pudl/etl/glue_assets.py b/src/pudl/etl/glue_assets.py index dbcd1953ea..7cb78f8767 100644 --- a/src/pudl/etl/glue_assets.py +++ b/src/pudl/etl/glue_assets.py @@ -266,7 +266,7 @@ def correct_epa_eia_plant_id_mapping(df: pd.DataFrame) -> pd.DataFrame: def core_epa__assn_eia_epacamd_subplant_ids( _core_epa__assn_eia_epacamd_unique: pd.DataFrame, core_eia860__scd_generators: pd.DataFrame, - emissions_unit_ids_epacems: pd.DataFrame, + _core_epacems__emissions_unit_ids: pd.DataFrame, core_eia860__assn_boiler_generator: pd.DataFrame, ) -> pd.DataFrame: """Groups units and generators into unique subplant groups. @@ -301,7 +301,7 @@ def core_epa__assn_eia_epacamd_subplant_ids( augement_crosswalk_with_generators_eia860( _core_epa__assn_eia_epacamd_unique, core_eia860__scd_generators ) - .pipe(augement_crosswalk_with_epacamd_ids, emissions_unit_ids_epacems) + .pipe(augement_crosswalk_with_epacamd_ids, _core_epacems__emissions_unit_ids) .pipe(augement_crosswalk_with_bga_eia860, core_eia860__assn_boiler_generator) ) # use graph analysis to identify subplants @@ -374,13 +374,13 @@ def augement_crosswalk_with_generators_eia860( def augement_crosswalk_with_epacamd_ids( - crosswalk_clean: pd.DataFrame, emissions_unit_ids_epacems: pd.DataFrame + crosswalk_clean: pd.DataFrame, _core_epacems__emissions_unit_ids: pd.DataFrame ) -> pd.DataFrame: """Merge all EPA CAMD IDs into the crosswalk.""" return crosswalk_clean.assign( emissions_unit_id_epa=lambda x: x.emissions_unit_id_epa.fillna(x.generator_id) ).merge( - emissions_unit_ids_epacems[ + _core_epacems__emissions_unit_ids[ ["plant_id_eia", "emissions_unit_id_epa"] ].drop_duplicates(), how="outer", diff --git a/src/pudl/extract/eia860.py b/src/pudl/extract/eia860.py index 8929ac73e3..4357982707 100644 --- a/src/pudl/extract/eia860.py +++ b/src/pudl/extract/eia860.py @@ -85,7 +85,7 @@ def get_dtypes(page, **partition): ) -eia860_raw_dfs = excel.raw_df_factory(Extractor, name="eia860") +raw_eia860__all_dfs = excel.raw_df_factory(Extractor, name="eia860") # TODO (bendnorman): Figure out type hint for context keyword and mutli_asset return @@ -93,7 +93,7 @@ def get_dtypes(page, **partition): outs={table_name: AssetOut() for table_name in sorted(raw_table_names)}, required_resource_keys={"datastore", "dataset_settings"}, ) -def extract_eia860(context, eia860_raw_dfs): +def extract_eia860(context, raw_eia860__all_dfs): """Extract raw EIA data from excel sheets into dataframes. Args: @@ -111,17 +111,18 @@ def extract_eia860(context, eia860_raw_dfs): eia860m_raw_dfs = pudl.extract.eia860m.Extractor(ds).extract( year_month=eia860m_date ) - eia860_raw_dfs = pudl.extract.eia860m.append_eia860m( - eia860_raw_dfs=eia860_raw_dfs, eia860m_raw_dfs=eia860m_raw_dfs + raw_eia860__all_dfs = pudl.extract.eia860m.append_eia860m( + eia860_raw_dfs=raw_eia860__all_dfs, eia860m_raw_dfs=eia860m_raw_dfs ) # create descriptive table_names - eia860_raw_dfs = { - "raw_eia860__" + table_name: df for table_name, df in eia860_raw_dfs.items() + raw_eia860__all_dfs = { + "raw_eia860__" + table_name: df + for table_name, df in raw_eia860__all_dfs.items() } - eia860_raw_dfs = dict(sorted(eia860_raw_dfs.items())) + raw_eia860__all_dfs = dict(sorted(raw_eia860__all_dfs.items())) return ( Output(output_name=table_name, value=df) - for table_name, df in eia860_raw_dfs.items() + for table_name, df in raw_eia860__all_dfs.items() ) diff --git a/src/pudl/extract/eia861.py b/src/pudl/extract/eia861.py index cc4ea32655..bff9b62d32 100644 --- a/src/pudl/extract/eia861.py +++ b/src/pudl/extract/eia861.py @@ -69,7 +69,7 @@ def get_dtypes(page, **partition): } -eia861_raw_dfs = excel.raw_df_factory(Extractor, name="eia861") +raw_eia861__all_dfs = excel.raw_df_factory(Extractor, name="eia861") @multi_asset( @@ -102,7 +102,7 @@ def get_dtypes(page, **partition): }, required_resource_keys={"datastore", "dataset_settings"}, ) -def extract_eia861(context, eia861_raw_dfs): +def extract_eia861(context, raw_eia861__all_dfs): """Extract raw EIA-861 data from Excel sheets into dataframes. Args: @@ -111,13 +111,13 @@ def extract_eia861(context, eia861_raw_dfs): Returns: A tuple of extracted EIA-861 dataframes. """ - eia861_raw_dfs = { + raw_eia861__all_dfs = { "raw_eia861__" + table_name.replace("_eia861", ""): df - for table_name, df in eia861_raw_dfs.items() + for table_name, df in raw_eia861__all_dfs.items() } - eia861_raw_dfs = dict(sorted(eia861_raw_dfs.items())) + raw_eia861__all_dfs = dict(sorted(raw_eia861__all_dfs.items())) return ( Output(output_name=table_name, value=df) - for table_name, df in eia861_raw_dfs.items() + for table_name, df in raw_eia861__all_dfs.items() ) diff --git a/src/pudl/extract/eia923.py b/src/pudl/extract/eia923.py index 9d4762abce..156a5743ed 100644 --- a/src/pudl/extract/eia923.py +++ b/src/pudl/extract/eia923.py @@ -104,7 +104,7 @@ def get_dtypes(page, **partition): ) -eia923_raw_dfs = excel.raw_df_factory(Extractor, name="eia923") +raw_eia923__all_dfs = excel.raw_df_factory(Extractor, name="eia923") # TODO (bendnorman): Figure out type hint for context keyword and mutli_asset return @@ -112,7 +112,7 @@ def get_dtypes(page, **partition): outs={table_name: AssetOut() for table_name in sorted(eia_raw_table_names)}, required_resource_keys={"datastore", "dataset_settings"}, ) -def extract_eia923(context, eia923_raw_dfs): +def extract_eia923(context, raw_eia923__all_dfs): """Extract raw EIA-923 data from excel sheets into dataframes. Args: @@ -122,13 +122,14 @@ def extract_eia923(context, eia923_raw_dfs): A tuple of extracted EIA dataframes. """ # create descriptive table_names - eia923_raw_dfs = { - "raw_eia923__" + table_name: df for table_name, df in eia923_raw_dfs.items() + raw_eia923__all_dfs = { + "raw_eia923__" + table_name: df + for table_name, df in raw_eia923__all_dfs.items() } - eia923_raw_dfs = dict(sorted(eia923_raw_dfs.items())) + raw_eia923__all_dfs = dict(sorted(raw_eia923__all_dfs.items())) return ( Output(output_name=table_name, value=df) - for table_name, df in eia923_raw_dfs.items() + for table_name, df in raw_eia923__all_dfs.items() ) diff --git a/src/pudl/extract/excel.py b/src/pudl/extract/excel.py index a0674e3a70..47874dbfd2 100644 --- a/src/pudl/extract/excel.py +++ b/src/pudl/extract/excel.py @@ -531,4 +531,4 @@ def raw_dfs() -> dict[str, pd.DataFrame]: # page in the spreadsheet based dataset using DynamicOut.collect() return concat_pages(dfs.collect()) - return graph_asset(name=f"{name}_raw_dfs")(raw_dfs) + return graph_asset(name=f"raw_{name}__all_dfs")(raw_dfs) diff --git a/src/pudl/extract/ferc1.py b/src/pudl/extract/ferc1.py index 9e04552afa..69fea2cb27 100644 --- a/src/pudl/extract/ferc1.py +++ b/src/pudl/extract/ferc1.py @@ -394,7 +394,9 @@ def create_raw_ferc1_assets() -> list[SourceAsset]: @asset -def raw_xbrl_metadata_json(context) -> dict[str, dict[str, list[dict[str, Any]]]]: +def raw_ferc1_xbrl__metadata_json( + context, +) -> dict[str, dict[str, list[dict[str, Any]]]]: """Extract the FERC 1 XBRL Taxonomy metadata we've stored as JSON. Returns: diff --git a/src/pudl/extract/phmsagas.py b/src/pudl/extract/phmsagas.py index 971ca22b83..511bcc59b7 100644 --- a/src/pudl/extract/phmsagas.py +++ b/src/pudl/extract/phmsagas.py @@ -74,7 +74,7 @@ def process_renamed(self, newdata: pd.DataFrame, page: str, **partition): "raw_phmsagas__yearly_transmission_gathering_pipe_miles_by_material", ) -phmsagas_raw_dfs = excel.raw_df_factory(Extractor, name="phmsagas") +raw_phmsagas__all_dfs = excel.raw_df_factory(Extractor, name="phmsagas") # # TODO (bendnorman): Figure out type hint for context keyword and multi_asset return @@ -82,7 +82,7 @@ def process_renamed(self, newdata: pd.DataFrame, page: str, **partition): outs={table_name: AssetOut() for table_name in sorted(raw_table_names)}, required_resource_keys={"datastore", "dataset_settings"}, ) -def extract_phmsagas(context, phmsagas_raw_dfs): +def extract_phmsagas(context, raw_phmsagas__all_dfs): """Extract raw PHMSA gas data from excel sheets into dataframes. Args: @@ -92,12 +92,13 @@ def extract_phmsagas(context, phmsagas_raw_dfs): A tuple of extracted PHMSA gas dataframes. """ # create descriptive table_names - phmsagas_raw_dfs = { - "raw_phmsagas__" + table_name: df for table_name, df in phmsagas_raw_dfs.items() + raw_phmsagas__all_dfs = { + "raw_phmsagas__" + table_name: df + for table_name, df in raw_phmsagas__all_dfs.items() } - phmsagas_raw_dfs = dict(sorted(phmsagas_raw_dfs.items())) + raw_phmsagas__all_dfs = dict(sorted(raw_phmsagas__all_dfs.items())) return ( Output(output_name=table_name, value=df) - for table_name, df in phmsagas_raw_dfs.items() + for table_name, df in raw_phmsagas__all_dfs.items() ) diff --git a/src/pudl/glue/ferc1_eia.py b/src/pudl/glue/ferc1_eia.py index 548d5dc459..4ae100f7f3 100644 --- a/src/pudl/glue/ferc1_eia.py +++ b/src/pudl/glue/ferc1_eia.py @@ -35,7 +35,7 @@ from dagster import AssetIn, Definitions, JobDefinition, asset, define_asset_job import pudl -from pudl.extract.ferc1 import raw_ferc1_assets, raw_xbrl_metadata_json +from pudl.extract.ferc1 import raw_ferc1_assets, raw_ferc1_xbrl__metadata_json from pudl.helpers import simplify_strings from pudl.io_managers import ferc1_dbf_sqlite_io_manager, ferc1_xbrl_sqlite_io_manager from pudl.metadata.fields import apply_pudl_dtypes @@ -44,7 +44,7 @@ from pudl.transform.ferc1 import ( Ferc1AbstractTableTransformer, TableIdFerc1, - clean_xbrl_metadata_json, + _core_ferc1_xbrl__metadata_json, ferc1_transform_asset_factory, ) from pudl.transform.params.ferc1 import FERC1_STRING_NORM @@ -313,7 +313,11 @@ def plants_ferc1_raw(**transformed_plant_tables): return Definitions( assets=transform_assets + raw_ferc1_assets - + [plants_ferc1_raw, raw_xbrl_metadata_json, clean_xbrl_metadata_json], + + [ + plants_ferc1_raw, + raw_ferc1_xbrl__metadata_json, + _core_ferc1_xbrl__metadata_json, + ], resources={ "ferc1_dbf_sqlite_io_manager": ferc1_dbf_sqlite_io_manager, "ferc1_xbrl_sqlite_io_manager": ferc1_xbrl_sqlite_io_manager, diff --git a/src/pudl/metadata/fields.py b/src/pudl/metadata/fields.py index bc7f22db8f..a26325c319 100644 --- a/src/pudl/metadata/fields.py +++ b/src/pudl/metadata/fields.py @@ -3046,7 +3046,7 @@ "opex_rents": {"description": "Rent expenses for the transmission line."}, "opex_total": {"description": "Overall expenses for the transmission line."}, }, - "core_ferc714__hourly_demand_pa": { + "out_ferc714__hourly_planning_area_demand": { "timezone": { "constraints": { "enum": [ diff --git a/src/pudl/metadata/resources/ferc714.py b/src/pudl/metadata/resources/ferc714.py index 76baba2466..99f6d0c9d8 100644 --- a/src/pudl/metadata/resources/ferc714.py +++ b/src/pudl/metadata/resources/ferc714.py @@ -18,7 +18,7 @@ "field_namespace": "ferc714", "etl_group": "ferc714", }, - "core_ferc714__hourly_demand_pa": { + "out_ferc714__hourly_planning_area_demand": { "description": ( "Hourly electricity demand by planning area. FERC Form 714, Part III, " "Schedule 2a." diff --git a/src/pudl/metadata/resources/pudl.py b/src/pudl/metadata/resources/pudl.py index 7ee3e73216..e0a0e14c68 100644 --- a/src/pudl/metadata/resources/pudl.py +++ b/src/pudl/metadata/resources/pudl.py @@ -82,7 +82,7 @@ "field_namespace": "pudl", "sources": ["pudl"], }, - "out_ferc714__hourly_predicted_state_demand": { + "out_ferc714__hourly_estimated_state_demand": { "title": "Estimated Hourly State Electricity Demand", "description": "Estimated hourly electricity demand for each state, scaled such that it matches the total electricity sales by state reported in EIA 861.", "schema": { diff --git a/src/pudl/output/censusdp1tract.py b/src/pudl/output/censusdp1tract.py index 8f1eeccd29..98c999da64 100644 --- a/src/pudl/output/censusdp1tract.py +++ b/src/pudl/output/censusdp1tract.py @@ -9,15 +9,23 @@ logger = pudl.logging_helpers.get_logger(__name__) +LAYER_NAMES: dict[str, str] = { + "state": "states", + "county": "counties", + "tract": "tracts", +} + def census_asset_factory(layer: str) -> AssetsDefinition: """An asset factory for finished EIA tables.""" @asset( - ins={"censusdp1tract_to_sqlite": AssetIn("censusdp1tract_to_sqlite")}, - name=f"core_censusdp1__entity_{layer}", + ins={ + "raw_censusdp1tract__all_tables": AssetIn("raw_censusdp1tract__all_tables") + }, + name=f"_core_censusdp1tract__{LAYER_NAMES[layer]}", ) - def census_layer(censusdp1tract_to_sqlite, **kwargs) -> gpd.GeoDataFrame: + def census_layer(raw_censusdp1tract__all_tables, **kwargs) -> gpd.GeoDataFrame: """Select one layer from the Census DP1 database. Uses information within the Census DP1 database to set the coordinate reference @@ -25,7 +33,7 @@ def census_layer(censusdp1tract_to_sqlite, **kwargs) -> gpd.GeoDataFrame: is renamed to "geom" as that's the default withing Geopandas. No other column names or types are altered. """ - census_conn = f"sqlite:///{censusdp1tract_to_sqlite}" + census_conn = f"sqlite:///{raw_censusdp1tract__all_tables}" dp1_engine = sa.create_engine(census_conn) def get_layer(layer, dp1_engine): @@ -74,6 +82,4 @@ def get_layer(layer, dp1_engine): return census_layer -census_dp1_layers = [ - census_asset_factory(layer) for layer in ["state", "county", "tract"] -] +census_dp1_layers = [census_asset_factory(layer) for layer in LAYER_NAMES] diff --git a/src/pudl/output/eia923.py b/src/pudl/output/eia923.py index 62a9ddf6b8..acdc3993bb 100644 --- a/src/pudl/output/eia923.py +++ b/src/pudl/output/eia923.py @@ -282,7 +282,7 @@ def out_eia923__fuel_receipts_costs( core_eia923__monthly_fuel_receipts_costs: pd.DataFrame, core_eia923__entity_coalmine: pd.DataFrame, _out_eia__plants_utilities: pd.DataFrame, - state_average_fuel_costs_eia: pd.DataFrame, + _out_eia__monthly_state_fuel_prices: pd.DataFrame, core_eia__entity_plants: pd.DataFrame, ) -> pd.DataFrame: """Denormalize the :ref:`core_eia923__monthly_fuel_receipts_costs` table.""" @@ -308,7 +308,7 @@ def out_eia923__fuel_receipts_costs( if context.op_config["fill"]: logger.info("filling in fuel cost NaNs") frc_df = _fill_fuel_costs_by_state( - frc_df, fuel_costs=state_average_fuel_costs_eia + frc_df, fuel_costs=_out_eia__monthly_state_fuel_prices ) # add the flag column to note that we didn't fill in with API data else: diff --git a/src/pudl/output/eia_bulk_elec.py b/src/pudl/output/eia_bulk_elec.py index 21fe311d03..d630436b78 100644 --- a/src/pudl/output/eia_bulk_elec.py +++ b/src/pudl/output/eia_bulk_elec.py @@ -8,7 +8,7 @@ @asset(io_manager_key=None, compute_kind="Python") -def state_average_fuel_costs_eia( +def _out_eia__monthly_state_fuel_prices( core_eia__yearly_fuel_receipts_costs_aggs: pd.DataFrame, ) -> pd.DataFrame: """Get state-level average fuel costs from EIA's bulk electricity data. diff --git a/src/pudl/output/ferc1.py b/src/pudl/output/ferc1.py index 0bd5594721..3e81e5f026 100644 --- a/src/pudl/output/ferc1.py +++ b/src/pudl/output/ferc1.py @@ -189,6 +189,31 @@ """ +def get_core_ferc1_asset_description(asset_name: str) -> str: + """Get the asset description portion of a core FERC FORM 1 asset. + + This is useful when programatically constructing output assets + from core assets using asset factories. + + Args: + asset_name: The name of the core asset. + + Returns: + asset_description: The asset description portion of the asset name. + """ + pattern = r"yearly_(.*?)_sched" + match = re.search(pattern, asset_name) + + if match: + asset_description = match.group(1) + else: + raise ValueError( + f"The asset description can not be parsed from {asset_name}" + "because it is not a valide core FERC Form 1 asset name." + ) + return asset_description + + @asset(io_manager_key="pudl_sqlite_io_manager", compute_kind="Python") def _out_ferc1__yearly_plants_utilities( core_pudl__assn_ferc1_pudl_plants: pd.DataFrame, @@ -1154,23 +1179,25 @@ class OffByFactoid(NamedTuple): @asset -def _out_ferc1__explosion_tags(table_dimensions_ferc1) -> pd.DataFrame: +def _out_ferc1__detailed_tags(_core_ferc1__table_dimensions) -> pd.DataFrame: """Grab the stored tables of tags and add inferred dimension.""" - rate_tags = _get_tags("xbrl_factoid_rate_base_tags.csv", table_dimensions_ferc1) + rate_tags = _get_tags( + "xbrl_factoid_rate_base_tags.csv", _core_ferc1__table_dimensions + ) rev_req_tags = _get_tags( - "xbrl_factoid_revenue_requirement_tags.csv", table_dimensions_ferc1 + "xbrl_factoid_revenue_requirement_tags.csv", _core_ferc1__table_dimensions ) rate_cats = _get_tags( - "xbrl_factoid_rate_base_category_tags.csv", table_dimensions_ferc1 + "xbrl_factoid_rate_base_category_tags.csv", _core_ferc1__table_dimensions ) plant_status_tags = _aggregatable_dimension_tags( - table_dimensions_ferc1, "plant_status" + _core_ferc1__table_dimensions, "plant_status" ) plant_function_tags = _aggregatable_dimension_tags( - table_dimensions_ferc1, "plant_function" + _core_ferc1__table_dimensions, "plant_function" ) utility_type_tags = _aggregatable_dimension_tags( - table_dimensions_ferc1, "utility_type" + _core_ferc1__table_dimensions, "utility_type" ) tag_dfs = [ rate_tags, @@ -1194,7 +1221,9 @@ def _out_ferc1__explosion_tags(table_dimensions_ferc1) -> pd.DataFrame: return tags_all -def _get_tags(file_name: str, table_dimensions_ferc1: pd.DataFrame) -> pd.DataFrame: +def _get_tags( + file_name: str, _core_ferc1__table_dimensions: pd.DataFrame +) -> pd.DataFrame: """Grab tags from a stored CSV file and apply :func:`make_calculation_dimensions_explicit`.""" tags_csv = importlib.resources.files("pudl.package_data.ferc1") / file_name tags_df = ( @@ -1204,7 +1233,7 @@ def _get_tags(file_name: str, table_dimensions_ferc1: pd.DataFrame) -> pd.DataFr .astype(pd.StringDtype()) .pipe( pudl.transform.ferc1.make_calculation_dimensions_explicit, - table_dimensions_ferc1, + _core_ferc1__table_dimensions, dimensions=["utility_type", "plant_function", "plant_status"], ) ) @@ -1212,12 +1241,12 @@ def _get_tags(file_name: str, table_dimensions_ferc1: pd.DataFrame) -> pd.DataFr def _aggregatable_dimension_tags( - table_dimensions_ferc1: pd.DataFrame, + _core_ferc1__table_dimensions: pd.DataFrame, dimension: Literal["plant_status", "plant_function"], ) -> pd.DataFrame: # make a new lil csv w the manually compiled plant status or dimension # add in the rest from the table_dims - # merge it into _out_ferc1__explosion_tags + # merge it into _out_ferc1__detailed_tags aggregatable_col = f"aggregatable_{dimension}" tags_csv = ( importlib.resources.files("pudl.package_data.ferc1") @@ -1231,17 +1260,17 @@ def _aggregatable_dimension_tags( .astype(pd.StringDtype()) .pipe( pudl.transform.ferc1.make_calculation_dimensions_explicit, - table_dimensions_ferc1, + _core_ferc1__table_dimensions, dimensions=dimensions, ) .set_index(idx) ) - table_dimensions_ferc1 = table_dimensions_ferc1.set_index(idx) + _core_ferc1__table_dimensions = _core_ferc1__table_dimensions.set_index(idx) tags_df = pd.concat( [ tags_df, - table_dimensions_ferc1.loc[ - table_dimensions_ferc1.index.difference(tags_df.index) + _core_ferc1__table_dimensions.loc[ + _core_ferc1__table_dimensions.index.difference(tags_df.index) ], ] ).reset_index() @@ -1259,37 +1288,43 @@ def exploded_table_asset_factory( ) -> AssetsDefinition: """Create an exploded table based on a set of related input tables.""" ins: Mapping[str, AssetIn] = { - "metadata_xbrl_ferc1": AssetIn("metadata_xbrl_ferc1"), - "calculation_components_xbrl_ferc1": AssetIn( - "calculation_components_xbrl_ferc1" + "_core_ferc1_xbrl__metadata": AssetIn("_core_ferc1_xbrl__metadata"), + "_core_ferc1_xbrl__calculation_components": AssetIn( + "_core_ferc1_xbrl__calculation_components" ), - "_out_ferc1__explosion_tags": AssetIn("_out_ferc1__explosion_tags"), + "_out_ferc1__detailed_tags": AssetIn("_out_ferc1__detailed_tags"), } ins |= {table_name: AssetIn(table_name) for table_name in table_names} - @asset(name=f"exploded_{root_table}", ins=ins, io_manager_key=io_manager_key) + @asset( + name=f"_out_ferc1__detailed_{get_core_ferc1_asset_description(root_table)}", + ins=ins, + io_manager_key=io_manager_key, + ) def exploded_tables_asset( **kwargs: dict[str, pd.DataFrame], ) -> pd.DataFrame: - metadata_xbrl_ferc1 = kwargs["metadata_xbrl_ferc1"] - calculation_components_xbrl_ferc1 = kwargs["calculation_components_xbrl_ferc1"] - tags = kwargs["_out_ferc1__explosion_tags"] + _core_ferc1_xbrl__metadata = kwargs["_core_ferc1_xbrl__metadata"] + _core_ferc1_xbrl__calculation_components = kwargs[ + "_core_ferc1_xbrl__calculation_components" + ] + tags = kwargs["_out_ferc1__detailed_tags"] tables_to_explode = { name: df for (name, df) in kwargs.items() if name not in [ - "metadata_xbrl_ferc1", - "calculation_components_xbrl_ferc1", - "_out_ferc1__explosion_tags", + "_core_ferc1_xbrl__metadata", + "_core_ferc1_xbrl__calculation_components", + "_out_ferc1__detailed_tags", "off_by_facts", ] } return Exploder( table_names=tables_to_explode.keys(), root_table=root_table, - metadata_xbrl_ferc1=metadata_xbrl_ferc1, - calculation_components_xbrl_ferc1=calculation_components_xbrl_ferc1, + metadata_xbrl_ferc1=_core_ferc1_xbrl__metadata, + calculation_components_xbrl_ferc1=_core_ferc1_xbrl__calculation_components, seed_nodes=seed_nodes, tags=tags, group_metric_checks=group_metric_checks, @@ -1308,7 +1343,7 @@ def create_exploded_table_assets() -> list[AssetsDefinition]: """ explosion_args = [ { - "root_table": "income_statement_ferc1", + "root_table": "core_ferc1__yearly_income_statements_sched114", "table_names": [ "core_ferc1__yearly_income_statements_sched114", "core_ferc1__yearly_depreciation_summary_sched336", @@ -1330,7 +1365,7 @@ def create_exploded_table_assets() -> list[AssetsDefinition]: "off_by_facts": [], }, { - "root_table": "balance_sheet_assets_ferc1", + "root_table": "core_ferc1__yearly_balance_sheet_assets_sched110", "table_names": [ "core_ferc1__yearly_balance_sheet_assets_sched110", "core_ferc1__yearly_utility_plant_summary_sched200", @@ -1389,7 +1424,7 @@ def create_exploded_table_assets() -> list[AssetsDefinition]: ], }, { - "root_table": "balance_sheet_liabilities_ferc1", + "root_table": "core_ferc1__yearly_balance_sheet_liabilities_sched110", "table_names": [ "core_ferc1__yearly_balance_sheet_liabilities_sched110", "core_ferc1__yearly_retained_earnings_sched118", diff --git a/src/pudl/output/ferc714.py b/src/pudl/output/ferc714.py index 426e38df5e..dc1029538d 100644 --- a/src/pudl/output/ferc714.py +++ b/src/pudl/output/ferc714.py @@ -557,7 +557,7 @@ def out_ferc714__respondents_with_fips( @asset(compute_kind="Python") def _out_ferc714__georeferenced_counties( out_ferc714__respondents_with_fips: pd.DataFrame, - core_censusdp1__entity_county: gpd.GeoDataFrame, + _core_censusdp1tract__counties: gpd.GeoDataFrame, ) -> gpd.GeoDataFrame: """Annual respondents with all associated county-level geometries. @@ -568,7 +568,8 @@ def _out_ferc714__georeferenced_counties( of the FIPS IDs so you can also still do ID based analyses. """ counties_gdf = pudl.analysis.service_territory.add_geometries( - out_ferc714__respondents_with_fips, census_gdf=core_censusdp1__entity_county + out_ferc714__respondents_with_fips, + census_gdf=_core_censusdp1tract__counties, ).pipe(apply_pudl_dtypes) return counties_gdf @@ -577,7 +578,7 @@ def _out_ferc714__georeferenced_counties( def _out_ferc714__georeferenced_respondents( out_ferc714__respondents_with_fips: pd.DataFrame, out_ferc714__summarized_demand: pd.DataFrame, - core_censusdp1__entity_county: gpd.GeoDataFrame, + _core_censusdp1tract__counties: gpd.GeoDataFrame, ) -> gpd.GeoDataFrame: """Annual respondents with a single all-encompassing geometry for each year. @@ -592,7 +593,7 @@ def _out_ferc714__georeferenced_respondents( respondents_gdf = ( pudl.analysis.service_territory.add_geometries( out_ferc714__respondents_with_fips, - census_gdf=core_censusdp1__entity_county, + census_gdf=_core_censusdp1tract__counties, dissolve=True, dissolve_by=["report_date", "respondent_id_ferc714"], ) @@ -609,7 +610,7 @@ def _out_ferc714__georeferenced_respondents( @asset(compute_kind="Python", io_manager_key="pudl_sqlite_io_manager") def out_ferc714__summarized_demand( _out_ferc714__annualized_respondents: pd.DataFrame, - core_ferc714__hourly_demand_pa: pd.DataFrame, + out_ferc714__hourly_planning_area_demand: pd.DataFrame, out_ferc714__respondents_with_fips: pd.DataFrame, _out_ferc714__categorized_respondents: pd.DataFrame, _out_ferc714__georeferenced_counties: gpd.GeoDataFrame, @@ -628,7 +629,7 @@ def out_ferc714__summarized_demand( demand_annual = ( pd.merge( _out_ferc714__annualized_respondents, - core_ferc714__hourly_demand_pa.loc[ + out_ferc714__hourly_planning_area_demand.loc[ :, ["report_date", "respondent_id_ferc714", "demand_mwh"] ], on=["report_date", "respondent_id_ferc714"], diff --git a/src/pudl/output/pudltabl.py b/src/pudl/output/pudltabl.py index 48df48b037..a30abf67aa 100644 --- a/src/pudl/output/pudltabl.py +++ b/src/pudl/output/pudltabl.py @@ -215,14 +215,14 @@ def _register_output_methods(self: Self): "out_eia923__AGG_generation_fuel_combined": "gf_eia923", # ferc714 "core_ferc714__respondent_id": "respondent_id_ferc714", - "core_ferc714__hourly_demand_pa": "demand_hourly_pa_ferc714", + "out_ferc714__hourly_planning_area_demand": "demand_hourly_pa_ferc714", "out_ferc714__respondents_with_fips": "fipsified_respondents_ferc714", "out_ferc714__summarized_demand": "summarized_demand_ferc714", # service territory "out_eia861__compiled_geometry_balancing_authorities": "compiled_geometry_balancing_authority_eia861", "out_eia861__compiled_geometry_utilities": "compiled_geometry_utility_eia861", # state demand - "out_ferc714__hourly_predicted_state_demand": "predicted_state_hourly_demand", + "out_ferc714__hourly_estimated_state_demand": "predicted_state_hourly_demand", # plant parts "out_eia__yearly_generators_by_ownership": "gens_mega_eia", "out_eia__yearly_plant_parts": "plant_parts_eia", diff --git a/src/pudl/transform/ferc1.py b/src/pudl/transform/ferc1.py index 41605af874..eb5b6663fd 100644 --- a/src/pudl/transform/ferc1.py +++ b/src/pudl/transform/ferc1.py @@ -43,18 +43,18 @@ @asset -def clean_xbrl_metadata_json( - raw_xbrl_metadata_json: dict[str, dict[str, list[dict[str, Any]]]], +def _core_ferc1_xbrl__metadata_json( + raw_ferc1_xbrl__metadata_json: dict[str, dict[str, list[dict[str, Any]]]], ) -> dict[str, dict[str, list[dict[str, Any]]]]: """Generate cleaned json xbrl metadata. For now, this only runs :func:`add_source_tables_to_xbrl_metadata`. """ - return add_source_tables_to_xbrl_metadata(raw_xbrl_metadata_json) + return add_source_tables_to_xbrl_metadata(raw_ferc1_xbrl__metadata_json) def add_source_tables_to_xbrl_metadata( - raw_xbrl_metadata_json: dict[str, dict[str, list[dict[str, Any]]]], + raw_ferc1_xbrl__metadata_json: dict[str, dict[str, list[dict[str, Any]]]], ) -> dict[str, dict[str, list[dict[str, Any]]]]: """Add a ``source_tables`` field into metadata calculation components. @@ -88,9 +88,9 @@ def label_source_tables(calc_component: dict, tables_to_fields: str) -> dict: logger.debug(f"Found no source table for {calc_component['name']}.") return calc_component - tables_to_fields = extract_tables_to_fields(raw_xbrl_metadata_json) + tables_to_fields = extract_tables_to_fields(raw_ferc1_xbrl__metadata_json) # for each table loop through all of the calculations within each field - for table_name, table_meta in raw_xbrl_metadata_json.items(): + for table_name, table_meta in raw_ferc1_xbrl__metadata_json.items(): for list_of_facts in table_meta.values(): for xbrl_fact in list_of_facts: # all facts have ``calculations``, but they are empty lists when null @@ -102,7 +102,7 @@ def label_source_tables(calc_component: dict, tables_to_fields: str) -> dict: ) else: calc_component["source_tables"] = [table_name] - return raw_xbrl_metadata_json + return raw_ferc1_xbrl__metadata_json ################################################################################ @@ -6039,7 +6039,7 @@ def ferc1_transform_asset_factory( f"raw_xbrl_duration__{tn}": AssetIn(f"raw_ferc1_xbrl__{tn}_duration") for tn in xbrl_tables } - ins["clean_xbrl_metadata_json"] = AssetIn("clean_xbrl_metadata_json") + ins["_core_ferc1_xbrl__metadata_json"] = AssetIn("_core_ferc1_xbrl__metadata_json") table_id = TableIdFerc1(table_name) @@ -6051,21 +6051,21 @@ def ferc1_transform_asset(**kwargs: dict[str, pd.DataFrame]) -> pd.DataFrame: raw_dbf: raw dbf table. raw_xbrl_instant: raw XBRL instant table. raw_xbrl_duration: raw XBRL duration table. - clean_xbrl_metadata_json: XBRL metadata json for all tables. + _core_ferc1_xbrl__metadata_json: XBRL metadata json for all tables. Returns: transformed FERC Form 1 table. """ # TODO: split the key by __, then groupby, then concatenate - clean_xbrl_metadata_json = kwargs["clean_xbrl_metadata_json"] + _core_ferc1_xbrl__metadata_json = kwargs["_core_ferc1_xbrl__metadata_json"] if generic: transformer = tfr_class( - xbrl_metadata_json=clean_xbrl_metadata_json[table_name], + xbrl_metadata_json=_core_ferc1_xbrl__metadata_json[table_name], table_id=table_id, ) else: transformer = tfr_class( - xbrl_metadata_json=clean_xbrl_metadata_json[table_name] + xbrl_metadata_json=_core_ferc1_xbrl__metadata_json[table_name] ) raw_dbf = pd.concat( @@ -6144,7 +6144,7 @@ def table_to_column_to_check() -> dict[str, list[str]]: if table_name != "plants_steam_ferc1" } ) -def table_dimensions_ferc1(**kwargs) -> pd.DataFrame: +def _core_ferc1__table_dimensions(**kwargs) -> pd.DataFrame: """Build a table of values of dimensions observed in the transformed data tables. Compile a dataframe indicating what distinct values are observed in the data for @@ -6181,19 +6181,19 @@ def table_dimensions_ferc1(**kwargs) -> pd.DataFrame: @asset( ins={ - "clean_xbrl_metadata_json": AssetIn("clean_xbrl_metadata_json"), - "table_dimensions_ferc1": AssetIn("table_dimensions_ferc1"), + "_core_ferc1_xbrl__metadata_json": AssetIn("_core_ferc1_xbrl__metadata_json"), + "_core_ferc1__table_dimensions": AssetIn("_core_ferc1__table_dimensions"), }, io_manager_key=None, # Change to sqlite_io_manager... ) -def metadata_xbrl_ferc1(**kwargs) -> pd.DataFrame: +def _core_ferc1_xbrl__metadata(**kwargs) -> pd.DataFrame: """Build a table of all of the tables' XBRL metadata.""" - clean_xbrl_metadata_json = kwargs["clean_xbrl_metadata_json"] - table_dimensions_ferc1 = kwargs["table_dimensions_ferc1"] + _core_ferc1_xbrl__metadata_json = kwargs["_core_ferc1_xbrl__metadata_json"] + _core_ferc1__table_dimensions = kwargs["_core_ferc1__table_dimensions"] tbl_metas = [] for table_name, trans in FERC1_TFR_CLASSES.items(): tbl_meta = ( - trans(xbrl_metadata_json=clean_xbrl_metadata_json[table_name]) + trans(xbrl_metadata_json=_core_ferc1_xbrl__metadata_json[table_name]) .xbrl_metadata[ [ "xbrl_factoid", @@ -6211,7 +6211,7 @@ def metadata_xbrl_ferc1(**kwargs) -> pd.DataFrame: .assign(**{dim: pd.NA for dim in dimensions}) .pipe( make_calculation_dimensions_explicit, - table_dimensions_ferc1=table_dimensions_ferc1, + table_dimensions_ferc1=_core_ferc1__table_dimensions, dimensions=dimensions, ) ) @@ -6220,22 +6220,22 @@ def metadata_xbrl_ferc1(**kwargs) -> pd.DataFrame: @asset( ins={ - "clean_xbrl_metadata_json": AssetIn("clean_xbrl_metadata_json"), - "table_dimensions_ferc1": AssetIn("table_dimensions_ferc1"), - "metadata_xbrl_ferc1": AssetIn("metadata_xbrl_ferc1"), + "_core_ferc1_xbrl__metadata_json": AssetIn("_core_ferc1_xbrl__metadata_json"), + "_core_ferc1__table_dimensions": AssetIn("_core_ferc1__table_dimensions"), + "_core_ferc1_xbrl__metadata": AssetIn("_core_ferc1_xbrl__metadata"), }, io_manager_key=None, # Change to sqlite_io_manager... ) -def calculation_components_xbrl_ferc1(**kwargs) -> pd.DataFrame: +def _core_ferc1_xbrl__calculation_components(**kwargs) -> pd.DataFrame: """Create calculation-component table from table-level metadata.""" - clean_xbrl_metadata_json = kwargs["clean_xbrl_metadata_json"] - table_dimensions_ferc1 = kwargs["table_dimensions_ferc1"] - metadata_xbrl_ferc1 = kwargs["metadata_xbrl_ferc1"] + _core_ferc1_xbrl__metadata_json = kwargs["_core_ferc1_xbrl__metadata_json"] + _core_ferc1__table_dimensions = kwargs["_core_ferc1__table_dimensions"] + _core_ferc1_xbrl__metadata = kwargs["_core_ferc1_xbrl__metadata"] # compile all of the calc comp tables. calc_metas = [] for table_name, transformer in FERC1_TFR_CLASSES.items(): calc_meta = transformer( - xbrl_metadata_json=clean_xbrl_metadata_json[table_name] + xbrl_metadata_json=_core_ferc1_xbrl__metadata_json[table_name] ).xbrl_calculations calc_metas.append(calc_meta) # squish all of the calc comp tables then add in the implicit table dimensions @@ -6245,18 +6245,18 @@ def calculation_components_xbrl_ferc1(**kwargs) -> pd.DataFrame: .astype({dim: pd.StringDtype() for dim in dimensions}) .pipe( make_calculation_dimensions_explicit, - table_dimensions_ferc1, + _core_ferc1__table_dimensions, dimensions=dimensions, ) .pipe( assign_parent_dimensions, - table_dimensions=table_dimensions_ferc1, + table_dimensions=_core_ferc1__table_dimensions, dimensions=dimensions, ) .pipe( infer_intra_factoid_totals, - meta_w_dims=metadata_xbrl_ferc1, - table_dimensions=table_dimensions_ferc1, + meta_w_dims=_core_ferc1_xbrl__metadata, + table_dimensions=_core_ferc1__table_dimensions, dimensions=dimensions, ) ) @@ -6293,7 +6293,7 @@ def check_calcs_vs_table( calcs=calc_components[ calc_components.table_name.isin(FERC1_TFR_CLASSES.keys()) ], - checked_table=metadata_xbrl_ferc1, + checked_table=_core_ferc1_xbrl__metadata, idx_calcs=calc_cols, idx_table=calc_cols, how="not_in", @@ -6303,17 +6303,17 @@ def check_calcs_vs_table( if not missing_calcs.empty: logger.warning( "Calculations found in calculation components table are missing from the " - "metadata_xbrl_ferc1 table." + "_core_ferc1_xbrl__metadata table." ) # which of these missing calculations actually show up in the transformed tables? # This handles dbf-only calculation components, whic are added to the - # metadata_xbrl_ferc1 table as part of each table's transformations but aren't - # observed (or therefore present in table_dimensions_ferc1) in the fast ETL or + # _core_ferc1_xbrl__metadata table as part of each table's transformations but aren't + # observed (or therefore present in _core_ferc1__table_dimensions) in the fast ETL or # in all subsets of years. We only want to flag calculation components as # missing when they're actually observed in the data. actually_missing_kids = check_calcs_vs_table( calcs=missing_calcs, - checked_table=table_dimensions_ferc1, + checked_table=_core_ferc1__table_dimensions, idx_calcs=child_cols, idx_table=child_cols, how="in", @@ -6433,7 +6433,7 @@ def make_calculation_dimensions_explicit( dimensions: list[str], parent: bool = False, ) -> pd.DataFrame: - """Fill in null dimensions w/ the values observed in :func:`table_dimensions_ferc1`. + """Fill in null dimensions w/ the values observed in :func:`_core_ferc1__table_dimensions`. In the raw XBRL metadata's calculations, there is an implicit assumption that calculated values are aggregated within categorical columns called Axes or @@ -6454,7 +6454,7 @@ def make_calculation_dimensions_explicit( This function uses the observed associations between ``table_name``, ``xbrl_factoid`` and the other dimension columns compiled by - :func:`table_dimensions_ferc1` to fill in missing (previously implied) dimension + :func:`_core_ferc1__table_dimensions` to fill in missing (previously implied) dimension values in the calculation components table. This is often a broadcast merge because many tables contain many values within these @@ -6724,18 +6724,18 @@ def infer_intra_factoid_totals( ] } | { - "calculation_components_xbrl_ferc1": AssetIn( - "calculation_components_xbrl_ferc1" + "_core_ferc1_xbrl__calculation_components": AssetIn( + "_core_ferc1_xbrl__calculation_components" ) }, ) def _core_ferc1__calculation_metric_checks(**kwargs): """Check calculation metrics for all transformed tables which have reconciled calcs.""" - calculation_components = kwargs["calculation_components_xbrl_ferc1"] + calculation_components = kwargs["_core_ferc1_xbrl__calculation_components"] transformed_ferc1_dfs = { name: df for (name, df) in kwargs.items() - if name not in ["calculation_components_xbrl_ferc1"] + if name not in ["_core_ferc1_xbrl__calculation_components"] } # standardize the two key columns we are going to use into generic names xbrl_factoid_name = table_to_xbrl_factoid_name() diff --git a/src/pudl/transform/ferc714.py b/src/pudl/transform/ferc714.py index 4051738905..706ef5fca9 100644 --- a/src/pudl/transform/ferc714.py +++ b/src/pudl/transform/ferc714.py @@ -234,7 +234,7 @@ "respondent_id": "respondent_id_ferc714", "respondent_name": "respondent_name_ferc714", }, - "core_ferc714__hourly_demand_pa": { + "out_ferc714__hourly_planning_area_demand": { "report_yr": "report_year", "plan_date": "report_date", "respondent_id": "respondent_id_ferc714", @@ -397,7 +397,7 @@ def core_ferc714__respondent_id( @asset(io_manager_key="pudl_sqlite_io_manager") -def core_ferc714__hourly_demand_pa( +def out_ferc714__hourly_planning_area_demand( raw_ferc714__demand_hourly_pa: pd.DataFrame, ) -> pd.DataFrame: """Transform the hourly demand time series by Planning Area. @@ -420,7 +420,8 @@ def core_ferc714__hourly_demand_pa( """ logger.info("Converting dates into pandas Datetime types.") df = _pre_process( - raw_ferc714__demand_hourly_pa, table_name="core_ferc714__hourly_demand_pa" + raw_ferc714__demand_hourly_pa, + table_name="out_ferc714__hourly_planning_area_demand", ) # Parse date strings @@ -520,5 +521,7 @@ def core_ferc714__hourly_demand_pa( "demand_mwh", ] df = df.drop(columns=set(df.columns) - set(columns)) - df = _post_process(df[columns], table_name="core_ferc714__hourly_demand_pa") + df = _post_process( + df[columns], table_name="out_ferc714__hourly_planning_area_demand" + ) return df diff --git a/test/conftest.py b/test/conftest.py index d2a8acbf14..3d5a9d7765 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -13,7 +13,7 @@ from pudl import resources from pudl.etl.cli import pudl_etl_job_factory -from pudl.extract.ferc1 import raw_xbrl_metadata_json +from pudl.extract.ferc1 import raw_ferc1_xbrl__metadata_json from pudl.ferc_to_sqlite.cli import ferc_to_sqlite_job_factory from pudl.io_managers import ( PudlSQLiteIOManager, @@ -272,10 +272,10 @@ def ferc1_xbrl_sql_engine( @pytest.fixture(scope="session", name="ferc1_xbrl_taxonomy_metadata") def ferc1_xbrl_taxonomy_metadata(ferc1_engine_xbrl: sa.Engine): """Read the FERC 1 XBRL taxonomy metadata from JSON.""" - result = materialize_to_memory([raw_xbrl_metadata_json]) + result = materialize_to_memory([raw_ferc1_xbrl__metadata_json]) assert result.success - return result.output_for_node("raw_xbrl_metadata_json") + return result.output_for_node("raw_ferc1_xbrl__metadata_json") @pytest.fixture(scope="session") diff --git a/test/integration/output_test.py b/test/integration/output_test.py index fcc204ca1c..97473c8418 100644 --- a/test/integration/output_test.py +++ b/test/integration/output_test.py @@ -190,7 +190,7 @@ def test_service_territory_outputs(pudl_engine, df_name): @pytest.mark.parametrize( "df_name", [ - "out_ferc714__hourly_predicted_state_demand", + "out_ferc714__hourly_estimated_state_demand", ], ) def test_state_demand_outputs(pudl_engine, df_name): diff --git a/test/unit/output/ferc1_test.py b/test/unit/output/ferc1_test.py index b7c02312a0..cec121a5db 100644 --- a/test/unit/output/ferc1_test.py +++ b/test/unit/output/ferc1_test.py @@ -22,6 +22,9 @@ import logging import pandas as pd +import pytest + +from pudl.output.ferc1 import get_core_ferc1_asset_description # from pudl.output.ferc1 import NodeId, XbrlCalculationForestFerc1 @@ -58,6 +61,19 @@ .set_index(EXPLODED_META_IDX) ) + +def test_get_core_ferc1_asset_description(): + valid_core_ferc1_asset_name = "core_ferc1__yearly_income_statements_sched114" + valid_core_ferc1_asset_name_result = get_core_ferc1_asset_description( + valid_core_ferc1_asset_name + ) + assert valid_core_ferc1_asset_name_result == "income_statements" + + invalid_core_ferc1_asset_name = "core_ferc1__income_statements" + with pytest.raises(ValueError): + get_core_ferc1_asset_description(invalid_core_ferc1_asset_name) + + # LEAF_NODE_1 = XbrlCalculationForestFerc1( # exploded_meta=TEST_EXPLODED_META, # seeds=[NodeId("table_1", "reported_1")], diff --git a/test/unit/transform/glue_test.py b/test/unit/transform/glue_test.py index 7ff18ef227..354a748bca 100644 --- a/test/unit/transform/glue_test.py +++ b/test/unit/transform/glue_test.py @@ -79,7 +79,7 @@ def test_epacamd_eia_subplant_ids(): epacamd_eia_subplant_ids_got = glue_assets.core_epa__assn_eia_epacamd_subplant_ids( _core_epa__assn_eia_epacamd_unique=epacamd_eia_test, core_eia860__scd_generators=generators_entity_eia_test, - emissions_unit_ids_epacems=emissions_unit_ids_epacems_test, + _core_epacems__emissions_unit_ids=emissions_unit_ids_epacems_test, core_eia860__assn_boiler_generator=boiler_generator_assn_eia860_test, )