Feature branch: Rename core + output assets to match new naming protocols

devtools/data-release.sh

@@ -57,7 +57,7 @@ gsutil -m cp "$GCS_URL/*.sqlite" "$GCS_URL/ferc*_xbrl_*.json" $RELEASE_DIR/pudl_
 
 echo "Copying Parquet datasets..."
 mkdir -p $RELEASE_DIR/pudl_data/parquet/epacems
-gsutil -m cp -r "$GCS_URL/hourly_emissions_epacems/*" $RELEASE_DIR/pudl_data/parquet/epacems
+gsutil -m cp -r "$GCS_URL/core_epacems__hourly_emissions/*" $RELEASE_DIR/pudl_data/parquet/epacems
 
 # Save the Docker image as a tarball so it can be archived with the data:
 echo "Saving Docker image: $DOCKER_IMAGE"

devtools/debug-eia-etl.ipynb

@@ -263,7 +263,7 @@
    "outputs": [],
    "source": [
     "%%time\n",
-    "asset_key = \"fuel_receipts_costs_eia923\"\n",
+    "asset_key = \"core_eia923__monthly_fuel_receipts_costs\"\n",
     "df = defs.load_asset_value(AssetKey(asset_key))\n",
     "\n",
     "df.head()"

devtools/debug-ferc1-etl.ipynb

@@ -99,7 +99,7 @@
    },
    "outputs": [],
    "source": [
-    "ferc1_xbrl_raw_dfs[\"fuel_ferc1\"][\"duration\"].report_year"
+    "ferc1_xbrl_raw_dfs[\"core_ferc1__yearly_fuel\"][\"duration\"].report_year"
    ]
   },
   {
@@ -206,7 +206,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "table_name = \"other_regulatory_liabilities_ferc1\"\n",
+    "table_name = \"core_ferc1__yearly_other_regulatory_liabilities\"\n",
     "TRANSFORMER = transformers[table_name]  # add a table here"
    ]
   },
@@ -326,8 +326,8 @@
    "source": [
     "transformed_tables = {}\n",
     "for table_name, transformer in transformers.items():\n",
-    "    if table_name == \"plants_steam_ferc1\":\n",
-    "        # plants_steam_ferc1 is a special case. It depends on the transformed fuel_ferc1 table.\n",
+    "    if table_name == \"core_ferc1__yearly_plants_steam\":\n",
+    "        # core_ferc1__yearly_plants_steam is a special case. It depends on the transformed core_ferc1__yearly_fuel table.\n",
     "        continue\n",
     "    transformed_tables[transformer.table_id.value] = transformer.transform(\n",
     "        raw_dbf=ferc1_dbf_raw_dfs[transformer.table_id.value],\n",
@@ -345,13 +345,13 @@
    },
    "outputs": [],
    "source": [
-    "# Handle special case for \"plants_steam_ferc1\"\n",
-    "transformer = transformers[\"plants_steam_ferc1\"]\n",
+    "# Handle special case for \"core_ferc1__yearly_plants_steam\"\n",
+    "transformer = transformers[\"core_ferc1__yearly_plants_steam\"]\n",
     "transformed_tables[transformer.table_id.value] = transformer.transform(\n",
     "    raw_dbf=ferc1_dbf_raw_dfs[transformer.table_id.value],\n",
     "    raw_xbrl_instant=ferc1_xbrl_raw_dfs[transformer.table_id.value][\"instant\"],\n",
     "    raw_xbrl_duration=ferc1_xbrl_raw_dfs[transformer.table_id.value][\"duration\"],\n",
-    "    transformed_fuel=transformed_tables[\"fuel_ferc1\"],\n",
+    "    transformed_fuel=transformed_tables[\"core_ferc1__yearly_fuel\"],\n",
     ")"
    ]
   }
@@ -372,7 +372,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.3"
+   "version": "3.11.5"
   }
  },
  "nbformat": 4,

devtools/inspect-assets.ipynb

@@ -50,10 +50,61 @@
     "\n",
     "from pudl.etl import defs\n",
     "\n",
-    "asset_key = \"raw_generator_existing_eia860\"\n",
+    "asset_key = \"exploded_balance_sheet_assets_ferc1\"\n",
     "df = defs.load_asset_value(AssetKey(asset_key))\n",
     "\n",
-    "df.head()"
+    "#df[df.row_type_xbrl == \"correction\"].xbrl_factoid.value_counts()\n",
+    "#df[(df.xbrl_factoid.isin([\"operation_expense\", \"maintenance_expense\"]))&(df.rel_diff.notnull())&(df.rel_diff!=0)].sort_values(['utility_id_ferc1', 'report_year', 'xbrl_factoid', 'rel_diff']).head(50)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b2d99594",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df[(df.xbrl_factoid==\"accumulated_depreciation\")&(df.plant_status==\"in_service\")&(df.plant_function==\"total\")]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "467111b1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df[df.xbrl_factoid.isin(factoids)&(df.utility_id_ferc1==9)&(df.report_year==1998)]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c6f7427a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "factoids = ['distribution_maintenance_expense_electric',\n",
+    " 'hydraulic_power_generation_maintenance_expense',\n",
+    " 'maintenance_of_general_plant',\n",
+    " 'nuclear_power_generation_maintenance_expense',\n",
+    " 'other_power_generation_maintenance_expense',\n",
+    " 'regional_market_maintenance_expense',\n",
+    " 'steam_power_generation_maintenance_expense',\n",
+    " 'transmission_maintenance_expense_electric']"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "951b718d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "asset_key = \"calculation_components_xbrl_ferc1\"\n",
+    "calcs = defs.load_asset_value(AssetKey(asset_key))\n",
+    "\n",
+    "calcs[(calcs.xbrl_factoid_parent == \"accumulated_depreciation\")].head(50)"
    ]
   },
   {
@@ -77,10 +128,24 @@
     "\n",
     "from pudl.etl import defs\n",
     "\n",
-    "asset_key = \"fuel_receipts_costs_eia923\"\n",
+    "asset_key = \"emissions_unit_ids_epacems\"\n",
     "df = defs.load_asset_value(AssetKey(asset_key))\n",
     "\n",
-    "df.head()"
+    "df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "9f0d118b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from pudl.output.epacems import epacems\n",
+    "\n",
+    "test_epacems = epacems(states = [\"ID\"], years = [2022])\n",
+    "\n",
+    "test_epacems[test_epacems.operating_datetime_utc>=\"2022-01-04\"].head(40)"
    ]
   }
  ],
@@ -100,7 +165,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.10"
+   "version": "3.11.5"
   }
  },
  "nbformat": 4,

devtools/python-output-table-conversion-debug.ipynb

@@ -109,7 +109,7 @@
     "@asset(io_manager_key=\"pudl_sqlite_io_manager\", compute_kind=\"Python\")\n",
     "def denorm_utilities_eia860(\n",
     "    utilities_entity_eia: pd.DataFrame,\n",
-    "    utilities_eia860: pd.DataFrame,\n",
+    "    core_eia860__scd_utilities: pd.DataFrame,\n",
     "    utilities_eia: pd.DataFrame,\n",
     "):\n",
     "    ... # joining logic\n",

docs/data_access.rst

@@ -83,7 +83,7 @@ AWS CLI, or programmatically via the S3 API. They can also be downloaded directl
 HTTPS using the following links:
 
 * `PUDL SQLite DB <https://s3.us-west-2.amazonaws.com/pudl.catalyst.coop/dev/pudl.sqlite>`__
-* `EPA CEMS Hourly Emissions Parquet (1995-2021) <https://s3.us-west-2.amazonaws.com/pudl.catalyst.coop/dev/hourly_emissions_epacems.parquet>`__
+* `EPA CEMS Hourly Emissions Parquet (1995-2021) <https://s3.us-west-2.amazonaws.com/pudl.catalyst.coop/dev/core_epacems__hourly_emissions.parquet>`__
 * `Census DP1 SQLite DB (2010) <https://s3.us-west-2.amazonaws.com/pudl.catalyst.coop/dev/censusdp1tract.sqlite>`__
 
 * Raw FERC Form 1:

docs/dev/annual_updates.rst

@@ -338,10 +338,10 @@ A. FERC 1 & EIA Plants & Utilities
 B. Missing EIA Plant Locations from CEMS
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 **6.B.1)** If there are any plants that appear in the EPA CEMS dataset that do not
-appear in the ``plants_entity_eia`` table, or that are missing latitude and longitude
-values, you'll get a warning when you try and materialize the ``epacamd`` asset group in
-Dagster. You'll need to manually compile the missing information and add it to
-``src/pudl/package_data/epacems/additional_epacems_plants.csv`` to enable accurate
+appear in the ``core_eia__entity_plants`` table, or that are missing latitude and
+longitude values, you'll get a warning when you try and materialize the ``core_epacamd``
+asset group in Dagster. You'll need to manually compile the missing information and add
+it to ``src/pudl/package_data/epacems/additional_epacems_plants.csv`` to enable accurate
 adjustment of the EPA CEMS timestamps to UTC. Using the Plant ID from the warning, look
 up the plant coordinates in the
 `EPA FACT API <https://www.epa.gov/airmarkets/field-audit-checklist-tool-fact-api>`__.

docs/dev/dev_setup.rst

@@ -266,7 +266,7 @@ store the path to this new directory:
     $ set -Ux PUDL_OUTPUT /absolute/path/to/pudl_output # fish
 
 The path stored in ``PUDL_OUTPUT`` contains all ETL outputs like
-``pudl.sqlite`` and ``hourly_emissions_epacems.parquet``.
+``pudl.sqlite`` and ``core_epacems__hourly_emissions.parquet``.
 
 **Make sure you create separate directories for these environment variables!
 It is recommended you create these directories outside of the pudl repository

docs/dev/naming_conventions.rst

@@ -19,7 +19,7 @@ as we come across them again in maintaining the code.
 * When dataframe outputs are built from multiple tables, identify the type of
   information being pulled (e.g. "plants") and the source of the tables (e.g.
   ``eia`` or ``ferc1``). When outputs are built from a single table, simply use
-  the table name (e.g. ``boiler_fuel_eia923``).
+  the table name (e.g. ``core_eia923__monthly_boiler_fuel``).
 
 .. _glossary:
 
@@ -89,7 +89,7 @@ Output Functions
 When dataframe outputs are built from multiple tables, identify the type of
 information being pulled (e.g. ``plants``) and the source of the tables (e.g.
 ``eia`` or ``ferc1``). When outputs are built from a single table, simply use
-the table name (e.g. ``boiler_fuel_eia923``).
+the table name (e.g. ``core_eia923__monthly_boiler_fuel``).
 
 Table Names
 -----------

docs/dev/run_the_etl.rst

@@ -490,7 +490,7 @@ The commands above should result in a bunch of Python :mod:`logging` output
 describing what the script is doing, and file outputs in the ``output``
 directory within your workspace. When the ETL is complete, you
 should see new files at ``output/ferc1.sqlite`` and ``output/pudl.sqlite`` as
-well as a new directory at ``output/hourly_emissions_epacems`` containing
+well as a new directory at ``output/core_epacems__hourly_emissions`` containing
 nested directories named by year and state.
 
 If you need to re-run ``ferc_to_sqlite`` and want to overwrite

docs/intro.rst

@@ -168,10 +168,10 @@ Denormalized Outputs
 We normalize the data to make storage more efficient and avoid data integrity issues,
 but you may want to combine information from more than one of the tables to make the
 data more readable and readily interpretable. For example, PUDL stores the name that EIA
-uses to refer to a power plant in the :ref:`plants_entity_eia` table in association with
-the plant's unique numeric ID. If you are working with data from the
-:ref:`fuel_receipts_costs_eia923` table, which records monthly per-plant fuel
-deliveries, you may want to have the name of the plant alongside the fuel delivery
+uses to refer to a power plant in the :ref:`core_eia__entity_plants` table in
+association with the plant's unique numeric ID. If you are working with data from the
+:ref:`core_eia923__monthly_fuel_receipts_costs` table, which records monthly per-plant
+fuel deliveries, you may want to have the name of the plant alongside the fuel delivery
 information since it's more recognizable than the plant ID.
 
 Rather than requiring everyone to write their own SQL ``SELECT`` and ``JOIN`` statements