Skip to content

Commit

Permalink
Fix row count asset check for vcerare (#3993)
Browse files Browse the repository at this point in the history
* Fix row count asset check for vcerare

* Fix typo

Co-authored-by: Dazhong Xia <[email protected]>

* Fix typo

Co-authored-by: Dazhong Xia <[email protected]>

* [pre-commit.ci] auto fixes from pre-commit.com hooks

For more information, see https://pre-commit.ci

---------

Co-authored-by: Dazhong Xia <[email protected]>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
  • Loading branch information
3 people authored Dec 19, 2024
1 parent 0dd0530 commit d6e385c
Showing 1 changed file with 11 additions and 3 deletions.
14 changes: 11 additions & 3 deletions src/pudl/transform/vcerare.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import pyarrow as pa
import pyarrow.parquet as pq
from dagster import (
AssetCheckExecutionContext,
AssetCheckResult,
asset,
asset_check,
Expand Down Expand Up @@ -359,16 +360,23 @@ def _load_duckdb_table():
blocking=True,
description="Check that row count matches expected.",
)
def check_rows() -> AssetCheckResult:
def check_rows(context: AssetCheckExecutionContext) -> AssetCheckResult:
"""Check rows."""
logger.info("Check VCE RARE hourly table is the expected length")

# Define row counts for fast/full etl
row_counts = {
"etl_full": 136437000,
"etl_fast": 27287400,
}

vce = _load_duckdb_table() # noqa: F841
(length,) = duckdb.query("SELECT COUNT(*) FROM vce").fetchone()
if length != 136437000:
if (expected_length := row_counts[context.op_execution_context.job_name]) != length:
return AssetCheckResult(
passed=False,
description="Table unexpected length",
metadata={"table_length": length, "expected_length": 136437000},
metadata={"table_length": length, "expected_length": expected_length},
)
return AssetCheckResult(passed=True)

Expand Down

0 comments on commit d6e385c

Please sign in to comment.