diff --git a/.codecov.yml b/.codecov.yml index 36ee70c9fd..9c178ea976 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -1,9 +1,12 @@ --- coverage: range: 70..100 - round: down + round: nearest precision: 1 +ignore: + - "src/pudl/validate.py" + codecov: token: 23a7ee04-6ac5-4d1b-9d36-86b0c50d40c5 require_ci_to_pass: true diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index 6bf97eb778..0000000000 --- a/.coveragerc +++ /dev/null @@ -1,22 +0,0 @@ -[run] -omit = - *__main__.py - *__init__.py - *_test.py - -[report] -precision = 2 -exclude_lines = - # Have to re-enable the standard pragma - pragma: no cover - - # Don't complain if tests don't hit defensive assertion code: - raise AssertionError - raise NotImplementedError - - # Don't complain if non-runnable code isn't run: - if 0: - if __name__ == .__main__.: - - # Stuff that's not expected to run normally... - logger.debug diff --git a/.gitattributes b/.gitattributes index c61cc555de..1fc6acc943 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,7 +1,7 @@ *.ipynb linguist-detectable=false *.html linguist-detectable=false eia861-transform.ipynb merge=ours -environments/conda-*lock.yml merge=ours +environments/conda-*lock.yml merge=ours linguist-generated=true *.csv text *.py text *.json text diff --git a/.github/ISSUE_TEMPLATE/annual_updates.md b/.github/ISSUE_TEMPLATE/annual_updates.md new file mode 100644 index 0000000000..c5e3bbbc79 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/annual_updates.md @@ -0,0 +1,24 @@ +--- +name: Integrate New Year of Data +about: Check-list for integrating a new year of data +title: '' +labels: new-data +assignees: '' + +--- + +### New year of data integration check-list: + +Based on the [Annual Updates Docs](https://catalystcoop-pudl.readthedocs.io/en/dev/dev/annual_updates.html) + + +- [ ] [Obtain fresh data](https://catalystcoop-pudl.readthedocs.io/en/latest/dev/annual_updates.html#obtain-fresh-data) +- [ ] [Map the structure of the new data](https://catalystcoop-pudl.readthedocs.io/en/latest/dev/annual_updates.html#map-the-structure-of-the-new-data) +- [ ] [Test data extraction](https://catalystcoop-pudl.readthedocs.io/en/latest/dev/annual_updates.html#test-data-extraction) +- [ ] [Update table and column transformations](https://catalystcoop-pudl.readthedocs.io/en/latest/dev/annual_updates.html#update-table-column-transformations) +- [ ] [Update the PUDL db schema](https://catalystcoop-pudl.readthedocs.io/en/latest/dev/annual_updates.html#update-the-pudl-db-schema) +- [ ] [Connect datasets](https://catalystcoop-pudl.readthedocs.io/en/latest/dev/annual_updates.html#connect-datasets) +- [ ] [Run the ETL](https://catalystcoop-pudl.readthedocs.io/en/latest/dev/annual_updates.html#run-the-etl) +- [ ] [Update the output routines and run full tests](https://catalystcoop-pudl.readthedocs.io/en/latest/dev/annual_updates.html#update-the-output-routines-and-run-full-tests) +- [ ] [Run and update data validations](https://catalystcoop-pudl.readthedocs.io/en/latest/dev/annual_updates.html#run-and-update-data-validations) +- [ ] [Update the documentation](https://catalystcoop-pudl.readthedocs.io/en/latest/dev/annual_updates.html#update-the-documentation) diff --git a/.github/ISSUE_TEMPLATE/new_dataset.md b/.github/ISSUE_TEMPLATE/new_dataset.md new file mode 100644 index 0000000000..f66566062c --- /dev/null +++ b/.github/ISSUE_TEMPLATE/new_dataset.md @@ -0,0 +1,30 @@ +--- +name: New dataset +about: Provide information about a new dataset you'd like to see in PUDL +title: '' +labels: new-data +assignees: '' +--- + +### Overview + +What is this dataset? + +Why do you want it in PUDL? + +Is it already partially in PUDL, or do we need to start from scratch? + +### Logistics + +Is this dataset publically available? + +Where can one download the actual data? + +How often does this dataset get updated? + +What licensing restrictions apply? + +### What do you know about it so far? + +What have you done with this dataset so far? Have you run into any problems with +it yet? diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 325f1bcb8a..24e2d667aa 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,49 +1,25 @@ +# Overview -# PR Overview +Closes #XXXX. - +# Testing -# PR Checklist +How did you make sure this worked? How can a reviewer verify this? -- [ ] Merge the most recent version of the branch you are merging into (probably `dev`). -- [ ] All CI checks are passing. [Run tests locally to debug failures](https://catalystcoop-pudl.readthedocs.io/en/latest/dev/testing.html#running-tests-with-tox) -- [ ] Make sure you've included good docstrings. +```[tasklist] +# To-do list +- [ ] Make sure full ETL runs & `make pytest-integration-full` passes locally - [ ] For major data coverage & analysis changes, [run data validation tests](https://catalystcoop-pudl.readthedocs.io/en/latest/dev/testing.html#data-validation) -- [ ] Include unit tests for new functions and classes. -- [ ] Defensive data quality/sanity checks in analyses & data processing functions. -- [ ] Update the [release notes](https://catalystcoop-pudl.readthedocs.io/en/latest/release_notes.html) and reference reference the PR and related issues. -- [ ] Do your own explanatory review of the PR to help the reviewer understand what's going on and identify issues preemptively. +- [ ] If updating analyses or data processing functions: make sure to update or write data validation tests +- [ ] Update the [release notes](../docs/release_notes.rst): reference the PR and related issues. +- [ ] Review the PR yourself and call out any questions or issues you have +``` diff --git a/.github/workflows/build-deploy-pudl.yml b/.github/workflows/build-deploy-pudl.yml index 46549e2fee..eec4318ecf 100644 --- a/.github/workflows/build-deploy-pudl.yml +++ b/.github/workflows/build-deploy-pudl.yml @@ -12,6 +12,7 @@ env: GITHUB_REF: ${{ github.ref_name }} # This is changed to dev if running on a schedule GCE_INSTANCE: pudl-deployment-tag # This is changed to pudl-deployment-dev if running on a schedule GCE_INSTANCE_ZONE: ${{ secrets.GCE_INSTANCE_ZONE }} + GCS_OUTPUT_BUCKET: gs://nightly-build-outputs.catalyst.coop jobs: build_and_deploy_pudl: @@ -27,13 +28,14 @@ jobs: echo "This action was triggered by a schedule." && echo "GCE_INSTANCE=pudl-deployment-dev" >> $GITHUB_ENV && echo "GITHUB_REF=dev" >> $GITHUB_ENV - name: Checkout Repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ env.GITHUB_REF }} - name: Get HEAD of the branch (main or dev) run: | echo "ACTION_SHA=$(git rev-parse HEAD)" >> $GITHUB_ENV + echo "SHORT_SHA=$(git rev-parse --short HEAD)" >> $GITHUB_ENV - name: Print action vars run: | @@ -53,17 +55,17 @@ jobs: type=ref,event=tag - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2.5.0 + uses: docker/setup-buildx-action@v3.0.0 - name: Login to DockerHub if: github.event_name != 'pull_request' - uses: docker/login-action@v2.1.0 + uses: docker/login-action@v3.0.0 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Build image and push to Docker Hub - uses: docker/build-push-action@v4.0.0 + uses: docker/build-push-action@v5.1.0 with: context: . file: docker/Dockerfile @@ -74,7 +76,7 @@ jobs: cache-to: type=gha,mode=max - id: "auth" - uses: "google-github-actions/auth@v1" + uses: "google-github-actions/auth@v2" with: workload_identity_provider: "projects/345950277072/locations/global/workloadIdentityPools/gh-actions-pool/providers/gh-actions-provider" service_account: "deploy-pudl-github-action@catalyst-cooperative-pudl.iam.gserviceaccount.com" @@ -83,6 +85,11 @@ jobs: - name: Set up Cloud SDK uses: google-github-actions/setup-gcloud@v1 + - name: Determine commit information + run: |- + echo "COMMIT_BRANCH=$(gitrev-parse --abbrev-ref HEAD)" >> $GITHUB_ENV + echo "COMMIT_TIME=$(git log -1 --format=%cd --date=format:%Y-%m-%d-%H%M)" >> $GITHUB_ENV + # Deploy PUDL image to GCE - name: Deploy env: @@ -119,6 +126,7 @@ jobs: --container-env DAGSTER_PG_DB="dagster-storage" \ --container-env FLY_ACCESS_TOKEN=${{ secrets.FLY_ACCESS_TOKEN }} \ --container-env PUDL_SETTINGS_YML="/home/mambauser/src/pudl/package_data/settings/etl_full.yml" \ + --container-env PUDL_GCS_OUTPUT=${{ env.GCS_OUTPUT_BUCKET }}/${{ env.COMMIT_TIME }}-${{ env.SHORT_SHA }}-${{ env.COMMIT_BRANCH }} # Start the VM - name: Start the deploy-pudl-vm @@ -129,6 +137,7 @@ jobs: uses: slackapi/slack-github-action@v1.24.0 with: channel-id: "C03FHB9N0PQ" - slack-message: "build-deploy-pudl status: ${{ job.status }}\n${{ env.ACTION_SHA }}-${{ env.GITHUB_REF }}" + slack-message: "build-deploy-pudl status: ${{ job.status }}\n${{ env.COMMIT_TIME}}-${{ env.SHORT_SHA }}-${{ env.COMMIT_BRANCH }}" env: + channel-id: "C03FHB9N0PQ" SLACK_BOT_TOKEN: ${{ secrets.PUDL_DEPLOY_SLACK_TOKEN }} diff --git a/.github/workflows/docker-build-test.yml b/.github/workflows/docker-build-test.yml index efd3e393c2..d82ebf74cf 100644 --- a/.github/workflows/docker-build-test.yml +++ b/.github/workflows/docker-build-test.yml @@ -13,7 +13,7 @@ jobs: id-token: write steps: - name: Checkout Repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Docker Metadata id: docker_metadata @@ -24,10 +24,10 @@ jobs: latest=auto - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2.5.0 + uses: docker/setup-buildx-action@v3.0.0 - name: Build image but do not push to Docker Hub - uses: docker/build-push-action@v4.0.0 + uses: docker/build-push-action@v5.1.0 with: context: . file: docker/Dockerfile diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 79088f646a..0fb01335f8 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -164,7 +164,7 @@ jobs: - name: Set default GCP credentials id: gcloud-auth continue-on-error: true - uses: "google-github-actions/auth@v1" + uses: "google-github-actions/auth@v2" with: workload_identity_provider: "projects/345950277072/locations/global/workloadIdentityPools/gh-actions-pool/providers/gh-actions-provider" service_account: "tox-pytest-github-action@catalyst-cooperative-pudl.iam.gserviceaccount.com" diff --git a/.github/workflows/run-etl.yml b/.github/workflows/run-etl.yml index bb3ce372d6..d8876e940c 100644 --- a/.github/workflows/run-etl.yml +++ b/.github/workflows/run-etl.yml @@ -13,7 +13,7 @@ jobs: id-token: write steps: - name: Checkout Repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Docker Metadata id: docker_metadata uses: docker/metadata-action@v4.4.0 @@ -24,15 +24,15 @@ jobs: latest=auto tags: type=sha - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2.5.0 + uses: docker/setup-buildx-action@v3.0.0 - name: Login to DockerHub if: github.event_name != 'pull_request' - uses: docker/login-action@v2.1.0 + uses: docker/login-action@v3.0.0 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Build image and push to Docker Hub - uses: docker/build-push-action@v4.0.0 + uses: docker/build-push-action@v5.1.0 with: context: . file: docker/Dockerfile @@ -48,7 +48,7 @@ jobs: contents: read steps: - id: gcloud-auth - uses: google-github-actions/auth@v1 + uses: google-github-actions/auth@v2 with: workload_identity_provider: "projects/345950277072/locations/global/workloadIdentityPools/gh-actions-pool/providers/gh-actions-provider" diff --git a/.github/workflows/zenodo-cache-sync.yml b/.github/workflows/zenodo-cache-sync.yml index 22da3fa684..f52d74be78 100644 --- a/.github/workflows/zenodo-cache-sync.yml +++ b/.github/workflows/zenodo-cache-sync.yml @@ -63,7 +63,7 @@ jobs: - name: Set default gcp credentials id: gcloud-auth - uses: "google-github-actions/auth@v1" + uses: "google-github-actions/auth@v2" with: workload_identity_provider: "projects/345950277072/locations/global/workloadIdentityPools/gh-actions-pool/providers/gh-actions-provider" service_account: "zenodo-cache-manager@catalyst-cooperative-pudl.iam.gserviceaccount.com" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 614d22aaea..3bc8081313 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -80,7 +80,7 @@ repos: verbose: false pass_filenames: false always_run: true - entry: pytest --doctest-modules src/pudl test/unit + entry: pytest --doctest-modules src/pudl test/unit -m "not slow" # Configuration for pre-commit.ci ci: diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst new file mode 100644 index 0000000000..fc5e40359a --- /dev/null +++ b/CONTRIBUTING.rst @@ -0,0 +1,100 @@ +------------------------- +Contributing Code to PUDL +------------------------- + +Welcome! We're so glad you're interested in contributing to PUDL! We would love +some help making PUDL data as complete as possible. + +.. _after-intro: + +.. IMPORTANT:: Already have a dataset in mind? + + If you **need data that's not in PUDL**, `open an issue + `__ + to tell us more about it! + + If you've **already written some code to wrangle a dataset**, find us at + `office hours `__ and we + can talk through next steps. + + +Your first contribution +----------------------- + +**Setup** + +You'll need to fork this repository and get the +`dev environment set up `__. + +**Pick an issue** + +* Look for issues with the `good first issue + `__ + tag in our `Community Kanban Board + `__. These + are issues that don't require a ton of PUDL-specific context, and are + relatively tightly scoped. + +* Comment on the issue and tag ``@catalyst-cooperative/com-dev`` (our Community + Development Team) to let us know you're working on it. Feel free to ask any + questions you might have! + +* Once you have an idea of how you want to tackle this issue, write out your + plan so we can guide you around obstacles in your way! Post a comment outlining: + * what steps have you broken this down into? + * what is the output of each step? + * how will one know that each step is working? + +* Once you've talked through your plan with someone from Catalyst, go forth and + develop! + +**Work on it!** + +* Make a branch on your fork and open a draft pull request (PR) early so we can + discuss concrete code! **Set the base branch to ``dev`` unless there's a good + reason otherwise.** Please don't wait until it's all polished up - it's much + easier for us to help you when we can see the code evolve over time. + +* Please make sure to write tests and documentation for your code - if you run + into trouble with writing tests, let us know in the comments and we can help! + We automatically run the test suite for all PRs, but some of those will have + to be manually approved by Catalyst members for safety reasons. + +* **Try to keep your changes relatively small:** stuff happens, and one's + bandwidth for volunteer work can fluctuate frequently. If you make a bunch of + small changes, it's much easier to pause on a project without losing a ton of + context. We try to keep PRs to **less than 500 lines of code.** + +**Get it merged in!** + +* Turn the draft PR into a normal PR and tag ``@catalyst-cooperative/com-dev`` + in a comment. We'll try to get back to you within a few days - the + smaller/simpler the PR, the faster we'll be able to get back to you. + +* The reviewer will leave comments - if they request changes, address their + concerns and re-request review. + +* There will probably be some back-and-forth until your PR is approved - this + is normal and a sign of good communication on your part! Don't be shy about + asking us for updates and re-requesting review! + +* Don't accidentally "start a review" when responding to comments! If this does + happen, don't forget to submit the review you've started so the other PR + participants can see your comments (they are invisible to others if marked + "Pending"). + +Next contributions +------------------ + +Hooray! You made your first contribution! To find another issue to tackle, check +out the `Community Kanban board +`__ where +we've picked out some issues that are + +* useful to work on + +* unlikely to become super time-sensitive + +* have some context, success criteria, and next steps information. + +Pick one of these and follow the contribution flow above! diff --git a/Makefile b/Makefile index bd98c22fe8..9b952b956b 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,6 @@ -covargs := --append --source=src/pudl gcs_cache_path := --gcs-cache-path=gs://zenodo-cache.catalyst.coop -pytest_covargs := --cov-append --cov=src/pudl --cov-report=xml -coverage_report := coverage report --sort=cover -pytest_args := --durations 20 ${pytest_covargs} ${gcs_cache_path} +covargs := --append +pytest_args := --durations 20 ${gcs_cache_path} etl_fast_yml := src/pudl/package_data/settings/etl_fast.yml etl_full_yml := src/pudl/package_data/settings/etl_full.yml @@ -96,10 +94,7 @@ ferc: rm -f ${PUDL_OUTPUT}/ferc*.sqlite rm -f ${PUDL_OUTPUT}/ferc*_xbrl_datapackage.json rm -f ${PUDL_OUTPUT}/ferc*_xbrl_taxonomy_metadata.json - coverage run ${covargs} -- \ - src/pudl/ferc_to_sqlite/cli.py \ - ${gcs_cache_path} \ - ${etl_full_yml} + coverage run ${covargs} -- src/pudl/ferc_to_sqlite/cli.py ${gcs_cache_path} ${etl_full_yml} # Remove the existing PUDL DB if it exists. # Create a new empty DB using alembic. @@ -108,7 +103,7 @@ ferc: pudl: rm -f ${PUDL_OUTPUT}/pudl.sqlite alembic upgrade head - coverage run ${covargs} -- src/pudl/cli/etl.py ${gcs_cache_path} ${etl_full_yml} + coverage run ${covargs} -- src/pudl/etl/cli.py ${gcs_cache_path} ${etl_full_yml} ######################################################################################## # Targets that are coordinated by pytest -- mostly they're actual tests. @@ -125,13 +120,13 @@ pytest-integration: coverage-erase: coverage erase -.PHONY: pytest-coverage -pytest-coverage: coverage-erase docs-build pytest-ci - ${coverage_report} - .PHONY: pytest-ci pytest-ci: pytest-unit pytest-integration +.PHONY: pytest-coverage +pytest-coverage: coverage-erase docs-build pytest-ci + coverage report + .PHONY: pytest-integration-full pytest-integration-full: pytest ${pytest_args} -n auto --live-dbs --etl-settings ${etl_full_yml} test/integration @@ -151,7 +146,7 @@ nuke: coverage-erase docs-build pytest-unit ferc pudl pudl_check_fks pytest ${pytest_args} -n auto --live-dbs --etl-settings ${etl_full_yml} test/integration pytest ${pytest_args} -n auto --live-dbs test/validate - ${coverage_report} + coverage report # Check that designated Jupyter notebooks can be run against the current DB .PHONY: pytest-jupyter diff --git a/devtools/data-release.sh b/devtools/data-release.sh deleted file mode 100755 index ac406af09b..0000000000 --- a/devtools/data-release.sh +++ /dev/null @@ -1,74 +0,0 @@ -#!/bin/sh -# A script to compile a Dockerized data release based on PUDL nightly build outputs. - -# Positional arguments: - -# First command line argument is the PUDL nightly build tag / ref. This indicates what -# build outputs to use. E.g. "dev" or "v2022.11.30" -PUDL_REF=$1 - -# Docker tag to use in the archive, e.g. "latest" or "2022.11.30". Will be used to -# pull the docker image using catalystcoop/pudl-jupyter:$DOCKER_TAG -DOCKER_TAG=$2 - -# Path to a local directory where the archive will be assembled. Should be in a place -# with at least 20GB of disk space. -# E.g. "./pudl-v2022.11.30" -RELEASE_DIR=$3 - -# Construct the GCS URL: -GCS_ROOT="gs://pudl.catalyst.coop" -GCS_URL="$GCS_ROOT/$PUDL_REF" - -# Construct the Docker image name -DOCKER_REPO="catalystcoop" -DOCKER_NAME="pudl-jupyter" -DOCKER_IMAGE="$DOCKER_REPO/$DOCKER_NAME:$DOCKER_TAG" - -echo "Started:" `date` -# Start with a clean slate: -rm -rf $RELEASE_DIR -mkdir -p $RELEASE_DIR -# The release container / environment is based on the pudl-examples repo: -git clone --depth 1 git@github.com:catalyst-cooperative/pudl-examples.git $RELEASE_DIR -rm -rf $RELEASE_DIR/.git* -# These directories are where the data will go. They're integrated with the -# Docker container that's defined in the pudl-examples repo: -mkdir -p $RELEASE_DIR/pudl_data -mkdir -p $RELEASE_DIR/user_data - -# Make sure we have the specified version of the Docker container: -docker pull $DOCKER_IMAGE -# Freeze the version of the Docker container: -cat $RELEASE_DIR/docker-compose.yml | sed -e "s/$DOCKER_NAME:latest/$DOCKER_NAME:$DOCKER_TAG/" > $RELEASE_DIR/new-docker-compose.yml -mv $RELEASE_DIR/new-docker-compose.yml $RELEASE_DIR/docker-compose.yml -# Set up a skeleton PUDL environment in the release dir: -pudl_setup $RELEASE_DIR/pudl_data - -# These are probably outdated now... see if they fail. -rm -rf $RELEASE_DIR/pudl_data/environment.yml -rm -rf $RELEASE_DIR/pudl_data/notebook -rm -rf $RELEASE_DIR/pudl_data/settings - -# Copy over all of the pre-processed data -echo "Copying SQLite databases..." -mkdir -p $RELEASE_DIR/pudl_data/sqlite/ -gsutil -m cp "$GCS_URL/*.sqlite" "$GCS_URL/ferc*_xbrl_*.json" $RELEASE_DIR/pudl_data/sqlite/ - -echo "Copying Parquet datasets..." -mkdir -p $RELEASE_DIR/pudl_data/parquet/epacems -gsutil -m cp -r "$GCS_URL/hourly_emissions_epacems/*" $RELEASE_DIR/pudl_data/parquet/epacems - -# Save the Docker image as a tarball so it can be archived with the data: -echo "Saving Docker image: $DOCKER_IMAGE" -docker save $DOCKER_IMAGE -o $RELEASE_DIR/pudl-jupyter.tar - -# List the high-level contents of the archive so we can see what it contains: -echo "Archive contents:" -find $RELEASE_DIR -maxdepth 3 - -# Create the archive -echo "Creating the archive tarball..." -tar -czf $RELEASE_DIR.tgz $RELEASE_DIR - -echo "Finished:" `date` diff --git a/devtools/datasette/fly/run.sh b/devtools/datasette/fly/run.sh index c17f3bdc86..71f0e98ca6 100755 --- a/devtools/datasette/fly/run.sh +++ b/devtools/datasette/fly/run.sh @@ -7,4 +7,4 @@ find /data/ -name '*.sqlite' -delete mv all_dbs.tar.zst /data zstd -f -d /data/all_dbs.tar.zst -o /data/all_dbs.tar tar -xf /data/all_dbs.tar --directory /data -datasette serve --host 0.0.0.0 /data/*.sqlite --cors --inspect-file inspect-data.json --metadata metadata.yml --setting sql_time_limit_ms 5000 --port $PORT +datasette serve --host 0.0.0.0 /data/pudl.sqlite /data/ferc*.sqlite /data/censusdp1tract.sqlite --cors --inspect-file inspect-data.json --metadata metadata.yml --setting sql_time_limit_ms 5000 --port $PORT diff --git a/devtools/sqlite_to_duckdb.py b/devtools/sqlite_to_duckdb.py old mode 100644 new mode 100755 index da49084829..f01389d1a6 --- a/devtools/sqlite_to_duckdb.py +++ b/devtools/sqlite_to_duckdb.py @@ -1,3 +1,4 @@ +#! /usr/bin/env python """A naive script for converting SQLite to DuckDB.""" import logging from pathlib import Path diff --git a/docker/Dockerfile b/docker/Dockerfile index 9247f98c78..26b291f4f2 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -48,10 +48,7 @@ ENV LD_LIBRARY_PATH=${CONDA_PREFIX}/lib # We need information from .git to get version with setuptools_scm so we mount that # directory without copying it into the image. RUN --mount=type=bind,source=.git,target=${PUDL_REPO}/.git \ - ${CONDA_RUN} pip install --no-cache-dir --no-deps --editable . && \ - # Run the PUDL setup script so we know where to read and write data - ${CONDA_RUN} pudl_setup - + ${CONDA_RUN} pip install --no-cache-dir --no-deps --editable . # Install awscli2 # Change back to root because the install script needs access to /usr/local/aws-cli diff --git a/docker/gcp_pudl_etl.sh b/docker/gcp_pudl_etl.sh index df44bcc060..57f259e692 100644 --- a/docker/gcp_pudl_etl.sh +++ b/docker/gcp_pudl_etl.sh @@ -2,6 +2,9 @@ # This script runs the entire ETL and validation tests in a docker container on a Google Compute Engine instance. # This script won't work locally because it needs adequate GCP permissions. +# Set PUDL_GCS_OUTPUT *only* if it is currently unset +: "${PUDL_GCS_OUTPUT:=gs://nightly-build-outputs.catalyst.coop/$ACTION_SHA-$GITHUB_REF}" + set -x function send_slack_msg() { @@ -22,31 +25,30 @@ function run_pudl_etl() { send_slack_msg ":large_yellow_circle: Deployment started for $ACTION_SHA-$GITHUB_REF :floppy_disk:" authenticate_gcp && \ alembic upgrade head && \ - pudl_setup && \ ferc_to_sqlite \ --loglevel DEBUG \ --gcs-cache-path gs://internal-zenodo-cache.catalyst.coop \ --workers 8 \ - $PUDL_SETTINGS_YML && \ - pudl_etl \ + $PUDL_SETTINGS_YML \ + && pudl_etl \ --loglevel DEBUG \ --gcs-cache-path gs://internal-zenodo-cache.catalyst.coop \ - $PUDL_SETTINGS_YML && \ - pytest \ + $PUDL_SETTINGS_YML \ + && pytest \ -n auto \ --gcs-cache-path gs://internal-zenodo-cache.catalyst.coop \ --etl-settings $PUDL_SETTINGS_YML \ - --live-dbs test/integration test/unit && \ - pytest \ + --live-dbs test/integration test/unit \ + && pytest \ -n auto \ --gcs-cache-path gs://internal-zenodo-cache.catalyst.coop \ --etl-settings $PUDL_SETTINGS_YML \ - --live-dbs test/validate + --live-dbs test/validate \ + && touch ${PUDL_OUTPUT}/success } function shutdown_vm() { # Copy the outputs to the GCS bucket - gsutil -m cp -r $PUDL_OUTPUT "gs://nightly-build-outputs.catalyst.coop/$ACTION_SHA-$GITHUB_REF" upload_file_to_slack $LOGFILE "pudl_etl logs for $ACTION_SHA-$GITHUB_REF:" @@ -59,6 +61,12 @@ function shutdown_vm() { curl -X POST -H "Content-Length: 0" -H "Authorization: Bearer ${ACCESS_TOKEN}" https://compute.googleapis.com/compute/v1/projects/catalyst-cooperative-pudl/zones/$GCE_INSTANCE_ZONE/instances/$GCE_INSTANCE/stop } +function copy_outputs_to_gcs() { + echo "Copying outputs to GCP bucket $PUDL_GCS_OUTPUT" + gsutil -m cp -r $PUDL_OUTPUT ${PUDL_GCS_OUTPUT} + rm ${PUDL_OUTPUT}/success +} + function copy_outputs_to_distribution_bucket() { echo "Copying outputs to GCP distribution bucket" gsutil -m -u $GCP_BILLING_PROJECT cp -r "$PUDL_OUTPUT/*" "gs://pudl.catalyst.coop/$GITHUB_REF" @@ -93,6 +101,8 @@ run_pudl_etl 2>&1 | tee $LOGFILE ETL_SUCCESS=${PIPESTATUS[0]} +copy_outputs_to_gcs + # if pipeline is successful, distribute + publish datasette if [[ $ETL_SUCCESS == 0 ]]; then # Deploy the updated data to datasette @@ -104,10 +114,14 @@ if [[ $ETL_SUCCESS == 0 ]]; then # Compress the SQLite DBs for easier distribution # Remove redundant multi-file EPA CEMS outputs prior to distribution gzip --verbose $PUDL_OUTPUT/*.sqlite && \ - rm -rf $PUDL_OUTPUT/hourly_emissions_epacems/ + rm -rf $PUDL_OUTPUT/hourly_emissions_epacems/ && \ + rm -f $PUDL_OUTPUT/metadata.yml ETL_SUCCESS=${PIPESTATUS[0]} # Dump outputs to s3 bucket if branch is dev or build was triggered by a tag + # TODO: this behavior should be controlled by on/off switch here and this logic + # should be moved to the triggering github action. Having it here feels + # fragmented. if [ $GITHUB_ACTION_TRIGGER = "push" ] || [ $GITHUB_REF = "dev" ]; then copy_outputs_to_distribution_bucket ETL_SUCCESS=${PIPESTATUS[0]} diff --git a/docs/CONTRIBUTING.rst b/docs/CONTRIBUTING.rst index 9e5bffc4c3..0581d1cbe2 100644 --- a/docs/CONTRIBUTING.rst +++ b/docs/CONTRIBUTING.rst @@ -2,111 +2,79 @@ Contributing to PUDL =============================================================================== + Welcome! We're excited that you're interested in contributing to the Public Utility -Data Liberation effort! The work is currently being coordinated by the members of the -`Catalyst Cooperative `__. PUDL is meant to serve a wide -variety of public interests including academic research, climate advocacy, data -journalism, and public policy making. This open source project has been supported by -a combination of volunteer contributions, grant funding from the `Alfred P. Sloan -Foundation `__, and reinvestment of net income from the -cooperative's client projects. +Data Liberation effort! + +We need lots of help with :ref:`user-feedback`, we welcome :ref:`code-contribs`, and +it would be great to :ref:`connect-orgs` that we can work with. + +Finally, `financial donations +`__ +are welcome too! + +--------------- +Code of Conduct +--------------- Please make sure you review our :doc:`code of conduct `, which is based on the `Contributor Covenant `__. We want to make the PUDL project welcoming to contributors with different levels of experience and diverse personal backgrounds. -------------------------------------------------------------------------------- -How to Get Involved -------------------------------------------------------------------------------- +.. _user-feedback: + +------------- +User feedback +------------- -We welcome just about any kind of contribution to the project. Alone, we'll never be -able to understand every use case or integrate all the available data. The project -will serve the community better if other folks get involved. +PUDL's goal is to help people use data to make change in the US energy landscape. +As such, it's critical that we understand our users' needs! `GitHub Discussions +`__ is our main forum +for all this. Since it's publicly readable, any conversation here can +potentially benefit other users too! -There are lots of ways to contribute -- it's not all about code! +We'd love it if you could: -* If you need help, someone else might need it too - ask for help in `Github - Discussions +* Tell us what problems you're running into, in the `Help Me! `__ - and maybe the ensuing discussion will be useful to other people too! -* `Suggest new data and features `__ that would be useful. + discussion board +* Tell us about what data you're looking for by opening an `issue + `__ +* Tell us what you're trying to do with PUDL data in `this thread + `__ * `File bug reports `__ on Github. -* Help expand and improve the documentation, or create new - `example notebooks `__ -* Help us create more and better software :doc:`test cases `. -* Give us feedback on overall usability using `GitHub Discussions +* Tell us what you'd like to see in PUDL in the `Ideas `__ - -- what's confusing? -* Tell us a story about how you're using of the data. -* Point us at interesting publications related to open energy data, open source energy - system modeling, how energy policy can be affected by better data, or open source - tools we should check out. -* Cite PUDL using - `DOIs from Zenodo `__ - if you use the software or data in your own published work. + discussion board + +.. _code-contribs: + +-------------------- +Code contributions +-------------------- + +.. include:: ../CONTRIBUTING.rst + :start-after: after-intro: + +.. _connect-orgs: + +----------------------------------- +Connect us with other organizations +----------------------------------- + +For PUDL to make a bigger impact, we need to find more people who need the data. +Here's how you can help: + +* Cite PUDL using `DOIs from Zenodo + `__ if you use the + software or data in your own published work. * Point us toward appropriate grant funding opportunities and meetings where we might present our work. +* Point us at interesting publications related to open energy data, open source + energy system modeling, how energy policy can be affected by better data, or + open source tools we should check out. * Share your Jupyter notebooks and other analyses that use PUDL. * `Hire Catalyst `__ to do analysis for your organization using the PUDL data -- contract work helps us self-fund ongoing open source development. -* Contribute code via - `pull requests `__. - See the :doc:`developer setup ` for more details. -* And of course... we also appreciate - `financial contributions `__. - -.. seealso:: - - * :doc:`dev/dev_setup` for instructions on how to set up the PUDL - development environment. - -------------------------------------------------------------------------------- -Find us on GitHub -------------------------------------------------------------------------------- -Github is the primary platform we use to manage the project, integrate -contributions, write and publish documentation, answer user questions, automate -testing & deployment, etc. -`Signing up for a GitHub account `__ -(even if you don't intend to write code) will allow you to participate in -online discussions and track projects that you're interested in. - -Asking (and answering) questions is a valuable contribution! As noted in `How to -support open-source software and stay sane -`__, it's much more efficient to -ask and answer questions in a public forum because then other users and contributors -who are having the same problem can find answers without having to re-ask the same -question. The forum we're using is our `Github discussions -`__. - -Even if you feel like you have a basic question, we want you to feel -comfortable asking for help in public -- we (Catalyst) only recently came to -this data work from being activists and policy wonks -- so it's easy for us to -remember when it all seemed frustrating and alien! Sometimes it still does. We -want people to use the software and data to do good things in the world. We -want you to be able to access it. Using a public forum also enables the -community of users to help each other! - -Don't hesitate to post a discussion with a `feature request -`__, -a pointer to energy data that needs liberating, or a reference to documentation -that's out of date, unclear, or missing. Understanding how people are using the -software, and how they would *like* to be using the software, is very valuable and -will help us make it more useful and usable. - -------------------------------------------------------------------------------- -Our design process -------------------------------------------------------------------------------- - -We do our technical design out in the open, so that community members can weigh -in. Here's the process we usually follow: - -1. Someone has a problem they'd like to solve. They post in the `Ideas - `__ - forum with their problem and some context. - -2. Discussion ensues. - -3. When the open questions are answered, we create an issue from the discussion, - which holds the conclusions of the discussion. diff --git a/docs/Makefile b/docs/Makefile deleted file mode 100644 index d4bb2cbb9e..0000000000 --- a/docs/Makefile +++ /dev/null @@ -1,20 +0,0 @@ -# Minimal makefile for Sphinx documentation -# - -# You can set these variables from the command line, and also -# from the environment for the first two. -SPHINXOPTS ?= -SPHINXBUILD ?= sphinx-build -SOURCEDIR = . -BUILDDIR = _build - -# Put it first so that "make" without argument is like "make help". -help: - @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - -.PHONY: help Makefile - -# Catch-all target: route all unknown targets to Sphinx using the new -# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). -%: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/conf.py b/docs/conf.py index 6721aec6c8..6935f09761 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -137,7 +137,7 @@ def data_dictionary_metadata_to_rst(app): """Export data dictionary metadata to RST for inclusion in the documentation.""" # Create an RST Data Dictionary for the PUDL DB: print("Exporting PUDL DB data dictionary metadata to RST.") - skip_names = ["datasets", "accumulated_depreciation_ferc1"] + skip_names = ["datasets", "accumulated_depreciation_ferc1", "entity_types_eia"] names = [name for name in RESOURCE_METADATA if name not in skip_names] package = Package.from_resource_ids(resource_ids=tuple(sorted(names))) # Sort fields within each resource by name: diff --git a/docs/dev/annual_updates.rst b/docs/dev/annual_updates.rst index bc076d4d7b..b7a8e1f6e4 100644 --- a/docs/dev/annual_updates.rst +++ b/docs/dev/annual_updates.rst @@ -45,11 +45,7 @@ fields such as ``source_format`` or ``path`` are still accurate. ``etl_fast.yml`` settings files stored under ``src/pudl/package_data/settings`` in the PUDL repo. -**1.5)** Update the settings files in your PUDL workspace to reflect the new -years by running ``pudl_setup {path to your pudl_work directory} -c``. Don't worry, it -won't remove any custom settings files you've added under a diffrent name. - -**1.6)** Use the ``pudl_datastore`` script (see :doc:`datastore`) to download the new +**1.5)** Use the ``pudl_datastore`` script (see :doc:`datastore`) to download the new raw data archives in bulk so that network hiccups don't cause issues during the ETL. 2. Map the Structure of the New Data diff --git a/docs/dev/dev_setup.rst b/docs/dev/dev_setup.rst index e94ad413da..de7ec65b6a 100644 --- a/docs/dev/dev_setup.rst +++ b/docs/dev/dev_setup.rst @@ -239,20 +239,13 @@ Creating a Workspace PUDL Workspace Setup ^^^^^^^^^^^^^^^^^^^^ -.. note:: - - If you used ``pudl_setup`` to set up your pudl workspace already, - skip ahead to :ref:`Legacy PUDL Setup`. If you haven't setup - a PUDL workspace before, read the remainder of this section. - -PUDL needs to know where to store its big piles of inputs and outputs. -The ``PUDL_OUTPUT`` and ``PUDL_INPUT`` environment variables let PUDL know where -all this stuff should go. We call this a "PUDL workspace". +PUDL needs to know where to store its big piles of inputs and outputs. The +``PUDL_OUTPUT`` and ``PUDL_INPUT`` environment variables let PUDL know where all this +stuff should go. We call this a "PUDL workspace". -First, create a directory to store local caches of raw PUDL data. You can put -this anywhere, but we put this in ``~/pudl_input`` in the documentation. -Then create an environment variable called ``PUDL_INPUT`` to store the path to -this new directory: +First, create a directory to store local caches of raw PUDL data. You can put this +anywhere, but we put this in ``~/pudl_input`` in the documentation. Then create an +environment variable called ``PUDL_INPUT`` to store the path to this new directory: .. code-block:: console @@ -260,8 +253,8 @@ this new directory: $ echo "export PUDL_INPUT=/absolute/path/to/pudl_input" >> ~/.bashrc # if you are using bash $ set -Ux PUDL_INPUT /absolute/path/to/pudl_input # if you are using fish shell -The directory stored in ``PUDL_INPUT`` contains versions of PUDL's -raw data archives on Zenodo for each datasource: +The directory stored in ``PUDL_INPUT`` contains versions of PUDL's raw data archives on +Zenodo for each datasource: .. code-block:: @@ -281,16 +274,15 @@ raw data archives on Zenodo for each datasource: .. warning:: - The data stored at the ``PUDL_INPUT`` directory can grow to be dozens - of gigabytes in size. This is because when the raw data are updated, - a new version of the archive is downloaded to the ``PUDL_INPUT`` - directory. To slim down the size you can always delete - out of date archives the code no longer depends on. + The data stored at the ``PUDL_INPUT`` directory can grow to be dozens of gigabytes + in size. This is because when the raw data are updated, a new version of the archive + is downloaded to the ``PUDL_INPUT`` directory. To slim down the size you can always + delete out of date archives the code no longer depends on. -Next, create a directory to store the outputs of the PUDL ETL. As above, you -can put this anywhere, but typically this is ``~/pudl_output``. Then, as -with ``PUDL_INPUT``, create an environment variable called ``PUDL_OUTPUT`` to -store the path to this new directory: +Next, create a directory to store the outputs of the PUDL ETL. As above, you can put +this anywhere, but typically this is ``~/pudl_output``. Then, as with ``PUDL_INPUT``, +create an environment variable called ``PUDL_OUTPUT`` to store the path to this new +directory: .. code-block:: console @@ -298,64 +290,20 @@ store the path to this new directory: $ echo "export PUDL_OUTPUT=/absolute/path/to/pudl_output" >> ~/.bashrc # bash $ set -Ux PUDL_OUTPUT /absolute/path/to/pudl_output # fish -The path stored in ``PUDL_OUTPUT`` contains all ETL outputs like -``pudl.sqlite`` and ``hourly_emissions_epacems.parquet``. - -**Make sure you create separate directories for these environment variables! -It is recommended you create these directories outside of the pudl repository -directory so the inputs and outputs are not tracked in git.** - -Also, activate profile changes above in the current session. - -.. code-block:: console - - $ export PUDL_OUTPUT=/absolute/path/to/pudl_output - $ export PUDL_INPUT=/absolute/path/to/pudl_input - -.. _Legacy PUDL Setup: - -PUDL Workspace Setup (legacy method) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -In previous versions of PUDL, the ``pudl_setup`` script created workspace directories. -PUDL is moving towards using the ``PUDL_OUTPUT`` and ``PUDL_INPUT`` environment -variables instead of the ``pudl_setup`` script because the environment variables are -easier to reference in the codebase. - -.. note:: - - If you set up your workspace using ``pudl_setup`` you don't need to change - anything about your setup. Just re-run ``pudl_setup`` and a new directory - called ``output/`` will be created in your . You will need to - point ``PUDL_OUTPUT`` at this new directory and ``PUDL_INPUT`` at the - ``data/`` directory in . +The path stored in ``PUDL_OUTPUT`` contains all ETL outputs like ``pudl.sqlite`` and +``hourly_emissions_epacems.parquet``. .. warning:: - In a future release the ``pudl_setup`` command will be removed. + Make sure you set these environment variables to point at separate directories! It + is also **strongly recommended** that you create these directories outside of the + pudl repository directory so the inputs and outputs are not tracked in git. -The ``pudl_setup`` script lets PUDL know where to store inputs and outputs. -The script will not create a new directory based on your arguemnts, so make -sure whatever directory path you pass as already exists. +Remember that you'll need to either source your shell profile after adding the new +environment variable definitions above, or export them at the command line for them to +be active in the current shell: .. code-block:: console - $ pudl_setup - - is the path to the directory where you want PUDL to do its -business -- this is where the datastore will be located and where any outputs -that are generated end up. The script will also put a configuration file called -``.pudl.yml`` in your home directory that records the location of this -workspace and uses it by default in the future. If you run ``pudl_setup`` with -no arguments, it assumes you want to use the current working directory. - -The workspace is laid out like this: - -==================== ========================================================== -**Directory / File** **Contents** --------------------- ---------------------------------------------------------- -``data/`` Raw data, automatically organized by source, year, etc. - This is the path ``PUDL_INPUT`` should point to. --------------------- ---------------------------------------------------------- -``output/`` The directory into which all the durable products of the - PUDL data processing pipeline will be written. -==================== ========================================================== + $ export PUDL_OUTPUT=/absolute/path/to/pudl_output + $ export PUDL_INPUT=/absolute/path/to/pudl_input diff --git a/docs/dev/nightly_data_builds.rst b/docs/dev/nightly_data_builds.rst index 4ad3343e9d..4c3c85c155 100644 --- a/docs/dev/nightly_data_builds.rst +++ b/docs/dev/nightly_data_builds.rst @@ -70,9 +70,14 @@ Then you can go to the `Google Compute Engine `__ page and restart it. -Once that's started, you should be able to SSH to the VM via ``gcloud ssh -vm-name``. You may run into some permissions issues here, in which case you -probably need the ``Service Account User`` role on your gcloud user. +Once that's started, you should be able to SSH to the VM using a command like: + +.. code:: + + gcloud compute ssh pudl-deployment-tag --zone=us-west1-a + +You may run into some permissions issues here, in which case you probably need the +``Service Account User`` role on your gcloud user. Now you want to get some logs about what's failing. diff --git a/docs/dev/run_the_etl.rst b/docs/dev/run_the_etl.rst index 7c1a6a6094..19139f56a1 100644 --- a/docs/dev/run_the_etl.rst +++ b/docs/dev/run_the_etl.rst @@ -343,14 +343,14 @@ We also have targets set up in the ``Makefile`` for running these scripts: Settings Files -------------- -These CLI commands use YAML settings files in place of command line arguments. -This avoids undue complexity and preserves a record of how the script was run. -The YAML file dictates which years, or states get run through the the processing -pipeline. Two example files are deployed in the ``settings`` folder that is created when -you run ``pudl_setup``. (see: :ref:`install-workspace`). +These CLI commands use YAML settings files in place of command line arguments. This +avoids undue complexity and preserves a record of how the script was run. The YAML file +dictates which years or states get run through the the processing pipeline. There are +two standard settings files that we use to run the integration tests and the nightly +builds included in the repository: -- ``etl_fast.yml`` processes one year of data -- ``etl_full.yml`` processes all years of data +- ``src/pudl/package_data/settings/etl_fast.yml`` processes 1-2 years of data. +- ``src/pudl/package_data/settings/etl_full.yml`` processes all available data. .. warning:: @@ -372,8 +372,8 @@ needs. The layout of these files is depicted below: | └── years ├── ferc1_xbrl_to_sqlite_settings | └── years - ├── ferc2_xbrl_to_sqlite_settings - | └── years + └── ferc2_xbrl_to_sqlite_settings + └── years # PUDL ETL settings name : unique name identifying the etl outputs @@ -381,9 +381,9 @@ needs. The layout of these files is depicted below: description : a longer description of the etl outputs datasets: ├── dataset name - │  └── dataset etl parameter (e.g. years) : editable list of years + │ └── dataset etl parameter (e.g. years) : editable list of years └── dataset name - │  └── dataset etl parameter (e.g. years) : editable list of years + └── dataset etl parameter (e.g. years) : editable list of years Both scripts enable you to choose which **years** you want to include: @@ -427,7 +427,7 @@ settings: - EPA CEMS cannot be loaded without EIA data unless you have existing PUDL database. -Now that your settings are configured, you're ready to run the scripts +Now that your settings are configured, you're ready to run the scripts. The Fast ETL ------------ @@ -450,8 +450,8 @@ CEMS should take around 2 hours. .. code-block:: console - $ ferc_to_sqlite settings/etl_full.yml - $ pudl_etl settings/etl_full.yml + $ ferc_to_sqlite src/pudl/package_data/settings/etl_full.yml + $ pudl_etl src/pudl/package_data/settings/etl_full.yml Custom ETL ---------- @@ -459,45 +459,17 @@ You've changed the settings and renamed the file to CUSTOM_ETL.yml .. code-block:: console - $ ferc_to_sqlite settings/CUSTOM_ETL.yml - $ pudl_etl settings/CUSTOM_ETL.yml - - -.. _add-cems-later: - -Processing EPA CEMS Separately ------------------------------- -As mentioned above, CEMS takes a while to process. Luckily, we've designed PUDL so that -if you delete or comment out CEMS lines in the settings file, you can process it -independently later without reprocessing the FERC and EIA data. The following script -will refer to your existing PUDL database for the information it needs and act as if the -FERC and EIA ETL had just been run. This may go without saying, but you need an existing -PUDL DB with the appropriate EIA files in order for the script to work. - -.. code-block:: console - - $ epacems_to_parquet -y [YEARS] -s [STATES] - -This script does not have a YAML settings file, so you must specify which years and -states to include via command line arguments. Run ``epacems_to_parquet --help`` to -verify your options. Changing CEMS settings in a YAML file will not inform this script! -Running the script without any arguments will automatically process all states and -years. - -.. warning:: + $ ferc_to_sqlite the/path/to/your/custom_etl.yml + $ pudl_etl the/path/to/your/custom_etl.yml - If you process the EPA CEMS data after the fact (i.e., with the - ``epacems_to_parquet`` script), be careful that the version of PUDL used to generate - the DB is the same as the one you're using to process the CEMS data. Otherwise the - process and data may be incompatible with unpredictable results. Additional Notes ---------------- The commands above should result in a bunch of Python :mod:`logging` output describing -what the script is doing, and file outputs in your ``$PUDL_OUTPUT`` directory. When the -ETL is complete, you should see new files at e.g. ``$PUDL_OUTPUT/ferc1_dbf.sqlite`` and -``output/pudl.sqlite`` as well as a new directory at ``output/hourly_emissions_epacems`` -containing nested directories named by year and state. +what the script is doing, and file outputs the directory you specified via the +``$PUDL_OUTPUT`` environment variable. When the ETL is complete, you should see new +files at e.g. ``$PUDL_OUTPUT/ferc1_dbf.sqlite``, ``$PUDL_OUTPUT/pudl.sqlite`` and +``$PUDL_OUTPUT/hourly_emissions_epacems.parquet``. If you need to re-run ``ferc_to_sqlite`` and want to overwrite their previous outputs you can add ``--clobber`` (run ``ferc_to_sqlite --clobber``). All of the PUDL scripts @@ -506,8 +478,8 @@ also have help messages if you want additional information (run ``script_name -- .. note:: The ``pudl_etl`` command does not have a ``--clobber`` option because each etl run - uses the same database file to read and write tables. This enables re-running - portions of the ETL. + uses the same database file to read and write tables. This facilitates re-running + small portions of the ETL using Dagster. Foreign Keys ------------ diff --git a/docs/intro.rst b/docs/intro.rst index 058b38ad1e..c3c85c1ab5 100644 --- a/docs/intro.rst +++ b/docs/intro.rst @@ -28,6 +28,7 @@ pages for each source: We also publish SQLite databases containing relatively pristine versions of our more difficult to parse inputs, especially the old Visual FoxPro (DBF, pre-2021) and new XBRL data (2021+) published by FERC: + * `FERC Form 1 (DBF) `__ * `FERC Form 1 (XBRL) `__ * `FERC Form 2 (DBF) `__ diff --git a/docs/release_notes.rst b/docs/release_notes.rst index 388343baca..78fe2922af 100644 --- a/docs/release_notes.rst +++ b/docs/release_notes.rst @@ -3,7 +3,23 @@ PUDL Release Notes ======================================================================================= --------------------------------------------------------------------------------------- -v2023.XX.XX +v2023.12.XX +--------------------------------------------------------------------------------------- +* The ``epacems_to_parquet`` and ``state_demand`` scripts have been retired in favor of + using the Dagster UI. See :issue:`3107` and :pr:`3086`. Visualizations of hourly + state-level electricity demand have been moved into our example notebooks which can + be found both `on Kaggle `__ + and `on GitHub `__ +* The ``pudl_setup`` script has been retired. All input/output locations are now set + using the ``$PUDL_INPUT`` and ``$PUDL_OUTPUT`` environment variables. See + :issue:`3107` and :pr:`3086`. +* The :func:`pudl.analysis.service_territory.pudl_service_territories` script has been + fixed, and can be used to generate `GeoParquet `__ + outputs describing historical utility and balancing authority service territories. See + :issue:`1174` and :pr:`3086`. + +--------------------------------------------------------------------------------------- +v2023.12.01 --------------------------------------------------------------------------------------- Dagster Adoption @@ -70,10 +86,13 @@ Dagster Adoption Data Coverage ^^^^^^^^^^^^^ -* Updated :doc:`data_sources/eia860` to include final release data from 2022. -* Updated :doc:`data_sources/eia861` to include final release data from 2022. +* Updated :doc:`data_sources/eia860` to include final release data from 2022, see + :issue:`3008` & PR :pr:`3040`. +* Updated :doc:`data_sources/eia861` to include final release data from 2022, see + :issue:`3034` & PR :pr:`3048`. * Updated :doc:`data_sources/eia923` to include final release data from 2022 and - monthly YTD data as of October 2023. + monthly YTD data as of October 2023, see :issue:`3009` & PR :pr:`#3073`. +* Extracted the raw ``raw_eia923__emissions_control`` table, see PR :pr:`3100`. * Updated :doc:`data_sources/epacems` to switch from the old FTP server to the new CAMPD API, and to include 2022 data. Due to changes in the ETL, Alaska, Puerto Rico and Hawaii are now included in CEMS processing. See issue :issue:`1264` & PRs @@ -204,7 +223,6 @@ Data Coverage centralized service companies. `FERC Form 60 will also be available on Datasette `__. - Data Cleaning ^^^^^^^^^^^^^ @@ -292,10 +310,6 @@ Deprecations * ``pudl.transform.eia860.transform()`` and ``pudl.transform.eia923.transform()`` functions have been deprecated. The table level EIA cleaning funtions are now coordinated using dagster. -* The :mod:`pudl.convert.epacems_to_parquet` command now executes the - ``hourly_emissions_epacems`` asset as a dagster job. The ``—partition`` option - is no longer supported. Now only creates a directory of parquet files - for each year/state partition. * ``pudl.transform.ferc1.transform()`` has been removed. The ferc1 table transformations are now being orchestrated with Dagster. * ``pudl.transform.ferc1.transform`` can no longer be executed as a script. @@ -746,7 +760,7 @@ SQLite and Parquet Outputs :issue:`1176,806`. * Data types, specified value constraints, and the uniqueness / non-null constraints on primary keys are validated during insertion into the SQLite DB. -* The PUDL ETL CLI :mod:`pudl.cli` now has flags to toggle various constraint +* The PUDL ETL CLI :mod:`pudl.etl.cli` now has flags to toggle various constraint checks including ``--ignore-foreign-key-constraints`` ``--ignore-type-constraints`` and ``--ignore-value-constraints``. diff --git a/docs/templates/epacems_child.rst.jinja b/docs/templates/epacems_child.rst.jinja index a56c1e0e8c..6c16c425b1 100644 --- a/docs/templates/epacems_child.rst.jinja +++ b/docs/templates/epacems_child.rst.jinja @@ -66,15 +66,13 @@ encompassing a year of data. CEMS is enourmous ----------------- CEMS is by far the largest dataset in PUDL what with hourly records for -thousands of plants spanning decades. For this reason, we house CEMS data in `Apache -Parquet `__ files rather than the main PUDL database. -Still, running the ETL with all of the CEMS data can take a long time. Note that you can -:ref:`process CEMS Data seperately ` from the main ETL -script if you'd like. - -Check out the `EPA CEMS example notebook `__ -in our `pudl-examples repository `__ -on GitHub for pointers on how to access this dataset efficiently using :mod:`dask`. +thousands of plants spanning decades. For this reason, we distribute CEMS data using `Apache +Parquet `__ files rather than in the main PUDL database. + +Check out the `PUDL data access example notebook on Kaggle +`__ or in our +`pudl-examples repository `__ on +GitHub for pointers on how to access this dataset efficiently using :mod:`dask`. EPA units vs. EIA units ----------------------- diff --git a/environments/conda-linux-64.lock.yml b/environments/conda-linux-64.lock.yml index 3d6da6f452..b3a384bb41 100644 --- a/environments/conda-linux-64.lock.yml +++ b/environments/conda-linux-64.lock.yml @@ -1,6 +1,6 @@ # Generated by conda-lock. # platform: linux-64 -# input_hash: 566bcbefc936d18bbf3a48ce8800e5d3209d9925447fc48984c419514bfaa6f6 +# input_hash: 25338bdeb54a81fb064efcdf38a56212bb8da4efc4654985f265ded511dcb38f channels: - conda-forge @@ -117,7 +117,7 @@ dependencies: - freetype=2.12.1=h267a509_2 - krb5=1.21.2=h659d440_0 - libarchive=3.7.2=h039dbb9_0 - - libglib=2.78.1=h783c2da_1 + - libglib=2.78.3=h783c2da_0 - libllvm15=15.0.7=h5cf9203_3 - libopenblas=0.3.25=pthreads_h413a1c8_0 - libthrift=0.19.0=hb90f79a_1 @@ -160,7 +160,7 @@ dependencies: - colorama=0.4.6=pyhd8ed1ab_0 - crashtest=0.4.1=pyhd8ed1ab_0 - cycler=0.12.1=pyhd8ed1ab_0 - - dagster-pipes=1.5.9=pyhd8ed1ab_0 + - dagster-pipes=1.5.10=pyhd8ed1ab_0 - dataclasses=0.8=pyhc8e2a94_3 - dbus=1.13.6=h5008d03_3 - debugpy=1.8.0=py311hb755f60_1 @@ -168,7 +168,7 @@ dependencies: - defusedxml=0.7.1=pyhd8ed1ab_0 - distlib=0.3.7=pyhd8ed1ab_0 - docstring_parser=0.15=pyhd8ed1ab_0 - - docutils=0.20.1=py311h38be061_2 + - docutils=0.20.1=py311h38be061_3 - entrypoints=0.4=pyhd8ed1ab_0 - et_xmlfile=1.1.0=pyhd8ed1ab_0 - exceptiongroup=1.2.0=pyhd8ed1ab_0 @@ -178,9 +178,9 @@ dependencies: - fontconfig=2.14.2=h14ed4e7_0 - freexl=2.0.0=h743c826_0 - frozenlist=1.4.0=py311h459d7ec_1 - - fsspec=2023.10.0=pyhca7485f_0 + - fsspec=2023.12.1=pyhca7485f_0 - gdk-pixbuf=2.42.10=h829c605_4 - - google-cloud-sdk=455.0.0=py311h38be061_0 + - google-cloud-sdk=456.0.0=py311h38be061_0 - greenlet=3.0.1=py311hb755f60_0 - gts=0.7.6=h977cf35_4 - hpack=4.0.0=pyh9f0ad1d_0 @@ -200,7 +200,7 @@ dependencies: - jsonpointer=2.4=py311h38be061_3 - jupyterlab_widgets=3.0.9=pyhd8ed1ab_0 - kiwisolver=1.4.5=py311h9547e67_1 - - lcms2=2.15=hb7c19ff_3 + - lcms2=2.16=hb7c19ff_0 - libblas=3.9.0=20_linux64_openblas - libcurl=8.4.0=hca28451_0 - libgrpc=1.59.2=hd6c4280_0 @@ -208,7 +208,7 @@ dependencies: - libwebp=1.3.2=h658648e_1 - llvmlite=0.41.1=py311ha6695c7_0 - locket=1.0.0=pyhd8ed1ab_0 - - lxml=4.9.3=py311h1a07684_1 + - lxml=4.9.3=py311h1a07684_2 - marko=2.0.2=pyhd8ed1ab_0 - markupsafe=2.1.3=py311h459d7ec_1 - mdurl=0.1.0=pyhd8ed1ab_0 @@ -218,7 +218,6 @@ dependencies: - msgpack-python=1.0.7=py311h9547e67_0 - multidict=6.0.4=py311h459d7ec_1 - multimethod=1.9.1=pyhd8ed1ab_0 - - munch=4.0.0=pyhd8ed1ab_0 - munkres=1.1.4=pyh9f0ad1d_0 - mypy_extensions=1.0.0=pyha770c72_0 - nest-asyncio=1.5.8=pyhd8ed1ab_0 @@ -232,6 +231,7 @@ dependencies: - pickleshare=0.7.5=py_1003 - pkginfo=1.9.6=pyhd8ed1ab_0 - pkgutil-resolve-name=1.3.10=pyhd8ed1ab_1 + - platformdirs=4.1.0=pyhd8ed1ab_0 - pluggy=1.3.0=pyhd8ed1ab_0 - prettier=3.1.0=h31abb78_0 - prometheus_client=0.19.0=pyhd8ed1ab_0 @@ -255,14 +255,14 @@ dependencies: - pywin32-on-windows=0.1.0=pyh1179c8e_3 - pyxlsb=1.0.10=pyhd8ed1ab_0 - pyyaml=6.0.1=py311h459d7ec_1 - - pyzmq=25.1.1=py311h34ded2d_2 + - pyzmq=25.1.2=py311h34ded2d_0 - regex=2023.10.3=py311h459d7ec_0 - rfc3986=2.0.0=pyhd8ed1ab_0 - rfc3986-validator=0.1.1=pyh9f0ad1d_0 - rpds-py=0.13.2=py311h46250e7_0 - rtree=1.1.0=py311h3bb2b0f_0 - ruamel.yaml.clib=0.2.7=py311h459d7ec_2 - - ruff=0.1.6=py311h7145743_0 + - ruff=0.1.7=py311h7145743_0 - send2trash=1.8.2=pyh41d4057_0 - setuptools=68.2.2=pyhd8ed1ab_0 - shellingham=1.5.4=pyhd8ed1ab_0 @@ -296,7 +296,7 @@ dependencies: - wcwidth=0.2.12=pyhd8ed1ab_0 - webcolors=1.13=pyhd8ed1ab_0 - webencodings=0.5.1=pyhd8ed1ab_2 - - websocket-client=1.6.4=pyhd8ed1ab_0 + - websocket-client=1.7.0=pyhd8ed1ab_0 - websockets=10.4=py311hd4cff14_1 - wheel=0.42.0=pyhd8ed1ab_0 - widgetsnbextension=4.0.9=pyhd8ed1ab_0 @@ -331,7 +331,7 @@ dependencies: - comm=0.1.4=pyhd8ed1ab_0 - coverage=7.3.2=py311h459d7ec_0 - curl=8.4.0=hca28451_0 - - fonttools=4.45.1=py311h459d7ec_0 + - fonttools=4.46.0=py311h459d7ec_0 - gitdb=4.0.11=pyhd8ed1ab_0 - graphql-core=3.2.3=pyhd8ed1ab_0 - grpcio=1.59.2=py311ha6695c7_0 @@ -340,7 +340,7 @@ dependencies: - hdf5=1.14.2=nompi_h4f84152_100 - html5lib=1.1=pyh9f0ad1d_0 - hypothesis=6.91.0=pyha770c72_0 - - importlib-metadata=6.8.0=pyha770c72_0 + - importlib-metadata=7.0.0=pyha770c72_0 - importlib_resources=6.1.1=pyhd8ed1ab_0 - isodate=0.6.1=pyhd8ed1ab_0 - janus=1.0.0=pyhd8ed1ab_0 @@ -349,6 +349,7 @@ dependencies: - jinja2=3.1.2=pyhd8ed1ab_1 - joblib=1.3.2=pyhd8ed1ab_0 - jsonlines=4.0.0=pyhd8ed1ab_0 + - jupyter_core=5.5.0=py311h38be061_0 - jupyterlab_pygments=0.3.0=pyhd8ed1ab_0 - latexcodec=2.0.1=pyh9f0ad1d_0 - libcblas=3.9.0=20_linux64_openblas @@ -366,12 +367,11 @@ dependencies: - pillow=10.1.0=py311ha6c5da5_0 - pint=0.22=pyhd8ed1ab_1 - pip=23.3.1=pyhd8ed1ab_0 - - platformdirs=4.0.0=pyhd8ed1ab_0 - postgresql=16.1=h8972f4a_0 - proj=9.3.0=h1d62c97_2 - prompt-toolkit=3.0.41=pyha770c72_0 - protobuf=4.24.4=py311h46cbc50_0 - - psycopg2=2.9.7=py311h03dec38_1 + - psycopg2=2.9.9=py311h03dec38_0 - pyasn1-modules=0.3.0=pyhd8ed1ab_0 - pyproject_hooks=1.0.0=pyhd8ed1ab_0 - pytest=7.4.3=pyhd8ed1ab_0 @@ -379,7 +379,7 @@ dependencies: - python-slugify=8.0.1=pyhd8ed1ab_2 - pyu2f=0.1.5=pyhd8ed1ab_0 - qtpy=2.4.1=pyhd8ed1ab_0 - - referencing=0.31.1=pyhd8ed1ab_0 + - referencing=0.32.0=pyhd8ed1ab_0 - restructuredtext_lint=1.4.0=pyhd8ed1ab_0 - rfc3339-validator=0.1.4=pyhd8ed1ab_0 - rsa=4.9=pyhd8ed1ab_0 @@ -391,6 +391,7 @@ dependencies: - typing_inspect=0.9.0=pyhd8ed1ab_0 - universal_pathlib=0.1.4=pyhd8ed1ab_0 - urllib3=1.26.18=pyhd8ed1ab_0 + - virtualenv=20.25.0=pyhd8ed1ab_0 - watchdog=3.0.0=py311h38be061_1 - xerces-c=3.2.4=hac6953d_3 - yarl=1.9.3=py311h459d7ec_0 @@ -401,23 +402,22 @@ dependencies: - arrow=1.3.0=pyhd8ed1ab_0 - async-timeout=4.0.3=pyhd8ed1ab_0 - aws-c-s3=0.4.1=hfadff92_0 - - botocore=1.33.5=pyhd8ed1ab_0 + - botocore=1.33.9=pyhd8ed1ab_0 - branca=0.7.0=pyhd8ed1ab_1 - croniter=2.0.1=pyhd8ed1ab_0 - - cryptography=41.0.5=py311h63ff55d_0 + - cryptography=41.0.7=py311hcb13ee4_1 - fqdn=1.5.1=pyhd8ed1ab_0 - geotiff=1.7.1=hf074850_14 - gitpython=3.1.40=pyhd8ed1ab_0 - google-crc32c=1.1.2=py311h9b08b9c_5 - - googleapis-common-protos=1.61.0=pyhd8ed1ab_0 + - googleapis-common-protos=1.62.0=pyhd8ed1ab_0 - gql=3.4.1=pyhd8ed1ab_0 - graphql-relay=3.2.0=pyhd8ed1ab_0 - grpcio-health-checking=1.59.2=pyhd8ed1ab_0 - harfbuzz=8.3.0=h3d44ed6_0 - httpcore=1.0.2=pyhd8ed1ab_0 - - importlib_metadata=6.8.0=hd8ed1ab_0 + - importlib_metadata=7.0.0=hd8ed1ab_0 - jsonschema-specifications=2023.11.2=pyhd8ed1ab_0 - - jupyter_core=5.5.0=py311h38be061_0 - jupyter_server_terminals=0.4.4=pyhd8ed1ab_1 - kealib=1.5.2=hcd42e92_1 - libnetcdf=4.9.2=nompi_h80fb2b6_112 @@ -428,7 +428,7 @@ dependencies: - pendulum=2.1.2=py311h459d7ec_6 - poppler=23.11.0=h590f24d_0 - prompt_toolkit=3.0.41=hd8ed1ab_0 - - psycopg2-binary=2.9.7=pyhd8ed1ab_1 + - psycopg2-binary=2.9.9=pyhd8ed1ab_0 - pybtex=0.24.0=pyhd8ed1ab_2 - pydantic-core=2.14.5=py311h46250e7_0 - pyproj=3.6.1=py311h1facc83_4 @@ -441,31 +441,30 @@ dependencies: - rich=13.7.0=pyhd8ed1ab_0 - sqlalchemy=2.0.23=py311h459d7ec_0 - stack_data=0.6.2=pyhd8ed1ab_0 - - starlette=0.32.0.post1=pyhd8ed1ab_0 + - starlette=0.33.0=pyhd8ed1ab_0 - tiledb=2.16.3=h8c794c1_3 - ukkonen=1.0.1=py311h9547e67_4 - uvicorn=0.24.0.post1=py311h38be061_0 - - virtualenv=20.24.7=pyhd8ed1ab_0 - watchfiles=0.21.0=py311h46250e7_0 - aiohttp=3.8.6=py311h459d7ec_1 - - alembic=1.12.1=pyhd8ed1ab_0 - - arelle-release=2.17.7=pyhd8ed1ab_0 + - alembic=1.13.0=pyhd8ed1ab_0 + - arelle-release=2.18.0=pyhd8ed1ab_0 - argon2-cffi=23.1.0=pyhd8ed1ab_0 - aws-crt-cpp=0.24.7=h97e63c7_6 - bottleneck=1.3.7=py311h1f0f07a_1 - cachecontrol=0.13.1=pyhd8ed1ab_0 - contourpy=1.2.0=py311h9547e67_0 - - dask-core=2023.11.0=pyhd8ed1ab_0 + - dask-core=2023.12.0=pyhd8ed1ab_0 - dnspython=2.4.2=pyhd8ed1ab_1 - ensureconda=1.4.3=pyhd8ed1ab_0 - - folium=0.15.0=pyhd8ed1ab_0 + - folium=0.15.1=pyhd8ed1ab_0 - google-resumable-media=2.6.0=pyhd8ed1ab_0 - graphene=3.3=pyhd8ed1ab_0 - grpcio-status=1.59.2=pyhd8ed1ab_0 - h3-py=3.7.6=py311hb755f60_1 - httpx=0.25.2=pyhd8ed1ab_0 - - identify=2.5.32=pyhd8ed1ab_0 - - ipython=8.18.1=pyh31011fe_1 + - identify=2.5.33=pyhd8ed1ab_0 + - ipython=8.18.1=pyh707e725_3 - isoduration=20.11.0=pyhd8ed1ab_0 - jsonschema=4.20.0=pyhd8ed1ab_0 - jupyter_client=8.6.0=pyhd8ed1ab_0 @@ -490,16 +489,16 @@ dependencies: - typer=0.9.0=pyhd8ed1ab_0 - uvicorn-standard=0.24.0.post1=h38be061_0 - aws-sdk-cpp=1.11.182=h8beafcf_7 - - boto3=1.33.5=pyhd8ed1ab_0 + - boto3=1.33.9=pyhd8ed1ab_0 - cachecontrol-with-filecache=0.13.1=pyhd8ed1ab_0 - - dagster=1.5.9=pyhd8ed1ab_0 + - dagster=1.5.10=pyhd8ed1ab_0 - datasette=0.64.4=pyhd8ed1ab_1 - doc8=1.1.1=pyhd8ed1ab_0 - email-validator=2.1.0.post1=pyhd8ed1ab_0 - frictionless=4.40.8=pyh6c4a22f_0 - gdal=3.8.0=py311h815a124_6 - geopandas-base=0.14.1=pyha770c72_0 - - google-auth=2.24.0=pyhca7485f_0 + - google-auth=2.25.1=pyhca7485f_0 - gql-with-requests=3.4.1=pyhd8ed1ab_0 - gtk2=2.24.33=h90689f9_2 - ipykernel=6.26.0=pyhf8b6a83_0 @@ -513,14 +512,14 @@ dependencies: - pre-commit=3.5.0=pyha770c72_0 - pydantic-settings=2.1.0=pyhd8ed1ab_1 - requests-oauthlib=1.3.1=pyhd8ed1ab_0 - - scikit-learn=1.3.2=py311hc009520_1 + - scikit-learn=1.3.2=py311hc009520_2 - timezonefinder=6.2.0=py311h459d7ec_2 - catalystcoop.ferc_xbrl_extractor=1.3.1=pyhd8ed1ab_0 - conda-lock=2.5.1=pyhd8ed1ab_0 - - dagster-graphql=1.5.9=pyhd8ed1ab_0 - - dagster-postgres=0.21.9=pyhd8ed1ab_1 - - fiona=1.9.5=py311hf8e0aa6_1 - - google-api-core=2.14.0=pyhd8ed1ab_0 + - dagster-graphql=1.5.10=pyhd8ed1ab_0 + - dagster-postgres=0.21.10=pyhd8ed1ab_0 + - fiona=1.9.5=py311hf8e0aa6_2 + - google-api-core=2.15.0=pyhd8ed1ab_0 - google-auth-oauthlib=1.1.0=pyhd8ed1ab_0 - graphviz=9.0.0=h78e8752_1 - jupyter_console=6.6.3=pyhd8ed1ab_0 @@ -531,28 +530,28 @@ dependencies: - qtconsole-base=5.5.1=pyha770c72_0 - recordlinkage=0.16=pyhd8ed1ab_0 - tabulator=1.53.5=pyhd8ed1ab_0 - - dagster-webserver=1.5.9=pyhd8ed1ab_0 + - dagster-webserver=1.5.10=pyhd8ed1ab_0 - geopandas=0.14.1=pyhd8ed1ab_0 - - google-cloud-core=2.3.3=pyhd8ed1ab_0 + - google-cloud-core=2.4.1=pyhd8ed1ab_0 - libarrow-acero=14.0.1=h59595ed_3_cpu - libarrow-flight=14.0.1=h120cb0d_3_cpu - libarrow-gandiva=14.0.1=hacb8726_3_cpu - libparquet=14.0.1=h352af49_3_cpu - - nbconvert-core=7.11.0=pyhd8ed1ab_0 + - nbconvert-core=7.12.0=pyhd8ed1ab_0 - pygraphviz=1.11=py311hbf5cbc9_2 - tableschema=1.19.3=pyh9f0ad1d_0 - datapackage=1.15.2=pyh44b312d_0 - google-cloud-storage=2.13.0=pyhca7485f_0 - - jupyter_server=2.11.1=pyhd8ed1ab_0 + - jupyter_server=2.12.1=pyhd8ed1ab_0 - libarrow-dataset=14.0.1=h59595ed_3_cpu - libarrow-flight-sql=14.0.1=h61ff412_3_cpu - - nbconvert-pandoc=7.11.0=pyhd8ed1ab_0 - - gcsfs=2023.10.0=pyhd8ed1ab_0 + - nbconvert-pandoc=7.12.0=pyhd8ed1ab_0 + - gcsfs=2023.12.1=pyhd8ed1ab_0 - jupyter-lsp=2.2.1=pyhd8ed1ab_0 - jupyter-resource-usage=1.0.1=pyhd8ed1ab_0 - jupyterlab_server=2.25.2=pyhd8ed1ab_0 - libarrow-substrait=14.0.1=h61ff412_3_cpu - - nbconvert=7.11.0=pyhd8ed1ab_0 + - nbconvert=7.12.0=pyhd8ed1ab_0 - notebook-shim=0.2.3=pyhd8ed1ab_0 - jupyterlab=4.0.9=pyhd8ed1ab_0 - pyarrow=14.0.1=py311h39c9aba_3_cpu diff --git a/environments/conda-lock.yml b/environments/conda-lock.yml index 148e867498..485dd1c1da 100644 --- a/environments/conda-lock.yml +++ b/environments/conda-lock.yml @@ -15,9 +15,9 @@ version: 1 metadata: content_hash: - linux-64: 566bcbefc936d18bbf3a48ce8800e5d3209d9925447fc48984c419514bfaa6f6 - osx-64: 39e56673d0def5503b315f36cec2b3e4bfd804758ec3bcea79dad417c6b146a2 - osx-arm64: c275fe8ff3012ad83a98252ba570f9b278f142720b73f42abfd4c4a1107f67e2 + linux-64: 25338bdeb54a81fb064efcdf38a56212bb8da4efc4654985f265ded511dcb38f + osx-64: 0c46f3c7a9c86a583e588c727d6a7e7d3214ce9d048dd4e49df3a792c36b3e9c + osx-arm64: 3e57b04f7b2300284a5c81502687a5c373c4be31786246211ea23f019d4a513f channels: - url: conda-forge used_env_vars: [] @@ -266,7 +266,7 @@ package: category: main optional: false - name: alembic - version: 1.12.1 + version: 1.13.0 manager: conda platform: linux-64 dependencies: @@ -276,14 +276,14 @@ package: python: ">=3.7" sqlalchemy: ">=1.3.0" typing-extensions: ">=4" - url: https://conda.anaconda.org/conda-forge/noarch/alembic-1.12.1-pyhd8ed1ab_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/alembic-1.13.0-pyhd8ed1ab_0.conda hash: - md5: 15de9992b4096a2a6656ca202fde6e4c - sha256: 24019b1af4777e32843b230dd7a9bf7082943eb21bba03379ceed0bda50facf9 + md5: 7f0372c1cfa41891787ddf267334c0c7 + sha256: d6dd010632bc5272fe7e51646651c5fb9ae9b7663113b94aa585d6bcb43834b0 category: main optional: false - name: alembic - version: 1.12.1 + version: 1.13.0 manager: conda platform: osx-64 dependencies: @@ -293,14 +293,14 @@ package: python: ">=3.7" sqlalchemy: ">=1.3.0" typing-extensions: ">=4" - url: https://conda.anaconda.org/conda-forge/noarch/alembic-1.12.1-pyhd8ed1ab_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/alembic-1.13.0-pyhd8ed1ab_0.conda hash: - md5: 15de9992b4096a2a6656ca202fde6e4c - sha256: 24019b1af4777e32843b230dd7a9bf7082943eb21bba03379ceed0bda50facf9 + md5: 7f0372c1cfa41891787ddf267334c0c7 + sha256: d6dd010632bc5272fe7e51646651c5fb9ae9b7663113b94aa585d6bcb43834b0 category: main optional: false - name: alembic - version: 1.12.1 + version: 1.13.0 manager: conda platform: osx-arm64 dependencies: @@ -310,10 +310,10 @@ package: python: ">=3.7" sqlalchemy: ">=1.3.0" typing-extensions: ">=4" - url: https://conda.anaconda.org/conda-forge/noarch/alembic-1.12.1-pyhd8ed1ab_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/alembic-1.13.0-pyhd8ed1ab_0.conda hash: - md5: 15de9992b4096a2a6656ca202fde6e4c - sha256: 24019b1af4777e32843b230dd7a9bf7082943eb21bba03379ceed0bda50facf9 + md5: 7f0372c1cfa41891787ddf267334c0c7 + sha256: d6dd010632bc5272fe7e51646651c5fb9ae9b7663113b94aa585d6bcb43834b0 category: main optional: false - name: aniso8601 @@ -536,7 +536,7 @@ package: category: main optional: false - name: arelle-release - version: 2.17.7 + version: 2.18.0 manager: conda platform: linux-64 dependencies: @@ -549,50 +549,50 @@ package: python: ">=3.8" python-dateutil: 2.* regex: "" - url: https://conda.anaconda.org/conda-forge/noarch/arelle-release-2.17.7-pyhd8ed1ab_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/arelle-release-2.18.0-pyhd8ed1ab_0.conda hash: - md5: b42bbf2e318b6bbbd9de2d81ecf8ed50 - sha256: 3094446e601ad9160677c2bb5b75b9946c81b679bebf42bf52c126e71d76fb43 + md5: cf492dd79a1d1d1ef2af299da2a604ca + sha256: f3fa253f2a26faa56348acc2a76eaf55778f871b995af429c9636d9159a4b483 category: main optional: false - name: arelle-release - version: 2.17.7 + version: 2.18.0 manager: conda platform: osx-64 dependencies: certifi: "" regex: "" python: ">=3.8" - numpy: 1.* python-dateutil: 2.* + numpy: 1.* isodate: 0.* lxml: 4.* openpyxl: 3.* pyparsing: 3.* - url: https://conda.anaconda.org/conda-forge/noarch/arelle-release-2.17.7-pyhd8ed1ab_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/arelle-release-2.18.0-pyhd8ed1ab_0.conda hash: - md5: b42bbf2e318b6bbbd9de2d81ecf8ed50 - sha256: 3094446e601ad9160677c2bb5b75b9946c81b679bebf42bf52c126e71d76fb43 + md5: cf492dd79a1d1d1ef2af299da2a604ca + sha256: f3fa253f2a26faa56348acc2a76eaf55778f871b995af429c9636d9159a4b483 category: main optional: false - name: arelle-release - version: 2.17.7 + version: 2.18.0 manager: conda platform: osx-arm64 dependencies: certifi: "" regex: "" python: ">=3.8" - numpy: 1.* python-dateutil: 2.* + numpy: 1.* isodate: 0.* lxml: 4.* openpyxl: 3.* pyparsing: 3.* - url: https://conda.anaconda.org/conda-forge/noarch/arelle-release-2.17.7-pyhd8ed1ab_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/arelle-release-2.18.0-pyhd8ed1ab_0.conda hash: - md5: b42bbf2e318b6bbbd9de2d81ecf8ed50 - sha256: 3094446e601ad9160677c2bb5b75b9946c81b679bebf42bf52c126e71d76fb43 + md5: cf492dd79a1d1d1ef2af299da2a604ca + sha256: f3fa253f2a26faa56348acc2a76eaf55778f871b995af429c9636d9159a4b483 category: main optional: false - name: argon2-cffi @@ -1914,52 +1914,52 @@ package: category: main optional: false - name: boto3 - version: 1.33.5 + version: 1.33.9 manager: conda platform: linux-64 dependencies: - botocore: ">=1.33.5,<1.34.0" + botocore: ">=1.33.9,<1.34.0" jmespath: ">=0.7.1,<2.0.0" python: ">=3.7" s3transfer: ">=0.8.2,<0.9.0" - url: https://conda.anaconda.org/conda-forge/noarch/boto3-1.33.5-pyhd8ed1ab_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/boto3-1.33.9-pyhd8ed1ab_0.conda hash: - md5: 7485d3ee00269cd33baa2ad64a0923ee - sha256: babecba07e296dc4cd26580427508e7734c522d1488a7f94a1f13204de3ca856 + md5: 0bc87faa4f6ff6321c4cef9b57e82041 + sha256: 685c90c40889cabc9bd03e67bd066546d019ba13667814d6b78352afe5dd751f category: main optional: false - name: boto3 - version: 1.33.5 + version: 1.33.9 manager: conda platform: osx-64 dependencies: python: ">=3.7" jmespath: ">=0.7.1,<2.0.0" s3transfer: ">=0.8.2,<0.9.0" - botocore: ">=1.33.5,<1.34.0" - url: https://conda.anaconda.org/conda-forge/noarch/boto3-1.33.5-pyhd8ed1ab_0.conda + botocore: ">=1.33.9,<1.34.0" + url: https://conda.anaconda.org/conda-forge/noarch/boto3-1.33.9-pyhd8ed1ab_0.conda hash: - md5: 7485d3ee00269cd33baa2ad64a0923ee - sha256: babecba07e296dc4cd26580427508e7734c522d1488a7f94a1f13204de3ca856 + md5: 0bc87faa4f6ff6321c4cef9b57e82041 + sha256: 685c90c40889cabc9bd03e67bd066546d019ba13667814d6b78352afe5dd751f category: main optional: false - name: boto3 - version: 1.33.5 + version: 1.33.9 manager: conda platform: osx-arm64 dependencies: python: ">=3.7" jmespath: ">=0.7.1,<2.0.0" s3transfer: ">=0.8.2,<0.9.0" - botocore: ">=1.33.5,<1.34.0" - url: https://conda.anaconda.org/conda-forge/noarch/boto3-1.33.5-pyhd8ed1ab_0.conda + botocore: ">=1.33.9,<1.34.0" + url: https://conda.anaconda.org/conda-forge/noarch/boto3-1.33.9-pyhd8ed1ab_0.conda hash: - md5: 7485d3ee00269cd33baa2ad64a0923ee - sha256: babecba07e296dc4cd26580427508e7734c522d1488a7f94a1f13204de3ca856 + md5: 0bc87faa4f6ff6321c4cef9b57e82041 + sha256: 685c90c40889cabc9bd03e67bd066546d019ba13667814d6b78352afe5dd751f category: main optional: false - name: botocore - version: 1.33.5 + version: 1.33.9 manager: conda platform: linux-64 dependencies: @@ -1967,14 +1967,14 @@ package: python: ">=3.7" python-dateutil: ">=2.1,<3.0.0" urllib3: ">=1.25.4,<1.27" - url: https://conda.anaconda.org/conda-forge/noarch/botocore-1.33.5-pyhd8ed1ab_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/botocore-1.33.9-pyhd8ed1ab_0.conda hash: - md5: 352c39ba5cd9ea01996358f0748e102e - sha256: 56566ea8f3a48c24190c1dcf50681c0a84b26821c335c21b5c3c5d238e4bdb14 + md5: 66b5f9da695d491e2df00c0045f073a3 + sha256: 7658e63b78ae37bb7e2047232a7f6d0d851e3a57231beeb32ea21a2da002e26b category: main optional: false - name: botocore - version: 1.33.5 + version: 1.33.9 manager: conda platform: osx-64 dependencies: @@ -1982,14 +1982,14 @@ package: python-dateutil: ">=2.1,<3.0.0" jmespath: ">=0.7.1,<2.0.0" urllib3: ">=1.25.4,<1.27" - url: https://conda.anaconda.org/conda-forge/noarch/botocore-1.33.5-pyhd8ed1ab_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/botocore-1.33.9-pyhd8ed1ab_0.conda hash: - md5: 352c39ba5cd9ea01996358f0748e102e - sha256: 56566ea8f3a48c24190c1dcf50681c0a84b26821c335c21b5c3c5d238e4bdb14 + md5: 66b5f9da695d491e2df00c0045f073a3 + sha256: 7658e63b78ae37bb7e2047232a7f6d0d851e3a57231beeb32ea21a2da002e26b category: main optional: false - name: botocore - version: 1.33.5 + version: 1.33.9 manager: conda platform: osx-arm64 dependencies: @@ -1997,10 +1997,10 @@ package: python-dateutil: ">=2.1,<3.0.0" jmespath: ">=0.7.1,<2.0.0" urllib3: ">=1.25.4,<1.27" - url: https://conda.anaconda.org/conda-forge/noarch/botocore-1.33.5-pyhd8ed1ab_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/botocore-1.33.9-pyhd8ed1ab_0.conda hash: - md5: 352c39ba5cd9ea01996358f0748e102e - sha256: 56566ea8f3a48c24190c1dcf50681c0a84b26821c335c21b5c3c5d238e4bdb14 + md5: 66b5f9da695d491e2df00c0045f073a3 + sha256: 7658e63b78ae37bb7e2047232a7f6d0d851e3a57231beeb32ea21a2da002e26b category: main optional: false - name: bottleneck @@ -3658,7 +3658,7 @@ package: category: main optional: false - name: cryptography - version: 41.0.5 + version: 41.0.7 manager: conda platform: linux-64 dependencies: @@ -3667,14 +3667,14 @@ package: openssl: ">=3.1.4,<4.0a0" python: ">=3.11,<3.12.0a0" python_abi: 3.11.* - url: https://conda.anaconda.org/conda-forge/linux-64/cryptography-41.0.5-py311h63ff55d_0.conda + url: https://conda.anaconda.org/conda-forge/linux-64/cryptography-41.0.7-py311hcb13ee4_1.conda hash: - md5: 22584e5c97ed8f1a6b63a0ff43dba827 - sha256: 236ed2218fb857fecaa11fc7fee23574f683b3d03576f8f26f628b7fd2ced5fa + md5: ca6e04ac7262ecaec846e483d6fdc6c8 + sha256: 0959d015727ae5f55f385556a0a19b9f6036752ea05f78a99cb534803e325cab category: main optional: false - name: cryptography - version: 41.0.5 + version: 41.0.7 manager: conda platform: osx-64 dependencies: @@ -3682,14 +3682,14 @@ package: openssl: ">=3.1.4,<4.0a0" python: ">=3.11,<3.12.0a0" python_abi: 3.11.* - url: https://conda.anaconda.org/conda-forge/osx-64/cryptography-41.0.5-py311hd51016d_0.conda + url: https://conda.anaconda.org/conda-forge/osx-64/cryptography-41.0.7-py311h48c7838_1.conda hash: - md5: 99f1edef251a9fe4edf620b527ee70ea - sha256: 26ee22b99771f0d338eca6299cbe866f695c544d855d5eab82539497b0a24fc1 + md5: 65293feff96135571de02c047ecbd5a2 + sha256: f4c5e683386acf06cfa85805d264c9cd540361ec6e86740cb03312e560aa706a category: main optional: false - name: cryptography - version: 41.0.5 + version: 41.0.7 manager: conda platform: osx-arm64 dependencies: @@ -3697,10 +3697,10 @@ package: openssl: ">=3.1.4,<4.0a0" python: ">=3.11,<3.12.0a0" python_abi: 3.11.* - url: https://conda.anaconda.org/conda-forge/osx-arm64/cryptography-41.0.5-py311h71175c2_0.conda + url: https://conda.anaconda.org/conda-forge/osx-arm64/cryptography-41.0.7-py311h08c85a6_1.conda hash: - md5: adc55f424334b834098d50e57efe0789 - sha256: 00c9b389b51b6e951a1f639aa04dceca9e329e144275c79b4f6baacd3fb90345 + md5: 729546a91873db64ab8e675008845fb5 + sha256: 9d51ba743069d19aae08361a7b051ca1f281fb3b3ccb162e96c2503cf5a7d365 category: main optional: false - name: curl @@ -3792,7 +3792,7 @@ package: category: main optional: false - name: dagster - version: 1.5.9 + version: 1.5.10 manager: conda platform: linux-64 dependencies: @@ -3800,7 +3800,7 @@ package: click: ">=5.0" coloredlogs: ">=6.1,<=14.0" croniter: ">=0.3.34" - dagster-pipes: ">=1.5.9,<1.5.10.0a0" + dagster-pipes: ">=1.5.10,<1.5.11.0a0" docstring_parser: "" grpcio: ">=1.44.0" grpcio-health-checking: ">=1.44.0" @@ -3826,14 +3826,14 @@ package: typing_extensions: ">=4.4.0" universal_pathlib: "" watchdog: ">=0.8.3" - url: https://conda.anaconda.org/conda-forge/noarch/dagster-1.5.9-pyhd8ed1ab_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/dagster-1.5.10-pyhd8ed1ab_0.conda hash: - md5: d8ab27112f82687ffcd456a3b88092e5 - sha256: 238b08bf9afbc98405cb0c8c9845514da7b4b21aac5817c2b5f0de04e3a19b1b + md5: bcf42b675edb2999669116bc3b1ed789 + sha256: dcee8473cbd1823005285f5fc82fbb16d9d1b2b46bcdf8371ee4fe2fdb0ad50c category: main optional: false - name: dagster - version: 1.5.9 + version: 1.5.10 manager: conda platform: osx-64 dependencies: @@ -3866,15 +3866,15 @@ package: alembic: ">=1.2.1,!=1.6.3,!=1.7.0,!=1.11.0" pydantic: ">1.10.0,!=1.10.7" pendulum: <3 - dagster-pipes: ">=1.5.9,<1.5.10.0a0" - url: https://conda.anaconda.org/conda-forge/noarch/dagster-1.5.9-pyhd8ed1ab_0.conda + dagster-pipes: ">=1.5.10,<1.5.11.0a0" + url: https://conda.anaconda.org/conda-forge/noarch/dagster-1.5.10-pyhd8ed1ab_0.conda hash: - md5: d8ab27112f82687ffcd456a3b88092e5 - sha256: 238b08bf9afbc98405cb0c8c9845514da7b4b21aac5817c2b5f0de04e3a19b1b + md5: bcf42b675edb2999669116bc3b1ed789 + sha256: dcee8473cbd1823005285f5fc82fbb16d9d1b2b46bcdf8371ee4fe2fdb0ad50c category: main optional: false - name: dagster - version: 1.5.9 + version: 1.5.10 manager: conda platform: osx-arm64 dependencies: @@ -3907,32 +3907,32 @@ package: alembic: ">=1.2.1,!=1.6.3,!=1.7.0,!=1.11.0" pydantic: ">1.10.0,!=1.10.7" pendulum: <3 - dagster-pipes: ">=1.5.9,<1.5.10.0a0" - url: https://conda.anaconda.org/conda-forge/noarch/dagster-1.5.9-pyhd8ed1ab_0.conda + dagster-pipes: ">=1.5.10,<1.5.11.0a0" + url: https://conda.anaconda.org/conda-forge/noarch/dagster-1.5.10-pyhd8ed1ab_0.conda hash: - md5: d8ab27112f82687ffcd456a3b88092e5 - sha256: 238b08bf9afbc98405cb0c8c9845514da7b4b21aac5817c2b5f0de04e3a19b1b + md5: bcf42b675edb2999669116bc3b1ed789 + sha256: dcee8473cbd1823005285f5fc82fbb16d9d1b2b46bcdf8371ee4fe2fdb0ad50c category: main optional: false - name: dagster-graphql - version: 1.5.9 + version: 1.5.10 manager: conda platform: linux-64 dependencies: - dagster: ">=1.5.9,<1.5.10.0a0" + dagster: ">=1.5.10,<1.5.11.0a0" gql-with-requests: ">=3.0.0" graphene: ">=3" python: ">=3.8" requests: "" starlette: "" - url: https://conda.anaconda.org/conda-forge/noarch/dagster-graphql-1.5.9-pyhd8ed1ab_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/dagster-graphql-1.5.10-pyhd8ed1ab_0.conda hash: - md5: 7dcd105a5451f9800aa6de278d86db72 - sha256: 8484c6b0db1a3505fc7d16e83e0da75b9c886ae3d497266fd06f72fcd3246786 + md5: fd3569582db65a0c88fbc7d1bb803853 + sha256: a142f082da2b7905713b09e0ad19eef971a5d4f66063ba7a2b19247e3932e129 category: dev optional: true - name: dagster-graphql - version: 1.5.9 + version: 1.5.10 manager: conda platform: osx-64 dependencies: @@ -3941,15 +3941,15 @@ package: python: ">=3.8" graphene: ">=3" gql-with-requests: ">=3.0.0" - dagster: ">=1.5.9,<1.5.10.0a0" - url: https://conda.anaconda.org/conda-forge/noarch/dagster-graphql-1.5.9-pyhd8ed1ab_0.conda + dagster: ">=1.5.10,<1.5.11.0a0" + url: https://conda.anaconda.org/conda-forge/noarch/dagster-graphql-1.5.10-pyhd8ed1ab_0.conda hash: - md5: 7dcd105a5451f9800aa6de278d86db72 - sha256: 8484c6b0db1a3505fc7d16e83e0da75b9c886ae3d497266fd06f72fcd3246786 + md5: fd3569582db65a0c88fbc7d1bb803853 + sha256: a142f082da2b7905713b09e0ad19eef971a5d4f66063ba7a2b19247e3932e129 category: dev optional: true - name: dagster-graphql - version: 1.5.9 + version: 1.5.10 manager: conda platform: osx-arm64 dependencies: @@ -3958,110 +3958,110 @@ package: python: ">=3.8" graphene: ">=3" gql-with-requests: ">=3.0.0" - dagster: ">=1.5.9,<1.5.10.0a0" - url: https://conda.anaconda.org/conda-forge/noarch/dagster-graphql-1.5.9-pyhd8ed1ab_0.conda + dagster: ">=1.5.10,<1.5.11.0a0" + url: https://conda.anaconda.org/conda-forge/noarch/dagster-graphql-1.5.10-pyhd8ed1ab_0.conda hash: - md5: 7dcd105a5451f9800aa6de278d86db72 - sha256: 8484c6b0db1a3505fc7d16e83e0da75b9c886ae3d497266fd06f72fcd3246786 + md5: fd3569582db65a0c88fbc7d1bb803853 + sha256: a142f082da2b7905713b09e0ad19eef971a5d4f66063ba7a2b19247e3932e129 category: dev optional: true - name: dagster-pipes - version: 1.5.9 + version: 1.5.10 manager: conda platform: linux-64 dependencies: python: ">=3.8" - url: https://conda.anaconda.org/conda-forge/noarch/dagster-pipes-1.5.9-pyhd8ed1ab_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/dagster-pipes-1.5.10-pyhd8ed1ab_0.conda hash: - md5: 0a9787859365c4d2425e589ac53c462b - sha256: eebc7dca517350678ebfb8b3fff7ec47c60aff62dae2e69b8c4845b6080ec3e8 + md5: 69600c68efc23fb84c05c2e9c1c05947 + sha256: a98a8b2501af4bc112d5b2f5e3edea6c22a084651cc720786c405877b8507630 category: main optional: false - name: dagster-pipes - version: 1.5.9 + version: 1.5.10 manager: conda platform: osx-64 dependencies: python: ">=3.8" - url: https://conda.anaconda.org/conda-forge/noarch/dagster-pipes-1.5.9-pyhd8ed1ab_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/dagster-pipes-1.5.10-pyhd8ed1ab_0.conda hash: - md5: 0a9787859365c4d2425e589ac53c462b - sha256: eebc7dca517350678ebfb8b3fff7ec47c60aff62dae2e69b8c4845b6080ec3e8 + md5: 69600c68efc23fb84c05c2e9c1c05947 + sha256: a98a8b2501af4bc112d5b2f5e3edea6c22a084651cc720786c405877b8507630 category: main optional: false - name: dagster-pipes - version: 1.5.9 + version: 1.5.10 manager: conda platform: osx-arm64 dependencies: python: ">=3.8" - url: https://conda.anaconda.org/conda-forge/noarch/dagster-pipes-1.5.9-pyhd8ed1ab_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/dagster-pipes-1.5.10-pyhd8ed1ab_0.conda hash: - md5: 0a9787859365c4d2425e589ac53c462b - sha256: eebc7dca517350678ebfb8b3fff7ec47c60aff62dae2e69b8c4845b6080ec3e8 + md5: 69600c68efc23fb84c05c2e9c1c05947 + sha256: a98a8b2501af4bc112d5b2f5e3edea6c22a084651cc720786c405877b8507630 category: main optional: false - name: dagster-postgres - version: 0.21.9 + version: 0.21.10 manager: conda platform: linux-64 dependencies: - dagster: 1.5.9.* + dagster: 1.5.10.* psycopg2-binary: "" python: ">=3.8" - url: https://conda.anaconda.org/conda-forge/noarch/dagster-postgres-0.21.9-pyhd8ed1ab_1.conda + url: https://conda.anaconda.org/conda-forge/noarch/dagster-postgres-0.21.10-pyhd8ed1ab_0.conda hash: - md5: 8c1a941fe77b920b1c7933a7a0c6bf2e - sha256: 0e947f376d6878bd8e505932277e84c373da492a38d2c4ef9d96fc25f5327845 + md5: ec155b3a7172590ccbc89f461427b5aa + sha256: 4eb986655ef547d4dc72cd34b60687c9c3c390806493c15187c4d26d89d58fc0 category: main optional: false - name: dagster-postgres - version: 0.21.9 + version: 0.21.10 manager: conda platform: osx-64 dependencies: psycopg2-binary: "" python: ">=3.8" - dagster: 1.5.9.* - url: https://conda.anaconda.org/conda-forge/noarch/dagster-postgres-0.21.9-pyhd8ed1ab_1.conda + dagster: 1.5.10.* + url: https://conda.anaconda.org/conda-forge/noarch/dagster-postgres-0.21.10-pyhd8ed1ab_0.conda hash: - md5: 8c1a941fe77b920b1c7933a7a0c6bf2e - sha256: 0e947f376d6878bd8e505932277e84c373da492a38d2c4ef9d96fc25f5327845 + md5: ec155b3a7172590ccbc89f461427b5aa + sha256: 4eb986655ef547d4dc72cd34b60687c9c3c390806493c15187c4d26d89d58fc0 category: main optional: false - name: dagster-postgres - version: 0.21.9 + version: 0.21.10 manager: conda platform: osx-arm64 dependencies: psycopg2-binary: "" python: ">=3.8" - dagster: 1.5.9.* - url: https://conda.anaconda.org/conda-forge/noarch/dagster-postgres-0.21.9-pyhd8ed1ab_1.conda + dagster: 1.5.10.* + url: https://conda.anaconda.org/conda-forge/noarch/dagster-postgres-0.21.10-pyhd8ed1ab_0.conda hash: - md5: 8c1a941fe77b920b1c7933a7a0c6bf2e - sha256: 0e947f376d6878bd8e505932277e84c373da492a38d2c4ef9d96fc25f5327845 + md5: ec155b3a7172590ccbc89f461427b5aa + sha256: 4eb986655ef547d4dc72cd34b60687c9c3c390806493c15187c4d26d89d58fc0 category: main optional: false - name: dagster-webserver - version: 1.5.9 + version: 1.5.10 manager: conda platform: linux-64 dependencies: click: ">=7.0,<9.0" - dagster: ">=1.5.9,<1.5.10.0a0" - dagster-graphql: ">=1.5.9,<1.5.10.0a0" + dagster: ">=1.5.10,<1.5.11.0a0" + dagster-graphql: ">=1.5.10,<1.5.11.0a0" python: ">=3.8" starlette: "" uvicorn-standard: "" - url: https://conda.anaconda.org/conda-forge/noarch/dagster-webserver-1.5.9-pyhd8ed1ab_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/dagster-webserver-1.5.10-pyhd8ed1ab_0.conda hash: - md5: 880fa7acdbf3494cef45759bb866bb63 - sha256: 2fce08b607d97f72d7452350a0c917d96419074381bf8791ebe116ec3a57b8f4 + md5: 6161623733c03c21098dce0af904ea6b + sha256: 975173f6a39f40d5fa505354007200741689714bb1eaf1cba8e52fab1a2bfc88 category: dev optional: true - name: dagster-webserver - version: 1.5.9 + version: 1.5.10 manager: conda platform: osx-64 dependencies: @@ -4069,16 +4069,16 @@ package: uvicorn-standard: "" python: ">=3.8" click: ">=7.0,<9.0" - dagster: ">=1.5.9,<1.5.10.0a0" - dagster-graphql: ">=1.5.9,<1.5.10.0a0" - url: https://conda.anaconda.org/conda-forge/noarch/dagster-webserver-1.5.9-pyhd8ed1ab_0.conda + dagster: ">=1.5.10,<1.5.11.0a0" + dagster-graphql: ">=1.5.10,<1.5.11.0a0" + url: https://conda.anaconda.org/conda-forge/noarch/dagster-webserver-1.5.10-pyhd8ed1ab_0.conda hash: - md5: 880fa7acdbf3494cef45759bb866bb63 - sha256: 2fce08b607d97f72d7452350a0c917d96419074381bf8791ebe116ec3a57b8f4 + md5: 6161623733c03c21098dce0af904ea6b + sha256: 975173f6a39f40d5fa505354007200741689714bb1eaf1cba8e52fab1a2bfc88 category: dev optional: true - name: dagster-webserver - version: 1.5.9 + version: 1.5.10 manager: conda platform: osx-arm64 dependencies: @@ -4086,16 +4086,16 @@ package: uvicorn-standard: "" python: ">=3.8" click: ">=7.0,<9.0" - dagster: ">=1.5.9,<1.5.10.0a0" - dagster-graphql: ">=1.5.9,<1.5.10.0a0" - url: https://conda.anaconda.org/conda-forge/noarch/dagster-webserver-1.5.9-pyhd8ed1ab_0.conda + dagster: ">=1.5.10,<1.5.11.0a0" + dagster-graphql: ">=1.5.10,<1.5.11.0a0" + url: https://conda.anaconda.org/conda-forge/noarch/dagster-webserver-1.5.10-pyhd8ed1ab_0.conda hash: - md5: 880fa7acdbf3494cef45759bb866bb63 - sha256: 2fce08b607d97f72d7452350a0c917d96419074381bf8791ebe116ec3a57b8f4 + md5: 6161623733c03c21098dce0af904ea6b + sha256: 975173f6a39f40d5fa505354007200741689714bb1eaf1cba8e52fab1a2bfc88 category: dev optional: true - name: dask-core - version: 2023.11.0 + version: 2023.12.0 manager: conda platform: linux-64 dependencies: @@ -4108,14 +4108,14 @@ package: python: ">=3.9" pyyaml: ">=5.3.1" toolz: ">=0.10.0" - url: https://conda.anaconda.org/conda-forge/noarch/dask-core-2023.11.0-pyhd8ed1ab_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/dask-core-2023.12.0-pyhd8ed1ab_0.conda hash: - md5: 3bf8f5c3fbab9e0cfffdf5914f021854 - sha256: f23b4e5d8f118d9d7916d8def04dab9a299d73879216da72dd7168c1c30ecb9e + md5: 95eae0785aed72998493140dc0115382 + sha256: e366163aa7325d14ca38ca72ca4672eeb4b7a7453573c47cfa90d0db60136b48 category: main optional: false - name: dask-core - version: 2023.11.0 + version: 2023.12.0 manager: conda platform: osx-64 dependencies: @@ -4128,14 +4128,14 @@ package: importlib_metadata: ">=4.13.0" fsspec: ">=2021.09.0" click: ">=8.1" - url: https://conda.anaconda.org/conda-forge/noarch/dask-core-2023.11.0-pyhd8ed1ab_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/dask-core-2023.12.0-pyhd8ed1ab_0.conda hash: - md5: 3bf8f5c3fbab9e0cfffdf5914f021854 - sha256: f23b4e5d8f118d9d7916d8def04dab9a299d73879216da72dd7168c1c30ecb9e + md5: 95eae0785aed72998493140dc0115382 + sha256: e366163aa7325d14ca38ca72ca4672eeb4b7a7453573c47cfa90d0db60136b48 category: main optional: false - name: dask-core - version: 2023.11.0 + version: 2023.12.0 manager: conda platform: osx-arm64 dependencies: @@ -4148,10 +4148,10 @@ package: importlib_metadata: ">=4.13.0" fsspec: ">=2021.09.0" click: ">=8.1" - url: https://conda.anaconda.org/conda-forge/noarch/dask-core-2023.11.0-pyhd8ed1ab_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/dask-core-2023.12.0-pyhd8ed1ab_0.conda hash: - md5: 3bf8f5c3fbab9e0cfffdf5914f021854 - sha256: f23b4e5d8f118d9d7916d8def04dab9a299d73879216da72dd7168c1c30ecb9e + md5: 95eae0785aed72998493140dc0115382 + sha256: e366163aa7325d14ca38ca72ca4672eeb4b7a7453573c47cfa90d0db60136b48 category: main optional: false - name: dataclasses @@ -4528,8 +4528,8 @@ package: dependencies: sniffio: "" python: ">=3.8.0,<4.0.0" - cryptography: ">=2.6,<42.0" httpcore: ">=0.17.3" + cryptography: ">=2.6,<42.0" idna: ">=2.1,<4.0" url: https://conda.anaconda.org/conda-forge/noarch/dnspython-2.4.2-pyhd8ed1ab_1.conda hash: @@ -4544,8 +4544,8 @@ package: dependencies: sniffio: "" python: ">=3.8.0,<4.0.0" - cryptography: ">=2.6,<42.0" httpcore: ">=0.17.3" + cryptography: ">=2.6,<42.0" idna: ">=2.1,<4.0" url: https://conda.anaconda.org/conda-forge/noarch/dnspython-2.4.2-pyhd8ed1ab_1.conda hash: @@ -4647,10 +4647,10 @@ package: dependencies: python: ">=3.11,<3.12.0a0" python_abi: 3.11.* - url: https://conda.anaconda.org/conda-forge/linux-64/docutils-0.20.1-py311h38be061_2.conda + url: https://conda.anaconda.org/conda-forge/linux-64/docutils-0.20.1-py311h38be061_3.conda hash: - md5: 33f8066e53679dd4be2355fec849bf01 - sha256: 4e90bbc89f9ab192cb247d8b8ebe54c33e57652f8a057f9f176d9d9dd32993b9 + md5: 1c33f55e5cdcc2a2b973c432b5225bfe + sha256: 0011a2193a5995a6706936156ea5d1021153ec11eb8869b6abfe15a8f6f22ea8 category: main optional: false - name: docutils @@ -4660,10 +4660,10 @@ package: dependencies: python: ">=3.11,<3.12.0a0" python_abi: 3.11.* - url: https://conda.anaconda.org/conda-forge/osx-64/docutils-0.20.1-py311h6eed73b_2.conda + url: https://conda.anaconda.org/conda-forge/osx-64/docutils-0.20.1-py311h6eed73b_3.conda hash: - md5: d56b49f1a2c908d05d1ca6b3f85d0fd5 - sha256: 869e919066b308794e399bc551fc508c175da5f9324b7a324eb259cef8adabf2 + md5: 2919376c4957faadc7b96f8894759bfb + sha256: 0fae62e203900a8a013ba2ede852645b87b1568980ddd8e11390c11dc24c3e3c category: main optional: false - name: docutils @@ -4673,10 +4673,10 @@ package: dependencies: python: ">=3.11,<3.12.0a0" python_abi: 3.11.* - url: https://conda.anaconda.org/conda-forge/osx-arm64/docutils-0.20.1-py311h267d04e_2.conda + url: https://conda.anaconda.org/conda-forge/osx-arm64/docutils-0.20.1-py311h267d04e_3.conda hash: - md5: e82ee6e9db96d5f7ddf289399744240d - sha256: 3bc810b946ef8f87681ea4bee2610e8c418f9f61772f5d1ff3ffa803ae7cfbb6 + md5: 29944e93a6f38b2e0fd4f6b743558959 + sha256: 21af625c067faa77cd3aa2bfd8ca2449cdacdb1b531da8b4cbb476f4a928fdd9 category: main optional: false - name: email-validator @@ -5027,26 +5027,24 @@ package: manager: conda platform: linux-64 dependencies: - attrs: ">=17" - click: ">=4.0" + attrs: ">=19.2.0" + click: ">=8.0,<9.dev0" click-plugins: ">=1.0" cligj: ">=0.5" gdal: "" - importlib-metadata: "" libgcc-ng: ">=12" libgdal: ">=3.8.0,<3.9.0a0" libstdcxx-ng: ">=12" - munch: "" numpy: ">=1.23.5,<2.0a0" python: ">=3.11,<3.12.0a0" python_abi: 3.11.* setuptools: "" shapely: "" - six: ">=1.7" - url: https://conda.anaconda.org/conda-forge/linux-64/fiona-1.9.5-py311hf8e0aa6_1.conda + six: "" + url: https://conda.anaconda.org/conda-forge/linux-64/fiona-1.9.5-py311hf8e0aa6_2.conda hash: - md5: 961758d24e419de785e99b038033f9ae - sha256: 5579deb516af98c167e11b0f9250ce53e1780eade803b03ad9507fb41b295a5c + md5: 01464abc0630e2285a799a2fa8370a8e + sha256: b01852ee1d8b23276bc69829c3013cbc1fe2004917cf5c5855fe2a22674112d7 category: main optional: false - name: fiona @@ -5055,25 +5053,23 @@ package: platform: osx-64 dependencies: __osx: ">=10.9" - attrs: ">=17" - click: ">=4.0" + attrs: ">=19.2.0" + click: ">=8.0,<9.dev0" click-plugins: ">=1.0" cligj: ">=0.5" gdal: "" - importlib-metadata: "" libcxx: ">=16.0.6" libgdal: ">=3.8.0,<3.9.0a0" - munch: "" numpy: ">=1.23.5,<2.0a0" python: ">=3.11,<3.12.0a0" python_abi: 3.11.* setuptools: "" shapely: "" - six: ">=1.7" - url: https://conda.anaconda.org/conda-forge/osx-64/fiona-1.9.5-py311h809632c_1.conda + six: "" + url: https://conda.anaconda.org/conda-forge/osx-64/fiona-1.9.5-py311h809632c_2.conda hash: - md5: fa38d43ecb08f4db5107fc6390949414 - sha256: 80cfa5135122c959a7ec656c7c1c1fcc60398669029d86fac1cb9a3d3c5cacc1 + md5: cbe710fd5d800f110896f829c3e73a73 + sha256: a6d7c1d5770ad0ebfb8e4642be9837f923853bc18be1dbf94c92ca36bd814e68 category: main optional: false - name: fiona @@ -5082,29 +5078,27 @@ package: platform: osx-arm64 dependencies: __osx: ">=10.9" - attrs: ">=17" - click: ">=4.0" + attrs: ">=19.2.0" + click: ">=8.0,<9.dev0" click-plugins: ">=1.0" cligj: ">=0.5" gdal: "" - importlib-metadata: "" libcxx: ">=16.0.6" libgdal: ">=3.8.0,<3.9.0a0" - munch: "" numpy: ">=1.23.5,<2.0a0" python: ">=3.11,<3.12.0a0" python_abi: 3.11.* setuptools: "" shapely: "" - six: ">=1.7" - url: https://conda.anaconda.org/conda-forge/osx-arm64/fiona-1.9.5-py311h4760b73_1.conda + six: "" + url: https://conda.anaconda.org/conda-forge/osx-arm64/fiona-1.9.5-py311h4760b73_2.conda hash: - md5: 0232bf494596b3d10e1cf21fbcbc8615 - sha256: 9d0ad417f817677f113608aacdbac93fad06bf2a149233d299a6ada5c56c6600 + md5: 39ce132c143b433122e08c00f59c5a04 + sha256: 3c2fe9936438ca53e0720c00c384f34263b22c63f5d890bc7b4a8b0f01657b84 category: main optional: false - name: folium - version: 0.15.0 + version: 0.15.1 manager: conda platform: linux-64 dependencies: @@ -5113,42 +5107,45 @@ package: numpy: "" python: ">=3.7" requests: "" - url: https://conda.anaconda.org/conda-forge/noarch/folium-0.15.0-pyhd8ed1ab_0.conda + xyzservices: "" + url: https://conda.anaconda.org/conda-forge/noarch/folium-0.15.1-pyhd8ed1ab_0.conda hash: - md5: 25f5dbce4f946240dea7d2ee79d34254 - sha256: afe869f136fca1dbda8be0c342392fda99d951c4c4612f134a70efbf5449ef30 + md5: 4fdfc338f6fb875b1078a7e20dbc99e2 + sha256: c0730ddfaf35e39ac92b7fd07315843e7948b2ce3361d164ec1b722dd0440c2b category: main optional: false - name: folium - version: 0.15.0 + version: 0.15.1 manager: conda platform: osx-64 dependencies: numpy: "" requests: "" + xyzservices: "" python: ">=3.7" jinja2: ">=2.9" branca: ">=0.7.0" - url: https://conda.anaconda.org/conda-forge/noarch/folium-0.15.0-pyhd8ed1ab_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/folium-0.15.1-pyhd8ed1ab_0.conda hash: - md5: 25f5dbce4f946240dea7d2ee79d34254 - sha256: afe869f136fca1dbda8be0c342392fda99d951c4c4612f134a70efbf5449ef30 + md5: 4fdfc338f6fb875b1078a7e20dbc99e2 + sha256: c0730ddfaf35e39ac92b7fd07315843e7948b2ce3361d164ec1b722dd0440c2b category: main optional: false - name: folium - version: 0.15.0 + version: 0.15.1 manager: conda platform: osx-arm64 dependencies: numpy: "" requests: "" + xyzservices: "" python: ">=3.7" jinja2: ">=2.9" branca: ">=0.7.0" - url: https://conda.anaconda.org/conda-forge/noarch/folium-0.15.0-pyhd8ed1ab_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/folium-0.15.1-pyhd8ed1ab_0.conda hash: - md5: 25f5dbce4f946240dea7d2ee79d34254 - sha256: afe869f136fca1dbda8be0c342392fda99d951c4c4612f134a70efbf5449ef30 + md5: 4fdfc338f6fb875b1078a7e20dbc99e2 + sha256: c0730ddfaf35e39ac92b7fd07315843e7948b2ce3361d164ec1b722dd0440c2b category: main optional: false - name: font-ttf-dejavu-sans-mono @@ -5409,7 +5406,7 @@ package: category: main optional: false - name: fonttools - version: 4.45.1 + version: 4.46.0 manager: conda platform: linux-64 dependencies: @@ -5418,14 +5415,14 @@ package: munkres: "" python: ">=3.11,<3.12.0a0" python_abi: 3.11.* - url: https://conda.anaconda.org/conda-forge/linux-64/fonttools-4.45.1-py311h459d7ec_0.conda + url: https://conda.anaconda.org/conda-forge/linux-64/fonttools-4.46.0-py311h459d7ec_0.conda hash: - md5: 5b24692ece82f89e5cb9a469d9619731 - sha256: 57d311f86568d46f33845ea8c7d1c9e449a1fa85e510baa17f09e2cae2283681 + md5: a14114f70e23f7fd5ab9941fec45b095 + sha256: a40f8415d9ceaf5f217034814b984d13017e4dab577085a83a2d0cc39b9d7239 category: main optional: false - name: fonttools - version: 4.45.1 + version: 4.46.0 manager: conda platform: osx-64 dependencies: @@ -5433,14 +5430,14 @@ package: munkres: "" python: ">=3.11,<3.12.0a0" python_abi: 3.11.* - url: https://conda.anaconda.org/conda-forge/osx-64/fonttools-4.45.1-py311he705e18_0.conda + url: https://conda.anaconda.org/conda-forge/osx-64/fonttools-4.46.0-py311he705e18_0.conda hash: - md5: 2910a2886c556ce4085fd59d73ae96f2 - sha256: 5a60241d7585b33160c169ae59b9bd9445c89bfb4604b2d77e7a0db48c04ccc5 + md5: c68a6370b0db24b87be1a4cd7a511545 + sha256: 05c4b4fe22ce74bd5145dacf261cec44bbadac82d1e65a0b3a03d2675330f1bb category: main optional: false - name: fonttools - version: 4.45.1 + version: 4.46.0 manager: conda platform: osx-arm64 dependencies: @@ -5448,10 +5445,10 @@ package: munkres: "" python: ">=3.11,<3.12.0a0" python_abi: 3.11.* - url: https://conda.anaconda.org/conda-forge/osx-arm64/fonttools-4.45.1-py311h05b510d_0.conda + url: https://conda.anaconda.org/conda-forge/osx-arm64/fonttools-4.46.0-py311h05b510d_0.conda hash: - md5: 40c7471beb6af15161a202c0ddf04d82 - sha256: 7d8d2c8de468dc5a432e8d083f3b56eeec4380749bcfd09a44c81d42cacceece + md5: 1bac60f34700b22c8e8364f32b502211 + sha256: 300d8cc1cc987ab913e3b24c38f8eaf679acff0e885747a3f71471fd92db3e33 category: main optional: false - name: fqdn @@ -5735,39 +5732,39 @@ package: category: main optional: false - name: fsspec - version: 2023.10.0 + version: 2023.12.1 manager: conda platform: linux-64 dependencies: python: ">=3.8" - url: https://conda.anaconda.org/conda-forge/noarch/fsspec-2023.10.0-pyhca7485f_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/fsspec-2023.12.1-pyhca7485f_0.conda hash: - md5: 5b86cf1ceaaa9be2ec4627377e538db1 - sha256: 1bbdfadb93cc768252fd207dca406cde928f9a81ff985ea1760b6539c55923e6 + md5: b38946846cdf39f9bce93f75f571d913 + sha256: 929e63a5916a8ebc50199d5404fdcedf75261580d8e229d9a1def57a05ef39eb category: main optional: false - name: fsspec - version: 2023.10.0 + version: 2023.12.1 manager: conda platform: osx-64 dependencies: python: ">=3.8" - url: https://conda.anaconda.org/conda-forge/noarch/fsspec-2023.10.0-pyhca7485f_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/fsspec-2023.12.1-pyhca7485f_0.conda hash: - md5: 5b86cf1ceaaa9be2ec4627377e538db1 - sha256: 1bbdfadb93cc768252fd207dca406cde928f9a81ff985ea1760b6539c55923e6 + md5: b38946846cdf39f9bce93f75f571d913 + sha256: 929e63a5916a8ebc50199d5404fdcedf75261580d8e229d9a1def57a05ef39eb category: main optional: false - name: fsspec - version: 2023.10.0 + version: 2023.12.1 manager: conda platform: osx-arm64 dependencies: python: ">=3.8" - url: https://conda.anaconda.org/conda-forge/noarch/fsspec-2023.10.0-pyhca7485f_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/fsspec-2023.12.1-pyhca7485f_0.conda hash: - md5: 5b86cf1ceaaa9be2ec4627377e538db1 - sha256: 1bbdfadb93cc768252fd207dca406cde928f9a81ff985ea1760b6539c55923e6 + md5: b38946846cdf39f9bce93f75f571d913 + sha256: 929e63a5916a8ebc50199d5404fdcedf75261580d8e229d9a1def57a05ef39eb category: main optional: false - name: furo @@ -5819,26 +5816,26 @@ package: category: main optional: false - name: gcsfs - version: 2023.10.0 + version: 2023.12.1 manager: conda platform: linux-64 dependencies: aiohttp: "" decorator: ">4.1.2" - fsspec: 2023.10.0 + fsspec: 2023.12.1 google-auth: ">=1.2" google-auth-oauthlib: "" google-cloud-storage: ">1.40" python: ">=3.7" requests: "" - url: https://conda.anaconda.org/conda-forge/noarch/gcsfs-2023.10.0-pyhd8ed1ab_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/gcsfs-2023.12.1-pyhd8ed1ab_0.conda hash: - md5: 500521931bdcc0f6d19c1c2e2ab4a5d9 - sha256: dd7559c5297359e475a125742e9cb30938579e93a17ce7537af64a04c98407a5 + md5: 2f48942fbcd4c2ef56af0fbf3cc56fb0 + sha256: b5e02b08137b4e90d97f4f2d5ff3e06d7076cdba6873061ca2b346576923b093 category: main optional: false - name: gcsfs - version: 2023.10.0 + version: 2023.12.1 manager: conda platform: osx-64 dependencies: @@ -5849,15 +5846,15 @@ package: google-auth: ">=1.2" decorator: ">4.1.2" google-cloud-storage: ">1.40" - fsspec: 2023.10.0 - url: https://conda.anaconda.org/conda-forge/noarch/gcsfs-2023.10.0-pyhd8ed1ab_0.conda + fsspec: 2023.12.1 + url: https://conda.anaconda.org/conda-forge/noarch/gcsfs-2023.12.1-pyhd8ed1ab_0.conda hash: - md5: 500521931bdcc0f6d19c1c2e2ab4a5d9 - sha256: dd7559c5297359e475a125742e9cb30938579e93a17ce7537af64a04c98407a5 + md5: 2f48942fbcd4c2ef56af0fbf3cc56fb0 + sha256: b5e02b08137b4e90d97f4f2d5ff3e06d7076cdba6873061ca2b346576923b093 category: main optional: false - name: gcsfs - version: 2023.10.0 + version: 2023.12.1 manager: conda platform: osx-arm64 dependencies: @@ -5868,11 +5865,11 @@ package: google-auth: ">=1.2" decorator: ">4.1.2" google-cloud-storage: ">1.40" - fsspec: 2023.10.0 - url: https://conda.anaconda.org/conda-forge/noarch/gcsfs-2023.10.0-pyhd8ed1ab_0.conda + fsspec: 2023.12.1 + url: https://conda.anaconda.org/conda-forge/noarch/gcsfs-2023.12.1-pyhd8ed1ab_0.conda hash: - md5: 500521931bdcc0f6d19c1c2e2ab4a5d9 - sha256: dd7559c5297359e475a125742e9cb30938579e93a17ce7537af64a04c98407a5 + md5: 2f48942fbcd4c2ef56af0fbf3cc56fb0 + sha256: b5e02b08137b4e90d97f4f2d5ff3e06d7076cdba6873061ca2b346576923b093 category: main optional: false - name: gdal @@ -5880,7 +5877,7 @@ package: manager: conda platform: linux-64 dependencies: - hdf5: ">=1.14.2,<1.14.3.0a0" + hdf5: ">=1.14.2,<1.14.4.0a0" libgcc-ng: ">=12" libgdal: 3.8.0 libstdcxx-ng: ">=12" @@ -5901,7 +5898,7 @@ package: platform: osx-64 dependencies: __osx: ">=10.9" - hdf5: ">=1.14.2,<1.14.3.0a0" + hdf5: ">=1.14.2,<1.14.4.0a0" libcxx: ">=16.0.6" libgdal: 3.8.0 libxml2: ">=2.11.6,<2.12.0a0" @@ -5921,7 +5918,7 @@ package: platform: osx-arm64 dependencies: __osx: ">=10.9" - hdf5: ">=1.14.2,<1.14.3.0a0" + hdf5: ">=1.14.2,<1.14.4.0a0" libcxx: ">=16.0.6" libgdal: 3.8.0 libxml2: ">=2.11.6,<2.12.0a0" @@ -6422,7 +6419,7 @@ package: category: main optional: false - name: google-api-core - version: 2.14.0 + version: 2.15.0 manager: conda platform: linux-64 dependencies: @@ -6431,14 +6428,14 @@ package: protobuf: ">=3.19.5,<5.0.0.dev0,!=3.20.0,!=3.20.1,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5" python: ">=3.7" requests: ">=2.18.0,<3.0.0.dev0" - url: https://conda.anaconda.org/conda-forge/noarch/google-api-core-2.14.0-pyhd8ed1ab_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/google-api-core-2.15.0-pyhd8ed1ab_0.conda hash: - md5: cebe18c719a6818849b921748aa91750 - sha256: 4bc666e51fe40266435b8e8a4137e47278e044ca26be34c05260236552914ebc + md5: e132b54eb1a75c1e536edfbb5ee7684c + sha256: ffc427bab6dcb6c6282046ebc17d2ff6918ebe55789d77256d28f85611e32f09 category: main optional: false - name: google-api-core - version: 2.14.0 + version: 2.15.0 manager: conda platform: osx-64 dependencies: @@ -6447,14 +6444,14 @@ package: googleapis-common-protos: ">=1.56.2,<2.0.dev0" protobuf: ">=3.19.5,<5.0.0.dev0,!=3.20.0,!=3.20.1,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5" requests: ">=2.18.0,<3.0.0.dev0" - url: https://conda.anaconda.org/conda-forge/noarch/google-api-core-2.14.0-pyhd8ed1ab_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/google-api-core-2.15.0-pyhd8ed1ab_0.conda hash: - md5: cebe18c719a6818849b921748aa91750 - sha256: 4bc666e51fe40266435b8e8a4137e47278e044ca26be34c05260236552914ebc + md5: e132b54eb1a75c1e536edfbb5ee7684c + sha256: ffc427bab6dcb6c6282046ebc17d2ff6918ebe55789d77256d28f85611e32f09 category: main optional: false - name: google-api-core - version: 2.14.0 + version: 2.15.0 manager: conda platform: osx-arm64 dependencies: @@ -6463,14 +6460,14 @@ package: googleapis-common-protos: ">=1.56.2,<2.0.dev0" protobuf: ">=3.19.5,<5.0.0.dev0,!=3.20.0,!=3.20.1,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5" requests: ">=2.18.0,<3.0.0.dev0" - url: https://conda.anaconda.org/conda-forge/noarch/google-api-core-2.14.0-pyhd8ed1ab_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/google-api-core-2.15.0-pyhd8ed1ab_0.conda hash: - md5: cebe18c719a6818849b921748aa91750 - sha256: 4bc666e51fe40266435b8e8a4137e47278e044ca26be34c05260236552914ebc + md5: e132b54eb1a75c1e536edfbb5ee7684c + sha256: ffc427bab6dcb6c6282046ebc17d2ff6918ebe55789d77256d28f85611e32f09 category: main optional: false - name: google-auth - version: 2.24.0 + version: 2.25.1 manager: conda platform: linux-64 dependencies: @@ -6483,14 +6480,14 @@ package: pyu2f: ">=0.1.5" requests: ">=2.20.0,<3.0.0" rsa: ">=3.1.4,<5" - url: https://conda.anaconda.org/conda-forge/noarch/google-auth-2.24.0-pyhca7485f_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/google-auth-2.25.1-pyhca7485f_0.conda hash: - md5: 5c80374ea4c24d3bd6822108d43715d0 - sha256: c270d1866bd01f3fa97e5c65496b853af6b1c8d58479132c8b3397534fbb2919 + md5: ea4120e492a1b82f298419e0f2210a1e + sha256: 4626124ab555cd8620ff9ff0cd4e1e2f8a161667c00c1571241f9fc722b0b8bd category: main optional: false - name: google-auth - version: 2.24.0 + version: 2.25.1 manager: conda platform: osx-64 dependencies: @@ -6503,14 +6500,14 @@ package: cachetools: ">=2.0.0,<6.0" aiohttp: ">=3.6.2,<4.0.0" cryptography: ">=38.0.3" - url: https://conda.anaconda.org/conda-forge/noarch/google-auth-2.24.0-pyhca7485f_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/google-auth-2.25.1-pyhca7485f_0.conda hash: - md5: 5c80374ea4c24d3bd6822108d43715d0 - sha256: c270d1866bd01f3fa97e5c65496b853af6b1c8d58479132c8b3397534fbb2919 + md5: ea4120e492a1b82f298419e0f2210a1e + sha256: 4626124ab555cd8620ff9ff0cd4e1e2f8a161667c00c1571241f9fc722b0b8bd category: main optional: false - name: google-auth - version: 2.24.0 + version: 2.25.1 manager: conda platform: osx-arm64 dependencies: @@ -6523,10 +6520,10 @@ package: cachetools: ">=2.0.0,<6.0" aiohttp: ">=3.6.2,<4.0.0" cryptography: ">=38.0.3" - url: https://conda.anaconda.org/conda-forge/noarch/google-auth-2.24.0-pyhca7485f_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/google-auth-2.25.1-pyhca7485f_0.conda hash: - md5: 5c80374ea4c24d3bd6822108d43715d0 - sha256: c270d1866bd01f3fa97e5c65496b853af6b1c8d58479132c8b3397534fbb2919 + md5: ea4120e492a1b82f298419e0f2210a1e + sha256: 4626124ab555cd8620ff9ff0cd4e1e2f8a161667c00c1571241f9fc722b0b8bd category: main optional: false - name: google-auth-oauthlib @@ -6575,87 +6572,87 @@ package: category: main optional: false - name: google-cloud-core - version: 2.3.3 + version: 2.4.1 manager: conda platform: linux-64 dependencies: google-api-core: ">=1.31.6,<3.0.0dev,!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0" google-auth: ">=1.25.0,<3.0dev" grpcio: ">=1.38.0,<2.0.0dev" - python: ">=3.7" - url: https://conda.anaconda.org/conda-forge/noarch/google-cloud-core-2.3.3-pyhd8ed1ab_0.conda + python: ">=3.8" + url: https://conda.anaconda.org/conda-forge/noarch/google-cloud-core-2.4.1-pyhd8ed1ab_0.conda hash: - md5: a26b1fa8555cc1d2f0f7ff9985303e66 - sha256: e8a840361b23ca7a9cfa62c1885fc66aa5ad94e48556782e9a032678c9f4b76e + md5: 1853cdebbfe25fb6ee253855a44945a6 + sha256: d01b787bad2ec4da9536ce2cedb3e53ed092fe6a4a596c043ab358bb9b2fbcdd category: main optional: false - name: google-cloud-core - version: 2.3.3 + version: 2.4.1 manager: conda platform: osx-64 dependencies: - python: ">=3.7" + python: ">=3.8" google-auth: ">=1.25.0,<3.0dev" google-api-core: ">=1.31.6,<3.0.0dev,!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0" grpcio: ">=1.38.0,<2.0.0dev" - url: https://conda.anaconda.org/conda-forge/noarch/google-cloud-core-2.3.3-pyhd8ed1ab_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/google-cloud-core-2.4.1-pyhd8ed1ab_0.conda hash: - md5: a26b1fa8555cc1d2f0f7ff9985303e66 - sha256: e8a840361b23ca7a9cfa62c1885fc66aa5ad94e48556782e9a032678c9f4b76e + md5: 1853cdebbfe25fb6ee253855a44945a6 + sha256: d01b787bad2ec4da9536ce2cedb3e53ed092fe6a4a596c043ab358bb9b2fbcdd category: main optional: false - name: google-cloud-core - version: 2.3.3 + version: 2.4.1 manager: conda platform: osx-arm64 dependencies: - python: ">=3.7" + python: ">=3.8" google-auth: ">=1.25.0,<3.0dev" google-api-core: ">=1.31.6,<3.0.0dev,!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0" grpcio: ">=1.38.0,<2.0.0dev" - url: https://conda.anaconda.org/conda-forge/noarch/google-cloud-core-2.3.3-pyhd8ed1ab_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/google-cloud-core-2.4.1-pyhd8ed1ab_0.conda hash: - md5: a26b1fa8555cc1d2f0f7ff9985303e66 - sha256: e8a840361b23ca7a9cfa62c1885fc66aa5ad94e48556782e9a032678c9f4b76e + md5: 1853cdebbfe25fb6ee253855a44945a6 + sha256: d01b787bad2ec4da9536ce2cedb3e53ed092fe6a4a596c043ab358bb9b2fbcdd category: main optional: false - name: google-cloud-sdk - version: 455.0.0 + version: 456.0.0 manager: conda platform: linux-64 dependencies: python: ">=3.11,<3.12.0a0" python_abi: 3.11.* - url: https://conda.anaconda.org/conda-forge/linux-64/google-cloud-sdk-455.0.0-py311h38be061_0.conda + url: https://conda.anaconda.org/conda-forge/linux-64/google-cloud-sdk-456.0.0-py311h38be061_0.conda hash: - md5: 1505ce6a9284c331e05de23b56d023ff - sha256: 17c439e5b01341a4aae55a2f1841878244d25b365cef52b39fb9bfd3e30c8315 + md5: 09eb149adf3420350bd241bda3d5fafe + sha256: f789db836de3cc6557c9250f5819ada0aaafa2f559c09b8fadd5bda703a2acdf category: main optional: false - name: google-cloud-sdk - version: 455.0.0 + version: 456.0.0 manager: conda platform: osx-64 dependencies: python: ">=3.11,<3.12.0a0" python_abi: 3.11.* - url: https://conda.anaconda.org/conda-forge/osx-64/google-cloud-sdk-455.0.0-py311h6eed73b_0.conda + url: https://conda.anaconda.org/conda-forge/osx-64/google-cloud-sdk-456.0.0-py311h6eed73b_0.conda hash: - md5: 0a635aa75ccc84e4dd16e06b559d3d49 - sha256: 8e133ed925ed75409a354b564ff2ebc2ebb3ebdd659f2d190b4c198b164c6f8e + md5: 287686faa4f60a48dae2884822a232ef + sha256: 9ce703e2c3b9df7f01dee5e21c3b40da1088114670ced4347af9ee368f304d4b category: main optional: false - name: google-cloud-sdk - version: 455.0.0 + version: 456.0.0 manager: conda platform: osx-arm64 dependencies: python: ">=3.11,<3.12.0a0" python_abi: 3.11.* - url: https://conda.anaconda.org/conda-forge/osx-arm64/google-cloud-sdk-455.0.0-py311h267d04e_0.conda + url: https://conda.anaconda.org/conda-forge/osx-arm64/google-cloud-sdk-456.0.0-py311h267d04e_0.conda hash: - md5: 2f60b4b18d39e85bdf3557f19bd407be - sha256: 9511a6c98a01a1e5013c73d8e7cb0d1200643e9d531cbc49ebebfb5cd9e71f27 + md5: 321f57034a4eb43fbe44c09d5a270eeb + sha256: 89e0793bafe2d57cf3c656bc22853f7a40eb05b03a34d6c308dcd5ff572ace04 category: main optional: false - name: google-cloud-storage @@ -6801,42 +6798,42 @@ package: category: main optional: false - name: googleapis-common-protos - version: 1.61.0 + version: 1.62.0 manager: conda platform: linux-64 dependencies: protobuf: ">=3.19.5,<5.0.0dev0,!=3.20.0,!=3.20.1,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5" python: ">=3.7" - url: https://conda.anaconda.org/conda-forge/noarch/googleapis-common-protos-1.61.0-pyhd8ed1ab_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/googleapis-common-protos-1.62.0-pyhd8ed1ab_0.conda hash: - md5: f315d7fdc1905dcc2e18a1c7bed22fa9 - sha256: aa25665205e8d4895ff1bf042d2fc358a20c207271238069e13b87535f92184e + md5: ca3d0c7ba3a15e943d9c715aba03ae62 + sha256: 70da3fc08a742022c666d9807f0caba60be1ddbf09b6642c168001bace18c724 category: main optional: false - name: googleapis-common-protos - version: 1.61.0 + version: 1.62.0 manager: conda platform: osx-64 dependencies: python: ">=3.7" protobuf: ">=3.19.5,<5.0.0dev0,!=3.20.0,!=3.20.1,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5" - url: https://conda.anaconda.org/conda-forge/noarch/googleapis-common-protos-1.61.0-pyhd8ed1ab_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/googleapis-common-protos-1.62.0-pyhd8ed1ab_0.conda hash: - md5: f315d7fdc1905dcc2e18a1c7bed22fa9 - sha256: aa25665205e8d4895ff1bf042d2fc358a20c207271238069e13b87535f92184e + md5: ca3d0c7ba3a15e943d9c715aba03ae62 + sha256: 70da3fc08a742022c666d9807f0caba60be1ddbf09b6642c168001bace18c724 category: main optional: false - name: googleapis-common-protos - version: 1.61.0 + version: 1.62.0 manager: conda platform: osx-arm64 dependencies: python: ">=3.7" protobuf: ">=3.19.5,<5.0.0dev0,!=3.20.0,!=3.20.1,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5" - url: https://conda.anaconda.org/conda-forge/noarch/googleapis-common-protos-1.61.0-pyhd8ed1ab_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/googleapis-common-protos-1.62.0-pyhd8ed1ab_0.conda hash: - md5: f315d7fdc1905dcc2e18a1c7bed22fa9 - sha256: aa25665205e8d4895ff1bf042d2fc358a20c207271238069e13b87535f92184e + md5: ca3d0c7ba3a15e943d9c715aba03ae62 + sha256: 70da3fc08a742022c666d9807f0caba60be1ddbf09b6642c168001bace18c724 category: main optional: false - name: gql @@ -8145,42 +8142,42 @@ package: category: main optional: false - name: identify - version: 2.5.32 + version: 2.5.33 manager: conda platform: linux-64 dependencies: python: ">=3.6" ukkonen: "" - url: https://conda.anaconda.org/conda-forge/noarch/identify-2.5.32-pyhd8ed1ab_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/identify-2.5.33-pyhd8ed1ab_0.conda hash: - md5: 3ef8e9bab1bfaf900bb0a5db8c0c742c - sha256: 0783aa58f43d1c113a2ec300a29ba3313184056f9893671c75037fbadaf9e546 + md5: 93c8f8ceb83827d88deeba796f07fba7 + sha256: ce2a64c18221af96226be23278d81f22ff9f64b3c047d8865590f6718915303f category: main optional: false - name: identify - version: 2.5.32 + version: 2.5.33 manager: conda platform: osx-64 dependencies: ukkonen: "" python: ">=3.6" - url: https://conda.anaconda.org/conda-forge/noarch/identify-2.5.32-pyhd8ed1ab_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/identify-2.5.33-pyhd8ed1ab_0.conda hash: - md5: 3ef8e9bab1bfaf900bb0a5db8c0c742c - sha256: 0783aa58f43d1c113a2ec300a29ba3313184056f9893671c75037fbadaf9e546 + md5: 93c8f8ceb83827d88deeba796f07fba7 + sha256: ce2a64c18221af96226be23278d81f22ff9f64b3c047d8865590f6718915303f category: main optional: false - name: identify - version: 2.5.32 + version: 2.5.33 manager: conda platform: osx-arm64 dependencies: ukkonen: "" python: ">=3.6" - url: https://conda.anaconda.org/conda-forge/noarch/identify-2.5.32-pyhd8ed1ab_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/identify-2.5.33-pyhd8ed1ab_0.conda hash: - md5: 3ef8e9bab1bfaf900bb0a5db8c0c742c - sha256: 0783aa58f43d1c113a2ec300a29ba3313184056f9893671c75037fbadaf9e546 + md5: 93c8f8ceb83827d88deeba796f07fba7 + sha256: ce2a64c18221af96226be23278d81f22ff9f64b3c047d8865590f6718915303f category: main optional: false - name: idna @@ -8292,78 +8289,78 @@ package: category: main optional: false - name: importlib-metadata - version: 6.8.0 + version: 7.0.0 manager: conda platform: linux-64 dependencies: python: ">=3.8" zipp: ">=0.5" - url: https://conda.anaconda.org/conda-forge/noarch/importlib-metadata-6.8.0-pyha770c72_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/importlib-metadata-7.0.0-pyha770c72_0.conda hash: - md5: 4e9f59a060c3be52bc4ddc46ee9b6946 - sha256: 2797ed927d65324309b6c630190d917b9f2111e0c217b721f80429aeb57f9fcf + md5: a941237cd06538837b25cd245fcd25d8 + sha256: 9731e82a00d36b182dc515e31723e711ac82890bb1ca86c6a17a4b471135564f category: main optional: false - name: importlib-metadata - version: 6.8.0 + version: 7.0.0 manager: conda platform: osx-64 dependencies: python: ">=3.8" zipp: ">=0.5" - url: https://conda.anaconda.org/conda-forge/noarch/importlib-metadata-6.8.0-pyha770c72_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/importlib-metadata-7.0.0-pyha770c72_0.conda hash: - md5: 4e9f59a060c3be52bc4ddc46ee9b6946 - sha256: 2797ed927d65324309b6c630190d917b9f2111e0c217b721f80429aeb57f9fcf + md5: a941237cd06538837b25cd245fcd25d8 + sha256: 9731e82a00d36b182dc515e31723e711ac82890bb1ca86c6a17a4b471135564f category: main optional: false - name: importlib-metadata - version: 6.8.0 + version: 7.0.0 manager: conda platform: osx-arm64 dependencies: python: ">=3.8" zipp: ">=0.5" - url: https://conda.anaconda.org/conda-forge/noarch/importlib-metadata-6.8.0-pyha770c72_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/importlib-metadata-7.0.0-pyha770c72_0.conda hash: - md5: 4e9f59a060c3be52bc4ddc46ee9b6946 - sha256: 2797ed927d65324309b6c630190d917b9f2111e0c217b721f80429aeb57f9fcf + md5: a941237cd06538837b25cd245fcd25d8 + sha256: 9731e82a00d36b182dc515e31723e711ac82890bb1ca86c6a17a4b471135564f category: main optional: false - name: importlib_metadata - version: 6.8.0 + version: 7.0.0 manager: conda platform: linux-64 dependencies: - importlib-metadata: ">=6.8.0,<6.8.1.0a0" - url: https://conda.anaconda.org/conda-forge/noarch/importlib_metadata-6.8.0-hd8ed1ab_0.conda + importlib-metadata: ">=7.0.0,<7.0.1.0a0" + url: https://conda.anaconda.org/conda-forge/noarch/importlib_metadata-7.0.0-hd8ed1ab_0.conda hash: - md5: b279b07ce18058034e5b3606ba103a8b - sha256: b96e01dc42d547d6d9ceb1c5b52a5232cc04e40153534350f702c3e0418a6b3f + md5: 12aff14f84c337be5e5636bf612f4140 + sha256: b9e8ed41df6c55222e3777f422e77a22a6a19ff779b2e65aa8dfdea792c1f7de category: main optional: false - name: importlib_metadata - version: 6.8.0 + version: 7.0.0 manager: conda platform: osx-64 dependencies: - importlib-metadata: ">=6.8.0,<6.8.1.0a0" - url: https://conda.anaconda.org/conda-forge/noarch/importlib_metadata-6.8.0-hd8ed1ab_0.conda + importlib-metadata: ">=7.0.0,<7.0.1.0a0" + url: https://conda.anaconda.org/conda-forge/noarch/importlib_metadata-7.0.0-hd8ed1ab_0.conda hash: - md5: b279b07ce18058034e5b3606ba103a8b - sha256: b96e01dc42d547d6d9ceb1c5b52a5232cc04e40153534350f702c3e0418a6b3f + md5: 12aff14f84c337be5e5636bf612f4140 + sha256: b9e8ed41df6c55222e3777f422e77a22a6a19ff779b2e65aa8dfdea792c1f7de category: main optional: false - name: importlib_metadata - version: 6.8.0 + version: 7.0.0 manager: conda platform: osx-arm64 dependencies: - importlib-metadata: ">=6.8.0,<6.8.1.0a0" - url: https://conda.anaconda.org/conda-forge/noarch/importlib_metadata-6.8.0-hd8ed1ab_0.conda + importlib-metadata: ">=7.0.0,<7.0.1.0a0" + url: https://conda.anaconda.org/conda-forge/noarch/importlib_metadata-7.0.0-hd8ed1ab_0.conda hash: - md5: b279b07ce18058034e5b3606ba103a8b - sha256: b96e01dc42d547d6d9ceb1c5b52a5232cc04e40153534350f702c3e0418a6b3f + md5: 12aff14f84c337be5e5636bf612f4140 + sha256: b9e8ed41df6c55222e3777f422e77a22a6a19ff779b2e65aa8dfdea792c1f7de category: main optional: false - name: importlib_resources @@ -8530,16 +8527,16 @@ package: matplotlib-inline: "" pexpect: ">4.3" pickleshare: "" - prompt-toolkit: ">=3.0.30,<3.1.0,!=3.0.37" + prompt-toolkit: ">=3.0.41,<3.1.0" pygments: ">=2.4.0" python: ">=3.9" stack_data: "" traitlets: ">=5" typing_extensions: "" - url: https://conda.anaconda.org/conda-forge/noarch/ipython-8.18.1-pyh31011fe_1.conda + url: https://conda.anaconda.org/conda-forge/noarch/ipython-8.18.1-pyh707e725_3.conda hash: - md5: ac2f9c2e10c2e90e8d135cef51f9753a - sha256: 67490e640faa372d663a5c5cd2d61f417cce22a019a4de82a9e5ddb1cf2ee181 + md5: 15c6f45a45f7ac27f6d60b0b084f6761 + sha256: d98d615ac8ad71de698afbc50e8269570d4b89706821c4ff3058a4ceec69bd9b category: main optional: false - name: ipython @@ -8559,11 +8556,11 @@ package: traitlets: ">=5" jedi: ">=0.16" pexpect: ">4.3" - prompt-toolkit: ">=3.0.30,<3.1.0,!=3.0.37" - url: https://conda.anaconda.org/conda-forge/noarch/ipython-8.18.1-pyh31011fe_1.conda + prompt-toolkit: ">=3.0.41,<3.1.0" + url: https://conda.anaconda.org/conda-forge/noarch/ipython-8.18.1-pyh707e725_3.conda hash: - md5: ac2f9c2e10c2e90e8d135cef51f9753a - sha256: 67490e640faa372d663a5c5cd2d61f417cce22a019a4de82a9e5ddb1cf2ee181 + md5: 15c6f45a45f7ac27f6d60b0b084f6761 + sha256: d98d615ac8ad71de698afbc50e8269570d4b89706821c4ff3058a4ceec69bd9b category: main optional: false - name: ipython @@ -8583,11 +8580,11 @@ package: traitlets: ">=5" jedi: ">=0.16" pexpect: ">4.3" - prompt-toolkit: ">=3.0.30,<3.1.0,!=3.0.37" - url: https://conda.anaconda.org/conda-forge/noarch/ipython-8.18.1-pyh31011fe_1.conda + prompt-toolkit: ">=3.0.41,<3.1.0" + url: https://conda.anaconda.org/conda-forge/noarch/ipython-8.18.1-pyh707e725_3.conda hash: - md5: ac2f9c2e10c2e90e8d135cef51f9753a - sha256: 67490e640faa372d663a5c5cd2d61f417cce22a019a4de82a9e5ddb1cf2ee181 + md5: 15c6f45a45f7ac27f6d60b0b084f6761 + sha256: d98d615ac8ad71de698afbc50e8269570d4b89706821c4ff3058a4ceec69bd9b category: main optional: false - name: ipywidgets @@ -9373,8 +9370,8 @@ package: dependencies: ipywidgets: "" notebook: "" - ipykernel: "" nbconvert: "" + ipykernel: "" qtconsole-base: "" jupyter_console: "" python: ">=3.6" @@ -9391,8 +9388,8 @@ package: dependencies: ipywidgets: "" notebook: "" - ipykernel: "" nbconvert: "" + ipykernel: "" qtconsole-base: "" jupyter_console: "" python: ">=3.6" @@ -9706,7 +9703,7 @@ package: category: main optional: false - name: jupyter_server - version: 2.11.1 + version: 2.12.1 manager: conda platform: linux-64 dependencies: @@ -9729,14 +9726,14 @@ package: tornado: ">=6.2.0" traitlets: ">=5.6.0" websocket-client: "" - url: https://conda.anaconda.org/conda-forge/noarch/jupyter_server-2.11.1-pyhd8ed1ab_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/jupyter_server-2.12.1-pyhd8ed1ab_0.conda hash: - md5: 0699b715659c026f7f81c27d0e744205 - sha256: 605825c0e2d5af7935b37319b9a46ff39e081e7a0f4dc973f0dd583f41c69ce5 + md5: e9781be1e6c93b5df2c180a9f9242420 + sha256: c4aabe2041afb8fde1f049549c2e292265612d07dd4d1156f3e183ba6a6f007b category: main optional: false - name: jupyter_server - version: 2.11.1 + version: 2.12.1 manager: conda platform: osx-64 dependencies: @@ -9759,14 +9756,14 @@ package: anyio: ">=3.1.0" send2trash: ">=1.8.2" jupyter_events: ">=0.9.0" - url: https://conda.anaconda.org/conda-forge/noarch/jupyter_server-2.11.1-pyhd8ed1ab_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/jupyter_server-2.12.1-pyhd8ed1ab_0.conda hash: - md5: 0699b715659c026f7f81c27d0e744205 - sha256: 605825c0e2d5af7935b37319b9a46ff39e081e7a0f4dc973f0dd583f41c69ce5 + md5: e9781be1e6c93b5df2c180a9f9242420 + sha256: c4aabe2041afb8fde1f049549c2e292265612d07dd4d1156f3e183ba6a6f007b category: main optional: false - name: jupyter_server - version: 2.11.1 + version: 2.12.1 manager: conda platform: osx-arm64 dependencies: @@ -9789,10 +9786,10 @@ package: anyio: ">=3.1.0" send2trash: ">=1.8.2" jupyter_events: ">=0.9.0" - url: https://conda.anaconda.org/conda-forge/noarch/jupyter_server-2.11.1-pyhd8ed1ab_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/jupyter_server-2.12.1-pyhd8ed1ab_0.conda hash: - md5: 0699b715659c026f7f81c27d0e744205 - sha256: 605825c0e2d5af7935b37319b9a46ff39e081e7a0f4dc973f0dd583f41c69ce5 + md5: e9781be1e6c93b5df2c180a9f9242420 + sha256: c4aabe2041afb8fde1f049549c2e292265612d07dd4d1156f3e183ba6a6f007b category: main optional: false - name: jupyter_server_terminals @@ -9871,8 +9868,8 @@ package: ipykernel: "" jupyter_core: "" python: ">=3.8" - jinja2: ">=3.0.3" tornado: ">=6.2.0" + jinja2: ">=3.0.3" importlib_metadata: ">=4.8.3" jupyter_server: ">=2.4.0,<3" importlib_resources: ">=1.4" @@ -9897,8 +9894,8 @@ package: ipykernel: "" jupyter_core: "" python: ">=3.8" - jinja2: ">=3.0.3" tornado: ">=6.2.0" + jinja2: ">=3.0.3" importlib_metadata: ">=4.8.3" jupyter_server: ">=2.4.0,<3" importlib_resources: ">=1.4" @@ -10052,7 +10049,7 @@ package: manager: conda platform: linux-64 dependencies: - hdf5: ">=1.14.2,<1.14.3.0a0" + hdf5: ">=1.14.2,<1.14.4.0a0" libgcc-ng: ">=12" libstdcxx-ng: ">=12" url: https://conda.anaconda.org/conda-forge/linux-64/kealib-1.5.2-hcd42e92_1.conda @@ -10066,7 +10063,7 @@ package: manager: conda platform: osx-64 dependencies: - hdf5: ">=1.14.2,<1.14.3.0a0" + hdf5: ">=1.14.2,<1.14.4.0a0" libcxx: ">=15.0.7" url: https://conda.anaconda.org/conda-forge/osx-64/kealib-1.5.2-h052fcf7_1.conda hash: @@ -10079,7 +10076,7 @@ package: manager: conda platform: osx-arm64 dependencies: - hdf5: ">=1.14.2,<1.14.3.0a0" + hdf5: ">=1.14.2,<1.14.4.0a0" libcxx: ">=15.0.7" url: https://conda.anaconda.org/conda-forge/osx-arm64/kealib-1.5.2-h47b5e36_1.conda hash: @@ -10273,43 +10270,43 @@ package: category: main optional: false - name: lcms2 - version: "2.15" + version: "2.16" manager: conda platform: linux-64 dependencies: libgcc-ng: ">=12" libjpeg-turbo: ">=3.0.0,<4.0a0" libtiff: ">=4.6.0,<4.7.0a0" - url: https://conda.anaconda.org/conda-forge/linux-64/lcms2-2.15-hb7c19ff_3.conda + url: https://conda.anaconda.org/conda-forge/linux-64/lcms2-2.16-hb7c19ff_0.conda hash: - md5: e96637dd92c5f340215c753a5c9a22d7 - sha256: cc0b2ddab52b20698b26fe8622ebe37e0d462d8691a1f324e7b00f7d904765e3 + md5: 51bb7010fc86f70eee639b4bb7a894f5 + sha256: 5c878d104b461b7ef922abe6320711c0d01772f4cd55de18b674f88547870041 category: main optional: false - name: lcms2 - version: "2.15" + version: "2.16" manager: conda platform: osx-64 dependencies: libjpeg-turbo: ">=3.0.0,<4.0a0" libtiff: ">=4.6.0,<4.7.0a0" - url: https://conda.anaconda.org/conda-forge/osx-64/lcms2-2.15-hd6ba6f3_3.conda + url: https://conda.anaconda.org/conda-forge/osx-64/lcms2-2.16-ha2f27b4_0.conda hash: - md5: 8059507d52f477fbd4b81841e085e25b - sha256: b2234f24e3b0030762430ec3414410119d1129804a95ef65af50ad36cabd9bd5 + md5: 1442db8f03517834843666c422238c9b + sha256: 222ebc0a55544b9922f61e75015d02861e65b48f12113af41d48ba0814e14e4e category: main optional: false - name: lcms2 - version: "2.15" + version: "2.16" manager: conda platform: osx-arm64 dependencies: libjpeg-turbo: ">=3.0.0,<4.0a0" libtiff: ">=4.6.0,<4.7.0a0" - url: https://conda.anaconda.org/conda-forge/osx-arm64/lcms2-2.15-hf2736f0_3.conda + url: https://conda.anaconda.org/conda-forge/osx-arm64/lcms2-2.16-ha0e7c42_0.conda hash: - md5: bbaac531169fed3e09ae31aff80aa069 - sha256: 3d07ba04602617c3084b302c8a6fa30b2e4b20511180f45992b289c312298018 + md5: 66f6c134e76fe13cce8a9ea5814b5dd5 + sha256: 151e0c84feb7e0747fabcc85006b8973b22f5abbc3af76a9add0b0ef0320ebe4 category: main optional: false - name: ld_impl_linux-64 @@ -11514,7 +11511,7 @@ package: geotiff: ">=1.7.1,<1.8.0a0" giflib: ">=5.2.1,<5.3.0a0" hdf4: ">=4.2.15,<4.2.16.0a0" - hdf5: ">=1.14.2,<1.14.3.0a0" + hdf5: ">=1.14.2,<1.14.4.0a0" json-c: ">=0.17,<0.18.0a0" kealib: ">=1.5.2,<1.6.0a0" lerc: ">=4.0.0,<5.0a0" @@ -11568,7 +11565,7 @@ package: geotiff: ">=1.7.1,<1.8.0a0" giflib: ">=5.2.1,<5.3.0a0" hdf4: ">=4.2.15,<4.2.16.0a0" - hdf5: ">=1.14.2,<1.14.3.0a0" + hdf5: ">=1.14.2,<1.14.4.0a0" json-c: ">=0.17,<0.18.0a0" kealib: ">=1.5.2,<1.6.0a0" lerc: ">=4.0.0,<5.0a0" @@ -11620,7 +11617,7 @@ package: geotiff: ">=1.7.1,<1.8.0a0" giflib: ">=5.2.1,<5.3.0a0" hdf4: ">=4.2.15,<4.2.16.0a0" - hdf5: ">=1.14.2,<1.14.3.0a0" + hdf5: ">=1.14.2,<1.14.4.0a0" json-c: ">=0.17,<0.18.0a0" kealib: ">=1.5.2,<1.6.0a0" lerc: ">=4.0.0,<5.0a0" @@ -11732,7 +11729,7 @@ package: category: main optional: false - name: libglib - version: 2.78.1 + version: 2.78.3 manager: conda platform: linux-64 dependencies: @@ -11743,14 +11740,14 @@ package: libstdcxx-ng: ">=12" libzlib: ">=1.2.13,<1.3.0a0" pcre2: ">=10.42,<10.43.0a0" - url: https://conda.anaconda.org/conda-forge/linux-64/libglib-2.78.1-h783c2da_1.conda + url: https://conda.anaconda.org/conda-forge/linux-64/libglib-2.78.3-h783c2da_0.conda hash: - md5: 70052d6c1e84643e30ffefb21ab6950f - sha256: 4e6fa28002f834cfc30a64792e95c1701d835cc3d3a4bb18d6e8d16bb8aba05b + md5: 9bd06b12bbfa6fd1740fd23af4b0f0c7 + sha256: b1b594294a0fe4c9a51596ef027efed9268d60827e8ae61fb7545c521a631e33 category: main optional: false - name: libglib - version: 2.78.1 + version: 2.78.3 manager: conda platform: osx-64 dependencies: @@ -11761,14 +11758,14 @@ package: libiconv: ">=1.17,<2.0a0" libzlib: ">=1.2.13,<1.3.0a0" pcre2: ">=10.42,<10.43.0a0" - url: https://conda.anaconda.org/conda-forge/osx-64/libglib-2.78.1-h198397b_1.conda + url: https://conda.anaconda.org/conda-forge/osx-64/libglib-2.78.3-h198397b_0.conda hash: - md5: fb318c3fed632cf2de190fb10496fbd1 - sha256: 73bcb4ea07af4291221271aa7aaa0795d59d851a6f43d6e0df22601f61725e4b + md5: e18624e441743b0d744116885b70f092 + sha256: 90e58879873b05617e0678ecfbf47bc740c1a2ed7840b8f7cd1241813b9037db category: main optional: false - name: libglib - version: 2.78.1 + version: 2.78.3 manager: conda platform: osx-arm64 dependencies: @@ -11779,10 +11776,10 @@ package: libiconv: ">=1.17,<2.0a0" libzlib: ">=1.2.13,<1.3.0a0" pcre2: ">=10.42,<10.43.0a0" - url: https://conda.anaconda.org/conda-forge/osx-arm64/libglib-2.78.1-hb438215_1.conda + url: https://conda.anaconda.org/conda-forge/osx-arm64/libglib-2.78.3-hb438215_0.conda hash: - md5: 3ce7984906f2ba4be662c219e8def77e - sha256: 3d94b6d8d27301f23b0d0c8b078c5e0bf6fc4f5f6260f3794cfaf7e247fe9e20 + md5: 8c98b7018b434236e2c0f14d7cf3c113 + sha256: f26afb1003e810e768138b0c849e9408c0ae8635062aeaf7abae381903a84e53 category: main optional: false - name: libgomp @@ -12160,7 +12157,7 @@ package: blosc: ">=1.21.4,<2.0a0" bzip2: ">=1.0.8,<2.0a0" hdf4: ">=4.2.15,<4.2.16.0a0" - hdf5: ">=1.14.2,<1.14.3.0a0" + hdf5: ">=1.14.2,<1.14.4.0a0" libaec: ">=1.0.6,<2.0a0" libcurl: ">=8.2.1,<9.0a0" libgcc-ng: ">=12" @@ -12185,7 +12182,7 @@ package: blosc: ">=1.21.4,<2.0a0" bzip2: ">=1.0.8,<2.0a0" hdf4: ">=4.2.15,<4.2.16.0a0" - hdf5: ">=1.14.2,<1.14.3.0a0" + hdf5: ">=1.14.2,<1.14.4.0a0" libaec: ">=1.0.6,<2.0a0" libcurl: ">=8.2.1,<9.0a0" libcxx: ">=15.0.7" @@ -12209,7 +12206,7 @@ package: blosc: ">=1.21.4,<2.0a0" bzip2: ">=1.0.8,<2.0a0" hdf4: ">=4.2.15,<4.2.16.0a0" - hdf5: ">=1.14.2,<1.14.3.0a0" + hdf5: ">=1.14.2,<1.14.4.0a0" libaec: ">=1.0.6,<2.0a0" libcurl: ">=8.2.1,<9.0a0" libcxx: ">=15.0.7" @@ -13498,15 +13495,15 @@ package: platform: linux-64 dependencies: libgcc-ng: ">=12" - libxml2: ">=2.11.5,<2.12.0a0" + libxml2: ">=2.11.6,<2.12.0a0" libxslt: ">=1.1.37,<2.0a0" libzlib: ">=1.2.13,<1.3.0a0" python: ">=3.11,<3.12.0a0" python_abi: 3.11.* - url: https://conda.anaconda.org/conda-forge/linux-64/lxml-4.9.3-py311h1a07684_1.conda + url: https://conda.anaconda.org/conda-forge/linux-64/lxml-4.9.3-py311h1a07684_2.conda hash: - md5: aab51e50d994e58efdfa5382139b0468 - sha256: 9ee461843278f695c5e301b4575e7dd02f69021e85023b62b17f7dfe2cd173e4 + md5: 76405b658bdd57a05bbdcee11d4714d2 + sha256: 4d7dd680fd7a6d5f34ca26d188ab73fd44378ade96d7ac50f25ccd4015517fde category: main optional: false - name: lxml @@ -13514,15 +13511,15 @@ package: manager: conda platform: osx-64 dependencies: - libxml2: ">=2.11.5,<2.12.0a0" + libxml2: ">=2.11.6,<2.12.0a0" libxslt: ">=1.1.37,<2.0a0" libzlib: ">=1.2.13,<1.3.0a0" python: ">=3.11,<3.12.0a0" python_abi: 3.11.* - url: https://conda.anaconda.org/conda-forge/osx-64/lxml-4.9.3-py311h19a211c_1.conda + url: https://conda.anaconda.org/conda-forge/osx-64/lxml-4.9.3-py311h719c1e2_2.conda hash: - md5: d3687d6ebe20ef8bf959dba786cdb28e - sha256: df952e80dc9ca98fbff11c2627288808135b51d18fc363a102f3e58eac8b4113 + md5: d3c23c905887e0c9999afb3a65650f85 + sha256: 37d48cbfac83c211c818a500b7c8852260146b2c2ed8006a4df8cc1b16a45ee2 category: main optional: false - name: lxml @@ -13530,15 +13527,15 @@ package: manager: conda platform: osx-arm64 dependencies: - libxml2: ">=2.11.5,<2.12.0a0" + libxml2: ">=2.11.6,<2.12.0a0" libxslt: ">=1.1.37,<2.0a0" libzlib: ">=1.2.13,<1.3.0a0" python: ">=3.11,<3.12.0a0" python_abi: 3.11.* - url: https://conda.anaconda.org/conda-forge/osx-arm64/lxml-4.9.3-py311hbafe683_1.conda + url: https://conda.anaconda.org/conda-forge/osx-arm64/lxml-4.9.3-py311hdef8331_2.conda hash: - md5: 5cfed11a5103844a5654446cf4d3f42a - sha256: e7ac6818b94242a5a1800a66d362a5e0262473e2399e731dd2061737b88c09b5 + md5: 99b75b0287466b51cafc6de0a8921825 + sha256: a33f16b34943fca0e5f113cf0b8c3be32271bda2718fe857d6e5c01e515c2158 category: main optional: false - name: lz4-c @@ -14260,42 +14257,6 @@ package: sha256: 7fcfda7b4a1d74205fcfdefd93804226a6eaffc74a319414c7d8d88f9249db3b category: main optional: false - - name: munch - version: 4.0.0 - manager: conda - platform: linux-64 - dependencies: - python: ">=3.8" - url: https://conda.anaconda.org/conda-forge/noarch/munch-4.0.0-pyhd8ed1ab_0.conda - hash: - md5: 376b32e8f9d3eacbd625f37d39bd507d - sha256: 093020ae2deb6c468120111a54909e1c576d70dfea6bc0eec5093e36d2fb8ff8 - category: main - optional: false - - name: munch - version: 4.0.0 - manager: conda - platform: osx-64 - dependencies: - python: ">=3.8" - url: https://conda.anaconda.org/conda-forge/noarch/munch-4.0.0-pyhd8ed1ab_0.conda - hash: - md5: 376b32e8f9d3eacbd625f37d39bd507d - sha256: 093020ae2deb6c468120111a54909e1c576d70dfea6bc0eec5093e36d2fb8ff8 - category: main - optional: false - - name: munch - version: 4.0.0 - manager: conda - platform: osx-arm64 - dependencies: - python: ">=3.8" - url: https://conda.anaconda.org/conda-forge/noarch/munch-4.0.0-pyhd8ed1ab_0.conda - hash: - md5: 376b32e8f9d3eacbd625f37d39bd507d - sha256: 093020ae2deb6c468120111a54909e1c576d70dfea6bc0eec5093e36d2fb8ff8 - category: main - optional: false - name: munkres version: 1.1.4 manager: conda @@ -14417,49 +14378,49 @@ package: category: main optional: false - name: nbconvert - version: 7.11.0 + version: 7.12.0 manager: conda platform: linux-64 dependencies: - nbconvert-core: 7.11.0 - nbconvert-pandoc: 7.11.0 + nbconvert-core: 7.12.0 + nbconvert-pandoc: 7.12.0 python: ">=3.8" - url: https://conda.anaconda.org/conda-forge/noarch/nbconvert-7.11.0-pyhd8ed1ab_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/nbconvert-7.12.0-pyhd8ed1ab_0.conda hash: - md5: e492b36cbea1c83d1663fa73a8abff9b - sha256: 6af7048b30c0ce6746297548df981037802f713853a1e856aedd2f8164946d39 + md5: 364e28ab12477494e72839aaa588073d + sha256: 0137330ab16bddf1fcaf60c0501c6145705b775fd547823708ed84364c934b76 category: main optional: false - name: nbconvert - version: 7.11.0 + version: 7.12.0 manager: conda platform: osx-64 dependencies: python: ">=3.8" - nbconvert-core: 7.11.0 - nbconvert-pandoc: 7.11.0 - url: https://conda.anaconda.org/conda-forge/noarch/nbconvert-7.11.0-pyhd8ed1ab_0.conda + nbconvert-core: 7.12.0 + nbconvert-pandoc: 7.12.0 + url: https://conda.anaconda.org/conda-forge/noarch/nbconvert-7.12.0-pyhd8ed1ab_0.conda hash: - md5: e492b36cbea1c83d1663fa73a8abff9b - sha256: 6af7048b30c0ce6746297548df981037802f713853a1e856aedd2f8164946d39 + md5: 364e28ab12477494e72839aaa588073d + sha256: 0137330ab16bddf1fcaf60c0501c6145705b775fd547823708ed84364c934b76 category: main optional: false - name: nbconvert - version: 7.11.0 + version: 7.12.0 manager: conda platform: osx-arm64 dependencies: python: ">=3.8" - nbconvert-core: 7.11.0 - nbconvert-pandoc: 7.11.0 - url: https://conda.anaconda.org/conda-forge/noarch/nbconvert-7.11.0-pyhd8ed1ab_0.conda + nbconvert-core: 7.12.0 + nbconvert-pandoc: 7.12.0 + url: https://conda.anaconda.org/conda-forge/noarch/nbconvert-7.12.0-pyhd8ed1ab_0.conda hash: - md5: e492b36cbea1c83d1663fa73a8abff9b - sha256: 6af7048b30c0ce6746297548df981037802f713853a1e856aedd2f8164946d39 + md5: 364e28ab12477494e72839aaa588073d + sha256: 0137330ab16bddf1fcaf60c0501c6145705b775fd547823708ed84364c934b76 category: main optional: false - name: nbconvert-core - version: 7.11.0 + version: 7.12.0 manager: conda platform: linux-64 dependencies: @@ -14480,14 +14441,14 @@ package: python: ">=3.8" tinycss2: "" traitlets: ">=5.0" - url: https://conda.anaconda.org/conda-forge/noarch/nbconvert-core-7.11.0-pyhd8ed1ab_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/nbconvert-core-7.12.0-pyhd8ed1ab_0.conda hash: - md5: d59e0cb1ca993f8f910cfdf393232acf - sha256: 81732e083c4c85a52248e20ff0e40a14b0b49db9cc7ce414e8aa7d6f8980dad0 + md5: 4d67c68fd0d130091ada039bc2d81b33 + sha256: 04c3ac88701d98d58139569e4899c3254bf99908179a898cc3dcadd8c0ef44b4 category: main optional: false - name: nbconvert-core - version: 7.11.0 + version: 7.12.0 manager: conda platform: osx-64 dependencies: @@ -14508,14 +14469,14 @@ package: pygments: ">=2.4.1" nbclient: ">=0.5.0" mistune: ">=2.0.3,<4" - url: https://conda.anaconda.org/conda-forge/noarch/nbconvert-core-7.11.0-pyhd8ed1ab_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/nbconvert-core-7.12.0-pyhd8ed1ab_0.conda hash: - md5: d59e0cb1ca993f8f910cfdf393232acf - sha256: 81732e083c4c85a52248e20ff0e40a14b0b49db9cc7ce414e8aa7d6f8980dad0 + md5: 4d67c68fd0d130091ada039bc2d81b33 + sha256: 04c3ac88701d98d58139569e4899c3254bf99908179a898cc3dcadd8c0ef44b4 category: main optional: false - name: nbconvert-core - version: 7.11.0 + version: 7.12.0 manager: conda platform: osx-arm64 dependencies: @@ -14536,52 +14497,52 @@ package: pygments: ">=2.4.1" nbclient: ">=0.5.0" mistune: ">=2.0.3,<4" - url: https://conda.anaconda.org/conda-forge/noarch/nbconvert-core-7.11.0-pyhd8ed1ab_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/nbconvert-core-7.12.0-pyhd8ed1ab_0.conda hash: - md5: d59e0cb1ca993f8f910cfdf393232acf - sha256: 81732e083c4c85a52248e20ff0e40a14b0b49db9cc7ce414e8aa7d6f8980dad0 + md5: 4d67c68fd0d130091ada039bc2d81b33 + sha256: 04c3ac88701d98d58139569e4899c3254bf99908179a898cc3dcadd8c0ef44b4 category: main optional: false - name: nbconvert-pandoc - version: 7.11.0 + version: 7.12.0 manager: conda platform: linux-64 dependencies: - nbconvert-core: 7.11.0 + nbconvert-core: 7.12.0 pandoc: "" python: ">=3.8" - url: https://conda.anaconda.org/conda-forge/noarch/nbconvert-pandoc-7.11.0-pyhd8ed1ab_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/nbconvert-pandoc-7.12.0-pyhd8ed1ab_0.conda hash: - md5: 51bd005efab7e5c5c2af2570327bd213 - sha256: 377d3c3f973b6885406ff6606d24c5e1fbd0d0fdc64c0dc17162f6daf35e08cf + md5: 460d7cac50322a39b61a833885a6a8d5 + sha256: ebf25caef387ec79f8d8f6771240d451ffaebcc2cdd127c0b152c6697d661d10 category: main optional: false - name: nbconvert-pandoc - version: 7.11.0 + version: 7.12.0 manager: conda platform: osx-64 dependencies: pandoc: "" python: ">=3.8" - nbconvert-core: 7.11.0 - url: https://conda.anaconda.org/conda-forge/noarch/nbconvert-pandoc-7.11.0-pyhd8ed1ab_0.conda + nbconvert-core: 7.12.0 + url: https://conda.anaconda.org/conda-forge/noarch/nbconvert-pandoc-7.12.0-pyhd8ed1ab_0.conda hash: - md5: 51bd005efab7e5c5c2af2570327bd213 - sha256: 377d3c3f973b6885406ff6606d24c5e1fbd0d0fdc64c0dc17162f6daf35e08cf + md5: 460d7cac50322a39b61a833885a6a8d5 + sha256: ebf25caef387ec79f8d8f6771240d451ffaebcc2cdd127c0b152c6697d661d10 category: main optional: false - name: nbconvert-pandoc - version: 7.11.0 + version: 7.12.0 manager: conda platform: osx-arm64 dependencies: pandoc: "" python: ">=3.8" - nbconvert-core: 7.11.0 - url: https://conda.anaconda.org/conda-forge/noarch/nbconvert-pandoc-7.11.0-pyhd8ed1ab_0.conda + nbconvert-core: 7.12.0 + url: https://conda.anaconda.org/conda-forge/noarch/nbconvert-pandoc-7.12.0-pyhd8ed1ab_0.conda hash: - md5: 51bd005efab7e5c5c2af2570327bd213 - sha256: 377d3c3f973b6885406ff6606d24c5e1fbd0d0fdc64c0dc17162f6daf35e08cf + md5: 460d7cac50322a39b61a833885a6a8d5 + sha256: ebf25caef387ec79f8d8f6771240d451ffaebcc2cdd127c0b152c6697d661d10 category: main optional: false - name: nbformat @@ -15563,8 +15524,8 @@ package: dependencies: numpy: "" pandas: "" - typing_extensions: "" packaging: "" + typing_extensions: "" pydantic: "" wrapt: "" multimethod: "" @@ -15584,8 +15545,8 @@ package: dependencies: numpy: "" pandas: "" - typing_extensions: "" packaging: "" + typing_extensions: "" pydantic: "" wrapt: "" multimethod: "" @@ -16337,42 +16298,39 @@ package: category: main optional: false - name: platformdirs - version: 4.0.0 + version: 4.1.0 manager: conda platform: linux-64 dependencies: - python: ">=3.7" - typing_extensions: ">=4.7.1" - url: https://conda.anaconda.org/conda-forge/noarch/platformdirs-4.0.0-pyhd8ed1ab_0.conda + python: ">=3.8" + url: https://conda.anaconda.org/conda-forge/noarch/platformdirs-4.1.0-pyhd8ed1ab_0.conda hash: - md5: 6bb4ee32cd435deaeac72776c001e7ac - sha256: 67381d3f7cadca7df7699238e0dcce680ad20d7fd28804bab48611fecb084937 + md5: 45a5065664da0d1dfa8f8cd2eaf05ab9 + sha256: 9e4ff17ce802159ed31344eb913eaa877688226765b77947b102b42255a53853 category: main optional: false - name: platformdirs - version: 4.0.0 + version: 4.1.0 manager: conda platform: osx-64 dependencies: - python: ">=3.7" - typing_extensions: ">=4.7.1" - url: https://conda.anaconda.org/conda-forge/noarch/platformdirs-4.0.0-pyhd8ed1ab_0.conda + python: ">=3.8" + url: https://conda.anaconda.org/conda-forge/noarch/platformdirs-4.1.0-pyhd8ed1ab_0.conda hash: - md5: 6bb4ee32cd435deaeac72776c001e7ac - sha256: 67381d3f7cadca7df7699238e0dcce680ad20d7fd28804bab48611fecb084937 + md5: 45a5065664da0d1dfa8f8cd2eaf05ab9 + sha256: 9e4ff17ce802159ed31344eb913eaa877688226765b77947b102b42255a53853 category: main optional: false - name: platformdirs - version: 4.0.0 + version: 4.1.0 manager: conda platform: osx-arm64 dependencies: - python: ">=3.7" - typing_extensions: ">=4.7.1" - url: https://conda.anaconda.org/conda-forge/noarch/platformdirs-4.0.0-pyhd8ed1ab_0.conda + python: ">=3.8" + url: https://conda.anaconda.org/conda-forge/noarch/platformdirs-4.1.0-pyhd8ed1ab_0.conda hash: - md5: 6bb4ee32cd435deaeac72776c001e7ac - sha256: 67381d3f7cadca7df7699238e0dcce680ad20d7fd28804bab48611fecb084937 + md5: 45a5065664da0d1dfa8f8cd2eaf05ab9 + sha256: 9e4ff17ce802159ed31344eb913eaa877688226765b77947b102b42255a53853 category: main optional: false - name: pluggy @@ -16935,18 +16893,18 @@ package: category: main optional: false - name: psycopg2 - version: 2.9.7 + version: 2.9.9 manager: conda platform: linux-64 dependencies: libgcc-ng: ">=12" - libpq: ">=16.0,<17.0a0" + libpq: ">=16.1,<17.0a0" python: ">=3.11,<3.12.0a0" python_abi: 3.11.* - url: https://conda.anaconda.org/conda-forge/linux-64/psycopg2-2.9.7-py311h03dec38_1.conda + url: https://conda.anaconda.org/conda-forge/linux-64/psycopg2-2.9.9-py311h03dec38_0.conda hash: - md5: 894f0e7a734b1f1182f87081f1ddffa3 - sha256: 047fe0687432b83762d4d77a31cd01572044a231f945f83e10f2cd3c2b44077d + md5: 3cc2decd316838bce14d73818e0bf7a4 + sha256: 4e78d9fe1799d028d9a2da3636a3a68a531aeca5d2c679d4fc78627a426b11cb category: main optional: false - name: psycopg2 @@ -16980,16 +16938,16 @@ package: category: main optional: false - name: psycopg2-binary - version: 2.9.7 + version: 2.9.9 manager: conda platform: linux-64 dependencies: - psycopg2: ">=2.9.7,<2.9.8.0a0" + psycopg2: ">=2.9.9,<2.9.10.0a0" python: ">=3.6" - url: https://conda.anaconda.org/conda-forge/noarch/psycopg2-binary-2.9.7-pyhd8ed1ab_1.conda + url: https://conda.anaconda.org/conda-forge/noarch/psycopg2-binary-2.9.9-pyhd8ed1ab_0.conda hash: - md5: 0212a5c5ae1ab578853364bfc5d9f657 - sha256: 5d82cb8b90daff6c12a4b6e0848fd32172522d82ceb5f093bfd55bfec09b3797 + md5: c15b2ec0570f8988819eea58286dbc19 + sha256: bb6184a3de8a6fddaed9104539ada9ac7c5e2bd900284ccf96ef5e4e285e75db category: main optional: false - name: psycopg2-binary @@ -18810,7 +18768,7 @@ package: category: main optional: false - name: pyzmq - version: 25.1.1 + version: 25.1.2 manager: conda platform: linux-64 dependencies: @@ -18820,40 +18778,44 @@ package: python: ">=3.11,<3.12.0a0" python_abi: 3.11.* zeromq: ">=4.3.5,<4.4.0a0" - url: https://conda.anaconda.org/conda-forge/linux-64/pyzmq-25.1.1-py311h34ded2d_2.conda + url: https://conda.anaconda.org/conda-forge/linux-64/pyzmq-25.1.2-py311h34ded2d_0.conda hash: - md5: ea365280db99687905b4d76cf6a3568c - sha256: a5ed6592f32b0caf3883a2f863e8a6258845310d4eebeab2eaf1c5abed04d6b8 + md5: 819aa640a0493d4b52faf938e94d129e + sha256: 54ccdde1370d8a373e516b84bd7fe4af394f8c6f3778eb050de82f04ffb86160 category: main optional: false - name: pyzmq - version: 25.1.1 + version: 25.1.2 manager: conda platform: osx-64 dependencies: + __osx: ">=10.9" + libcxx: ">=16.0.6" libsodium: ">=1.0.18,<1.0.19.0a0" python: ">=3.11,<3.12.0a0" python_abi: 3.11.* zeromq: ">=4.3.5,<4.4.0a0" - url: https://conda.anaconda.org/conda-forge/osx-64/pyzmq-25.1.1-py311he3804a1_2.conda + url: https://conda.anaconda.org/conda-forge/osx-64/pyzmq-25.1.2-py311h889d6d6_0.conda hash: - md5: 9b1ea037c51fcdb06bd2d95804270860 - sha256: 7a9af16e04752c50675ca99ab06888aaf8305efb5d292f62f7268ad911c967f4 + md5: 241fde77a74bd223562662af26f4828b + sha256: a8cb598edd68b3d2ca88cd2cdbc60c9180a392c393dd58aaf25e9897697d28d3 category: main optional: false - name: pyzmq - version: 25.1.1 + version: 25.1.2 manager: conda platform: osx-arm64 dependencies: + __osx: ">=10.9" + libcxx: ">=16.0.6" libsodium: ">=1.0.18,<1.0.19.0a0" python: ">=3.11,<3.12.0a0" python_abi: 3.11.* zeromq: ">=4.3.5,<4.4.0a0" - url: https://conda.anaconda.org/conda-forge/osx-arm64/pyzmq-25.1.1-py311he9c0408_2.conda + url: https://conda.anaconda.org/conda-forge/osx-arm64/pyzmq-25.1.2-py311h6727e71_0.conda hash: - md5: 51b7458a36011c4982261478fcc62026 - sha256: 03b78fe912c02547b284bc3404194bb4c1d9a2680e4b46f45c131a0d13d10b48 + md5: c0ab7687c09ec2c12d4110c2d5ba7050 + sha256: 684dc254a778600fb4ce31d6e3a82f18bf3a2779d71b06d237e76357dda8be9e category: main optional: false - name: qtconsole-base @@ -19142,45 +19104,45 @@ package: category: main optional: false - name: referencing - version: 0.31.1 + version: 0.32.0 manager: conda platform: linux-64 dependencies: attrs: ">=22.2.0" python: ">=3.8" rpds-py: ">=0.7.0" - url: https://conda.anaconda.org/conda-forge/noarch/referencing-0.31.1-pyhd8ed1ab_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/referencing-0.32.0-pyhd8ed1ab_0.conda hash: - md5: ae08039cf63eb82637b867aea3f04758 - sha256: efb91b7d2f6e729c01676e52e99071db819628a9f0a3a519c8969f0d2350a371 + md5: a7b5a535cd614e384594530aee7e6061 + sha256: dfd40282910a45e58882ed94b502b2a09f475efb04eaaa3bd8b3b5a9b21a19c3 category: main optional: false - name: referencing - version: 0.31.1 + version: 0.32.0 manager: conda platform: osx-64 dependencies: python: ">=3.8" attrs: ">=22.2.0" rpds-py: ">=0.7.0" - url: https://conda.anaconda.org/conda-forge/noarch/referencing-0.31.1-pyhd8ed1ab_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/referencing-0.32.0-pyhd8ed1ab_0.conda hash: - md5: ae08039cf63eb82637b867aea3f04758 - sha256: efb91b7d2f6e729c01676e52e99071db819628a9f0a3a519c8969f0d2350a371 + md5: a7b5a535cd614e384594530aee7e6061 + sha256: dfd40282910a45e58882ed94b502b2a09f475efb04eaaa3bd8b3b5a9b21a19c3 category: main optional: false - name: referencing - version: 0.31.1 + version: 0.32.0 manager: conda platform: osx-arm64 dependencies: python: ">=3.8" attrs: ">=22.2.0" rpds-py: ">=0.7.0" - url: https://conda.anaconda.org/conda-forge/noarch/referencing-0.31.1-pyhd8ed1ab_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/referencing-0.32.0-pyhd8ed1ab_0.conda hash: - md5: ae08039cf63eb82637b867aea3f04758 - sha256: efb91b7d2f6e729c01676e52e99071db819628a9f0a3a519c8969f0d2350a371 + md5: a7b5a535cd614e384594530aee7e6061 + sha256: dfd40282910a45e58882ed94b502b2a09f475efb04eaaa3bd8b3b5a9b21a19c3 category: main optional: false - name: regex @@ -19806,7 +19768,7 @@ package: category: main optional: false - name: ruff - version: 0.1.6 + version: 0.1.7 manager: conda platform: linux-64 dependencies: @@ -19814,14 +19776,14 @@ package: libstdcxx-ng: ">=12" python: ">=3.11,<3.12.0a0" python_abi: 3.11.* - url: https://conda.anaconda.org/conda-forge/linux-64/ruff-0.1.6-py311h7145743_0.conda + url: https://conda.anaconda.org/conda-forge/linux-64/ruff-0.1.7-py311h7145743_0.conda hash: - md5: aff8387edd5157da054c4b46cc38ffdc - sha256: dd8f7a3e2e7bc65fb6c2c32aae79ebc8623c6b87cbdbc8d2651be9ccd63e29d0 + md5: 7e4329efd6a902a64470bfbe21a83e21 + sha256: c82eb1b904671ab0eb3212f18aa7af14e51b66a8e893d18a3089d29a334e1e0d category: main optional: false - name: ruff - version: 0.1.6 + version: 0.1.7 manager: conda platform: osx-64 dependencies: @@ -19829,14 +19791,14 @@ package: libcxx: ">=16.0.6" python: ">=3.11,<3.12.0a0" python_abi: 3.11.* - url: https://conda.anaconda.org/conda-forge/osx-64/ruff-0.1.6-py311hec6fdf1_0.conda + url: https://conda.anaconda.org/conda-forge/osx-64/ruff-0.1.7-py311hec6fdf1_0.conda hash: - md5: f0fa30260ad0ac05ef120b758c68d7ba - sha256: 2587bd6a04c6a1178b63438de97c091bcfca15f9bb5ea0d6a1f109a66187e33e + md5: d21a7d577aa62ce3a2b1eb124da50446 + sha256: 8aac6c5324504cd6b453be725b0ba8cca478468fb5c328cec79301b552477dc7 category: main optional: false - name: ruff - version: 0.1.6 + version: 0.1.7 manager: conda platform: osx-arm64 dependencies: @@ -19844,10 +19806,10 @@ package: libcxx: ">=16.0.6" python: ">=3.11,<3.12.0a0" python_abi: 3.11.* - url: https://conda.anaconda.org/conda-forge/osx-arm64/ruff-0.1.6-py311h6fc163c_0.conda + url: https://conda.anaconda.org/conda-forge/osx-arm64/ruff-0.1.7-py311h6fc163c_0.conda hash: - md5: c9ff47502a21c1d072e8da209d9efba3 - sha256: 8bde8b2d66f34a242ea6759df3c21f3321d17a8cdb4d421ae489f97f42452920 + md5: 86298e47df0a6b448df8632df3d1f4a9 + sha256: 37d5b2a72c3a5114d43d49a250dffd81d1b064d673e5faf3932c6bf56f2e7c76 category: main optional: false - name: s2n @@ -19916,10 +19878,10 @@ package: python_abi: 3.11.* scipy: "" threadpoolctl: ">=2.0.0" - url: https://conda.anaconda.org/conda-forge/linux-64/scikit-learn-1.3.2-py311hc009520_1.conda + url: https://conda.anaconda.org/conda-forge/linux-64/scikit-learn-1.3.2-py311hc009520_2.conda hash: - md5: 6b92d3d0680eae9d1d9860a721f7fb51 - sha256: 638253cba17e44081674b2dd7bee2025c202e91b653182da511ca57de942689d + md5: 9821f8e497a791858226f535e5e0be62 + sha256: 1133cd9209207528d4fdd88ffb300a04794942e5d474c607ed1f0578fe218fd2 category: main optional: false - name: scikit-learn @@ -19930,16 +19892,16 @@ package: __osx: ">=10.9" joblib: ">=1.1.1" libcxx: ">=16.0.6" - llvm-openmp: ">=16.0.6" + llvm-openmp: ">=17.0.6" numpy: ">=1.23.5,<2.0a0" python: ">=3.11,<3.12.0a0" python_abi: 3.11.* scipy: "" threadpoolctl: ">=2.0.0" - url: https://conda.anaconda.org/conda-forge/osx-64/scikit-learn-1.3.2-py311h66081b9_1.conda + url: https://conda.anaconda.org/conda-forge/osx-64/scikit-learn-1.3.2-py311h66081b9_2.conda hash: - md5: a7053f3e86b6b9139593a027c54e3097 - sha256: 0b4aee092d4689d07aa95ed1d1071eeb74a5e011d34bc9ebd9b27e9b2166db7e + md5: a774c2628441ed6f4db9d7026ab1c7f4 + sha256: 8a2b2660524d06c32c7a2a5af9f3f918b17f34a01990f6190e1e7f37233cdb47 category: main optional: false - name: scikit-learn @@ -19950,16 +19912,16 @@ package: __osx: ">=10.9" joblib: ">=1.1.1" libcxx: ">=16.0.6" - llvm-openmp: ">=16.0.6" + llvm-openmp: ">=17.0.6" numpy: ">=1.23.5,<2.0a0" python: ">=3.11,<3.12.0a0" python_abi: 3.11.* scipy: "" threadpoolctl: ">=2.0.0" - url: https://conda.anaconda.org/conda-forge/osx-arm64/scikit-learn-1.3.2-py311ha25ca4d_1.conda + url: https://conda.anaconda.org/conda-forge/osx-arm64/scikit-learn-1.3.2-py311ha25ca4d_2.conda hash: - md5: dea73952c589de24ec577c41fac181c5 - sha256: 589bca23648b8969b16a36f87d7114a12fbe88d665cde32427c7126e5b34e63c + md5: f427eff14595109354bd19c94a3d3624 + sha256: d24397673a1f0b609796a716c80b45e3971eb09b987b2a30519a30f5dabeae14 category: main optional: false - name: scipy @@ -21167,45 +21129,45 @@ package: category: main optional: false - name: starlette - version: 0.32.0.post1 + version: 0.33.0 manager: conda platform: linux-64 dependencies: anyio: <5,>=3.4.0 python: ">=3.8" typing_extensions: ">=3.10.0" - url: https://conda.anaconda.org/conda-forge/noarch/starlette-0.32.0.post1-pyhd8ed1ab_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/starlette-0.33.0-pyhd8ed1ab_0.conda hash: - md5: 9aa6d56db739eee2ff473becbe178fd1 - sha256: 9692b83467670b473dc71137376f735249ef2ee6eeefce9068b0dec94810c24c + md5: 55027cf7f50803f0f5ece8b661eff47b + sha256: 3923f4c3e31d8c3a9c574779585137ff834a6108558a8956ef93022d4fcb37a8 category: dev optional: true - name: starlette - version: 0.32.0.post1 + version: 0.33.0 manager: conda platform: osx-64 dependencies: python: ">=3.8" typing_extensions: ">=3.10.0" anyio: <5,>=3.4.0 - url: https://conda.anaconda.org/conda-forge/noarch/starlette-0.32.0.post1-pyhd8ed1ab_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/starlette-0.33.0-pyhd8ed1ab_0.conda hash: - md5: 9aa6d56db739eee2ff473becbe178fd1 - sha256: 9692b83467670b473dc71137376f735249ef2ee6eeefce9068b0dec94810c24c + md5: 55027cf7f50803f0f5ece8b661eff47b + sha256: 3923f4c3e31d8c3a9c574779585137ff834a6108558a8956ef93022d4fcb37a8 category: dev optional: true - name: starlette - version: 0.32.0.post1 + version: 0.33.0 manager: conda platform: osx-arm64 dependencies: python: ">=3.8" typing_extensions: ">=3.10.0" anyio: <5,>=3.4.0 - url: https://conda.anaconda.org/conda-forge/noarch/starlette-0.32.0.post1-pyhd8ed1ab_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/starlette-0.33.0-pyhd8ed1ab_0.conda hash: - md5: 9aa6d56db739eee2ff473becbe178fd1 - sha256: 9692b83467670b473dc71137376f735249ef2ee6eeefce9068b0dec94810c24c + md5: 55027cf7f50803f0f5ece8b661eff47b + sha256: 3923f4c3e31d8c3a9c574779585137ff834a6108558a8956ef93022d4fcb37a8 category: dev optional: true - name: stevedore @@ -22876,7 +22838,7 @@ package: category: main optional: false - name: virtualenv - version: 20.24.7 + version: 20.25.0 manager: conda platform: linux-64 dependencies: @@ -22884,14 +22846,14 @@ package: filelock: <4,>=3.12.2 platformdirs: <5,>=3.9.1 python: ">=3.8" - url: https://conda.anaconda.org/conda-forge/noarch/virtualenv-20.24.7-pyhd8ed1ab_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/virtualenv-20.25.0-pyhd8ed1ab_0.conda hash: - md5: db990278c2c00b268eed778de44f6057 - sha256: ad661ae59c64bd73c25dfadb00c601659f4d9cafbf428e36a690075e52bac96a + md5: c119653cba436d8183c27bf6d190e587 + sha256: 50827c3721a9dbf973b568709d4381add2a6552fa562f26a385c5edc16a534af category: main optional: false - name: virtualenv - version: 20.24.7 + version: 20.25.0 manager: conda platform: osx-64 dependencies: @@ -22899,14 +22861,14 @@ package: distlib: <1,>=0.3.7 filelock: <4,>=3.12.2 platformdirs: <5,>=3.9.1 - url: https://conda.anaconda.org/conda-forge/noarch/virtualenv-20.24.7-pyhd8ed1ab_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/virtualenv-20.25.0-pyhd8ed1ab_0.conda hash: - md5: db990278c2c00b268eed778de44f6057 - sha256: ad661ae59c64bd73c25dfadb00c601659f4d9cafbf428e36a690075e52bac96a + md5: c119653cba436d8183c27bf6d190e587 + sha256: 50827c3721a9dbf973b568709d4381add2a6552fa562f26a385c5edc16a534af category: main optional: false - name: virtualenv - version: 20.24.7 + version: 20.25.0 manager: conda platform: osx-arm64 dependencies: @@ -22914,10 +22876,10 @@ package: distlib: <1,>=0.3.7 filelock: <4,>=3.12.2 platformdirs: <5,>=3.9.1 - url: https://conda.anaconda.org/conda-forge/noarch/virtualenv-20.24.7-pyhd8ed1ab_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/virtualenv-20.25.0-pyhd8ed1ab_0.conda hash: - md5: db990278c2c00b268eed778de44f6057 - sha256: ad661ae59c64bd73c25dfadb00c601659f4d9cafbf428e36a690075e52bac96a + md5: c119653cba436d8183c27bf6d190e587 + sha256: 50827c3721a9dbf973b568709d4381add2a6552fa562f26a385c5edc16a534af category: main optional: false - name: watchdog @@ -23114,39 +23076,39 @@ package: category: main optional: false - name: websocket-client - version: 1.6.4 + version: 1.7.0 manager: conda platform: linux-64 dependencies: python: ">=3.8" - url: https://conda.anaconda.org/conda-forge/noarch/websocket-client-1.6.4-pyhd8ed1ab_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/websocket-client-1.7.0-pyhd8ed1ab_0.conda hash: - md5: bdb77b28cf16deac0eef431a068320e8 - sha256: df45b89862edcd7cd5180ec7b8c0c0ca9fb4d3f7d49ddafccdc76afcf50d8da6 + md5: 50ad31e07d706aae88b14a4ac9c73f23 + sha256: d9b537d5b7c5aa7a02a4ce4c6b755e458bd8083b67752a73c92d113ccec6c10f category: main optional: false - name: websocket-client - version: 1.6.4 + version: 1.7.0 manager: conda platform: osx-64 dependencies: python: ">=3.8" - url: https://conda.anaconda.org/conda-forge/noarch/websocket-client-1.6.4-pyhd8ed1ab_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/websocket-client-1.7.0-pyhd8ed1ab_0.conda hash: - md5: bdb77b28cf16deac0eef431a068320e8 - sha256: df45b89862edcd7cd5180ec7b8c0c0ca9fb4d3f7d49ddafccdc76afcf50d8da6 + md5: 50ad31e07d706aae88b14a4ac9c73f23 + sha256: d9b537d5b7c5aa7a02a4ce4c6b755e458bd8083b67752a73c92d113ccec6c10f category: main optional: false - name: websocket-client - version: 1.6.4 + version: 1.7.0 manager: conda platform: osx-arm64 dependencies: python: ">=3.8" - url: https://conda.anaconda.org/conda-forge/noarch/websocket-client-1.6.4-pyhd8ed1ab_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/websocket-client-1.7.0-pyhd8ed1ab_0.conda hash: - md5: bdb77b28cf16deac0eef431a068320e8 - sha256: df45b89862edcd7cd5180ec7b8c0c0ca9fb4d3f7d49ddafccdc76afcf50d8da6 + md5: 50ad31e07d706aae88b14a4ac9c73f23 + sha256: d9b537d5b7c5aa7a02a4ce4c6b755e458bd8083b67752a73c92d113ccec6c10f category: main optional: false - name: websockets diff --git a/environments/conda-osx-64.lock.yml b/environments/conda-osx-64.lock.yml index 12d78b1972..b13e20814c 100644 --- a/environments/conda-osx-64.lock.yml +++ b/environments/conda-osx-64.lock.yml @@ -1,6 +1,6 @@ # Generated by conda-lock. # platform: osx-64 -# input_hash: 39e56673d0def5503b315f36cec2b3e4bfd804758ec3bcea79dad417c6b146a2 +# input_hash: 0c46f3c7a9c86a583e588c727d6a7e7d3214ce9d048dd4e49df3a792c36b3e9c channels: - conda-forge @@ -90,7 +90,7 @@ dependencies: - libedit=3.1.20191231=h0678c8f_2 - libevent=2.1.12=ha90c15b_1 - libgfortran=5.0.0=13_2_0_h97931a8_1 - - libglib=2.78.1=h198397b_1 + - libglib=2.78.3=h198397b_0 - libkml=1.3.0=hab3ca0e_1018 - libllvm15=15.0.7=he4b1e75_3 - libnghttp2=1.58.0=h64cf6d3_0 @@ -114,7 +114,7 @@ dependencies: - gdk-pixbuf=2.42.10=hbb5a27d_4 - gts=0.7.6=h53e17e3_4 - krb5=1.21.2=hb884880_0 - - lcms2=2.15=hd6ba6f3_3 + - lcms2=2.16=ha2f27b4_0 - libopenblas=0.3.25=openmp_hfef2a42_0 - libthrift=0.19.0=h064b379_1 - libwebp=1.3.2=h44782d1_1 @@ -152,14 +152,14 @@ dependencies: - colorama=0.4.6=pyhd8ed1ab_0 - crashtest=0.4.1=pyhd8ed1ab_0 - cycler=0.12.1=pyhd8ed1ab_0 - - dagster-pipes=1.5.9=pyhd8ed1ab_0 + - dagster-pipes=1.5.10=pyhd8ed1ab_0 - dataclasses=0.8=pyhc8e2a94_3 - debugpy=1.8.0=py311hdf8f085_1 - decorator=5.1.1=pyhd8ed1ab_0 - defusedxml=0.7.1=pyhd8ed1ab_0 - distlib=0.3.7=pyhd8ed1ab_0 - docstring_parser=0.15=pyhd8ed1ab_0 - - docutils=0.20.1=py311h6eed73b_2 + - docutils=0.20.1=py311h6eed73b_3 - entrypoints=0.4=pyhd8ed1ab_0 - et_xmlfile=1.1.0=pyhd8ed1ab_0 - exceptiongroup=1.2.0=pyhd8ed1ab_0 @@ -167,8 +167,8 @@ dependencies: - executing=2.0.1=pyhd8ed1ab_0 - filelock=3.13.1=pyhd8ed1ab_0 - frozenlist=1.4.0=py311h2725bcf_1 - - fsspec=2023.10.0=pyhca7485f_0 - - google-cloud-sdk=455.0.0=py311h6eed73b_0 + - fsspec=2023.12.1=pyhca7485f_0 + - google-cloud-sdk=456.0.0=py311h6eed73b_0 - greenlet=3.0.1=py311hd39e593_0 - hpack=4.0.0=pyh9f0ad1d_0 - httptools=0.6.1=py311he705e18_0 @@ -193,7 +193,7 @@ dependencies: - libpq=16.1=h6dd4ff7_0 - llvmlite=0.41.1=py311hb5c2e0a_0 - locket=1.0.0=pyhd8ed1ab_0 - - lxml=4.9.3=py311h19a211c_1 + - lxml=4.9.3=py311h719c1e2_2 - marko=2.0.2=pyhd8ed1ab_0 - markupsafe=2.1.3=py311h2725bcf_1 - mdurl=0.1.0=pyhd8ed1ab_0 @@ -203,7 +203,6 @@ dependencies: - msgpack-python=1.0.7=py311h7bea37d_0 - multidict=6.0.4=py311h5547dcb_1 - multimethod=1.9.1=pyhd8ed1ab_0 - - munch=4.0.0=pyhd8ed1ab_0 - munkres=1.1.4=pyh9f0ad1d_0 - mypy_extensions=1.0.0=pyha770c72_0 - nest-asyncio=1.5.8=pyhd8ed1ab_0 @@ -217,6 +216,7 @@ dependencies: - pillow=10.1.0=py311hea5c87a_0 - pkginfo=1.9.6=pyhd8ed1ab_0 - pkgutil-resolve-name=1.3.10=pyhd8ed1ab_1 + - platformdirs=4.1.0=pyhd8ed1ab_0 - pluggy=1.3.0=pyhd8ed1ab_0 - prometheus_client=0.19.0=pyhd8ed1ab_0 - psutil=5.9.5=py311h2725bcf_1 @@ -239,14 +239,14 @@ dependencies: - pywin32-on-windows=0.1.0=pyh1179c8e_3 - pyxlsb=1.0.10=pyhd8ed1ab_0 - pyyaml=6.0.1=py311h2725bcf_1 - - pyzmq=25.1.1=py311he3804a1_2 + - pyzmq=25.1.2=py311h889d6d6_0 - regex=2023.10.3=py311h2725bcf_0 - rfc3986=2.0.0=pyhd8ed1ab_0 - rfc3986-validator=0.1.1=pyh9f0ad1d_0 - rpds-py=0.13.2=py311h5e0f0e4_0 - rtree=1.1.0=py311hbc1f44b_0 - ruamel.yaml.clib=0.2.7=py311h2725bcf_2 - - ruff=0.1.6=py311hec6fdf1_0 + - ruff=0.1.7=py311hec6fdf1_0 - setuptools=68.2.2=pyhd8ed1ab_0 - shellingham=1.5.4=pyhd8ed1ab_0 - simpleeval=0.9.13=pyhd8ed1ab_1 @@ -279,7 +279,7 @@ dependencies: - wcwidth=0.2.12=pyhd8ed1ab_0 - webcolors=1.13=pyhd8ed1ab_0 - webencodings=0.5.1=pyhd8ed1ab_2 - - websocket-client=1.6.4=pyhd8ed1ab_0 + - websocket-client=1.7.0=pyhd8ed1ab_0 - websockets=10.4=py311h5547dcb_1 - wheel=0.42.0=pyhd8ed1ab_0 - widgetsnbextension=4.0.9=pyhd8ed1ab_0 @@ -310,7 +310,7 @@ dependencies: - comm=0.1.4=pyhd8ed1ab_0 - coverage=7.3.2=py311h2725bcf_0 - curl=8.4.0=h726d00d_0 - - fonttools=4.45.1=py311he705e18_0 + - fonttools=4.46.0=py311he705e18_0 - gitdb=4.0.11=pyhd8ed1ab_0 - graphql-core=3.2.3=pyhd8ed1ab_0 - grpcio=1.59.2=py311hfd95bfa_0 @@ -320,7 +320,7 @@ dependencies: - hdf5=1.14.2=nompi_hedada53_100 - html5lib=1.1=pyh9f0ad1d_0 - hypothesis=6.91.0=pyha770c72_0 - - importlib-metadata=6.8.0=pyha770c72_0 + - importlib-metadata=7.0.0=pyha770c72_0 - importlib_resources=6.1.1=pyhd8ed1ab_0 - isodate=0.6.1=pyhd8ed1ab_0 - janus=1.0.0=pyhd8ed1ab_0 @@ -329,6 +329,7 @@ dependencies: - jinja2=3.1.2=pyhd8ed1ab_1 - joblib=1.3.2=pyhd8ed1ab_0 - jsonlines=4.0.0=pyhd8ed1ab_0 + - jupyter_core=5.5.0=py311h6eed73b_0 - jupyterlab_pygments=0.3.0=pyhd8ed1ab_0 - latexcodec=2.0.1=pyh9f0ad1d_0 - libcblas=3.9.0=20_osx64_openblas @@ -344,7 +345,6 @@ dependencies: - pexpect=4.8.0=pyh1a96a4e_2 - pint=0.22=pyhd8ed1ab_1 - pip=23.3.1=pyhd8ed1ab_0 - - platformdirs=4.0.0=pyhd8ed1ab_0 - poppler=23.11.0=hdd5a5e8_0 - postgresql=16.1=h413614c_0 - proj=9.3.0=h23b96cc_2 @@ -359,7 +359,7 @@ dependencies: - python-slugify=8.0.1=pyhd8ed1ab_2 - pyu2f=0.1.5=pyhd8ed1ab_0 - qtpy=2.4.1=pyhd8ed1ab_0 - - referencing=0.31.1=pyhd8ed1ab_0 + - referencing=0.32.0=pyhd8ed1ab_0 - restructuredtext_lint=1.4.0=pyhd8ed1ab_0 - rfc3339-validator=0.1.4=pyhd8ed1ab_0 - rsa=4.9=pyhd8ed1ab_0 @@ -371,6 +371,7 @@ dependencies: - typing_inspect=0.9.0=pyhd8ed1ab_0 - universal_pathlib=0.1.4=pyhd8ed1ab_0 - urllib3=1.26.18=pyhd8ed1ab_0 + - virtualenv=20.25.0=pyhd8ed1ab_0 - watchdog=3.0.0=py311h5ef12f2_1 - xerces-c=3.2.4=h6314983_3 - yarl=1.9.3=py311he705e18_0 @@ -381,22 +382,21 @@ dependencies: - arrow=1.3.0=pyhd8ed1ab_0 - async-timeout=4.0.3=pyhd8ed1ab_0 - aws-crt-cpp=0.24.7=hf3941dc_6 - - botocore=1.33.5=pyhd8ed1ab_0 + - botocore=1.33.9=pyhd8ed1ab_0 - branca=0.7.0=pyhd8ed1ab_1 - croniter=2.0.1=pyhd8ed1ab_0 - - cryptography=41.0.5=py311hd51016d_0 + - cryptography=41.0.7=py311h48c7838_1 - fqdn=1.5.1=pyhd8ed1ab_0 - geotiff=1.7.1=h889ec99_14 - gitpython=3.1.40=pyhd8ed1ab_0 - google-crc32c=1.1.2=py311h0b57020_5 - - googleapis-common-protos=1.61.0=pyhd8ed1ab_0 + - googleapis-common-protos=1.62.0=pyhd8ed1ab_0 - gql=3.4.1=pyhd8ed1ab_0 - graphql-relay=3.2.0=pyhd8ed1ab_0 - grpcio-health-checking=1.59.2=pyhd8ed1ab_0 - httpcore=1.0.2=pyhd8ed1ab_0 - - importlib_metadata=6.8.0=hd8ed1ab_0 + - importlib_metadata=7.0.0=hd8ed1ab_0 - jsonschema-specifications=2023.11.2=pyhd8ed1ab_0 - - jupyter_core=5.5.0=py311h6eed73b_0 - jupyter_server_terminals=0.4.4=pyhd8ed1ab_1 - kealib=1.5.2=h052fcf7_1 - libnetcdf=4.9.2=nompi_h6a32802_112 @@ -421,32 +421,31 @@ dependencies: - rich=13.7.0=pyhd8ed1ab_0 - sqlalchemy=2.0.23=py311he705e18_0 - stack_data=0.6.2=pyhd8ed1ab_0 - - starlette=0.32.0.post1=pyhd8ed1ab_0 + - starlette=0.33.0=pyhd8ed1ab_0 - tiledb=2.16.3=hd3a41d5_3 - ukkonen=1.0.1=py311h5fe6e05_4 - uvicorn=0.24.0.post1=py311h6eed73b_0 - - virtualenv=20.24.7=pyhd8ed1ab_0 - watchfiles=0.21.0=py311h5e0f0e4_0 - aiohttp=3.8.6=py311he705e18_1 - - alembic=1.12.1=pyhd8ed1ab_0 - - arelle-release=2.17.7=pyhd8ed1ab_0 + - alembic=1.13.0=pyhd8ed1ab_0 + - arelle-release=2.18.0=pyhd8ed1ab_0 - argon2-cffi=23.1.0=pyhd8ed1ab_0 - aws-sdk-cpp=1.11.182=h28d282b_7 - bottleneck=1.3.7=py311h4a70a88_1 - cachecontrol=0.13.1=pyhd8ed1ab_0 - contourpy=1.2.0=py311h7bea37d_0 - - dask-core=2023.11.0=pyhd8ed1ab_0 + - dask-core=2023.12.0=pyhd8ed1ab_0 - dnspython=2.4.2=pyhd8ed1ab_1 - ensureconda=1.4.3=pyhd8ed1ab_0 - - folium=0.15.0=pyhd8ed1ab_0 + - folium=0.15.1=pyhd8ed1ab_0 - google-resumable-media=2.6.0=pyhd8ed1ab_0 - graphene=3.3=pyhd8ed1ab_0 - grpcio-status=1.59.2=pyhd8ed1ab_0 - gtk2=2.24.33=h7c1209e_2 - h3-py=3.7.6=py311hdf8f085_1 - httpx=0.25.2=pyhd8ed1ab_0 - - identify=2.5.32=pyhd8ed1ab_0 - - ipython=8.18.1=pyh31011fe_1 + - identify=2.5.33=pyhd8ed1ab_0 + - ipython=8.18.1=pyh707e725_3 - isoduration=20.11.0=pyhd8ed1ab_0 - jsonschema=4.20.0=pyhd8ed1ab_0 - jupyter_client=8.6.0=pyhd8ed1ab_0 @@ -471,16 +470,16 @@ dependencies: - typeguard=4.1.5=pyhd8ed1ab_1 - typer=0.9.0=pyhd8ed1ab_0 - uvicorn-standard=0.24.0.post1=h6eed73b_0 - - boto3=1.33.5=pyhd8ed1ab_0 + - boto3=1.33.9=pyhd8ed1ab_0 - cachecontrol-with-filecache=0.13.1=pyhd8ed1ab_0 - - dagster=1.5.9=pyhd8ed1ab_0 + - dagster=1.5.10=pyhd8ed1ab_0 - datasette=0.64.4=pyhd8ed1ab_1 - doc8=1.1.1=pyhd8ed1ab_0 - email-validator=2.1.0.post1=pyhd8ed1ab_0 - frictionless=4.40.8=pyh6c4a22f_0 - gdal=3.8.0=py311h5646c56_6 - geopandas-base=0.14.1=pyha770c72_0 - - google-auth=2.24.0=pyhca7485f_0 + - google-auth=2.25.1=pyhca7485f_0 - gql-with-requests=3.4.1=pyhd8ed1ab_0 - graphviz=9.0.0=hee74176_1 - ipykernel=6.26.0=pyh3cd1d5f_0 @@ -493,14 +492,14 @@ dependencies: - pre-commit=3.5.0=pyha770c72_0 - pydantic-settings=2.1.0=pyhd8ed1ab_1 - requests-oauthlib=1.3.1=pyhd8ed1ab_0 - - scikit-learn=1.3.2=py311h66081b9_1 + - scikit-learn=1.3.2=py311h66081b9_2 - timezonefinder=6.2.0=py311he705e18_2 - catalystcoop.ferc_xbrl_extractor=1.3.1=pyhd8ed1ab_0 - conda-lock=2.5.1=pyhd8ed1ab_0 - - dagster-graphql=1.5.9=pyhd8ed1ab_0 - - dagster-postgres=0.21.9=pyhd8ed1ab_1 - - fiona=1.9.5=py311h809632c_1 - - google-api-core=2.14.0=pyhd8ed1ab_0 + - dagster-graphql=1.5.10=pyhd8ed1ab_0 + - dagster-postgres=0.21.10=pyhd8ed1ab_0 + - fiona=1.9.5=py311h809632c_2 + - google-api-core=2.15.0=pyhd8ed1ab_0 - google-auth-oauthlib=1.1.0=pyhd8ed1ab_0 - jupyter_console=6.6.3=pyhd8ed1ab_0 - jupyter_events=0.9.0=pyhd8ed1ab_0 @@ -514,23 +513,23 @@ dependencies: - qtconsole-base=5.5.1=pyha770c72_0 - recordlinkage=0.16=pyhd8ed1ab_0 - tabulator=1.53.5=pyhd8ed1ab_0 - - dagster-webserver=1.5.9=pyhd8ed1ab_0 + - dagster-webserver=1.5.10=pyhd8ed1ab_0 - geopandas=0.14.1=pyhd8ed1ab_0 - - google-cloud-core=2.3.3=pyhd8ed1ab_0 + - google-cloud-core=2.4.1=pyhd8ed1ab_0 - libarrow-dataset=14.0.1=hc222712_3_cpu - libarrow-flight-sql=14.0.1=h2cc6c1c_3_cpu - - nbconvert-core=7.11.0=pyhd8ed1ab_0 + - nbconvert-core=7.12.0=pyhd8ed1ab_0 - tableschema=1.19.3=pyh9f0ad1d_0 - datapackage=1.15.2=pyh44b312d_0 - google-cloud-storage=2.13.0=pyhca7485f_0 - - jupyter_server=2.11.1=pyhd8ed1ab_0 + - jupyter_server=2.12.1=pyhd8ed1ab_0 - libarrow-substrait=14.0.1=h2cc6c1c_3_cpu - - nbconvert-pandoc=7.11.0=pyhd8ed1ab_0 - - gcsfs=2023.10.0=pyhd8ed1ab_0 + - nbconvert-pandoc=7.12.0=pyhd8ed1ab_0 + - gcsfs=2023.12.1=pyhd8ed1ab_0 - jupyter-lsp=2.2.1=pyhd8ed1ab_0 - jupyter-resource-usage=1.0.1=pyhd8ed1ab_0 - jupyterlab_server=2.25.2=pyhd8ed1ab_0 - - nbconvert=7.11.0=pyhd8ed1ab_0 + - nbconvert=7.12.0=pyhd8ed1ab_0 - notebook-shim=0.2.3=pyhd8ed1ab_0 - pyarrow=14.0.1=py311h98a0319_3_cpu - jupyterlab=4.0.9=pyhd8ed1ab_0 diff --git a/environments/conda-osx-arm64.lock.yml b/environments/conda-osx-arm64.lock.yml index 4cd3f6d3c9..9f64382dfb 100644 --- a/environments/conda-osx-arm64.lock.yml +++ b/environments/conda-osx-arm64.lock.yml @@ -1,6 +1,6 @@ # Generated by conda-lock. # platform: osx-arm64 -# input_hash: c275fe8ff3012ad83a98252ba570f9b278f142720b73f42abfd4c4a1107f67e2 +# input_hash: 3e57b04f7b2300284a5c81502687a5c373c4be31786246211ea23f019d4a513f channels: - conda-forge @@ -90,7 +90,7 @@ dependencies: - libedit=3.1.20191231=hc8eb9b7_2 - libevent=2.1.12=h2757513_1 - libgfortran=5.0.0=13_2_0_hd922786_1 - - libglib=2.78.1=hb438215_1 + - libglib=2.78.3=hb438215_0 - libkml=1.3.0=h1eb4d9f_1018 - libllvm15=15.0.7=h504e6bf_3 - libnghttp2=1.58.0=ha4dd798_0 @@ -114,7 +114,7 @@ dependencies: - gdk-pixbuf=2.42.10=h15fa40c_4 - gts=0.7.6=he42f4ea_4 - krb5=1.21.2=h92f50d5_0 - - lcms2=2.15=hf2736f0_3 + - lcms2=2.16=ha0e7c42_0 - libopenblas=0.3.25=openmp_h6c19121_0 - libthrift=0.19.0=h026a170_1 - libwebp=1.3.2=hf30222e_1 @@ -152,14 +152,14 @@ dependencies: - colorama=0.4.6=pyhd8ed1ab_0 - crashtest=0.4.1=pyhd8ed1ab_0 - cycler=0.12.1=pyhd8ed1ab_0 - - dagster-pipes=1.5.9=pyhd8ed1ab_0 + - dagster-pipes=1.5.10=pyhd8ed1ab_0 - dataclasses=0.8=pyhc8e2a94_3 - debugpy=1.8.0=py311ha891d26_1 - decorator=5.1.1=pyhd8ed1ab_0 - defusedxml=0.7.1=pyhd8ed1ab_0 - distlib=0.3.7=pyhd8ed1ab_0 - docstring_parser=0.15=pyhd8ed1ab_0 - - docutils=0.20.1=py311h267d04e_2 + - docutils=0.20.1=py311h267d04e_3 - entrypoints=0.4=pyhd8ed1ab_0 - et_xmlfile=1.1.0=pyhd8ed1ab_0 - exceptiongroup=1.2.0=pyhd8ed1ab_0 @@ -167,8 +167,8 @@ dependencies: - executing=2.0.1=pyhd8ed1ab_0 - filelock=3.13.1=pyhd8ed1ab_0 - frozenlist=1.4.0=py311heffc1b2_1 - - fsspec=2023.10.0=pyhca7485f_0 - - google-cloud-sdk=455.0.0=py311h267d04e_0 + - fsspec=2023.12.1=pyhca7485f_0 + - google-cloud-sdk=456.0.0=py311h267d04e_0 - greenlet=3.0.1=py311hbaf5611_0 - hpack=4.0.0=pyh9f0ad1d_0 - httptools=0.6.1=py311h05b510d_0 @@ -193,7 +193,7 @@ dependencies: - libpq=16.1=hd435d45_0 - llvmlite=0.41.1=py311hf5d242d_0 - locket=1.0.0=pyhd8ed1ab_0 - - lxml=4.9.3=py311hbafe683_1 + - lxml=4.9.3=py311hdef8331_2 - marko=2.0.2=pyhd8ed1ab_0 - markupsafe=2.1.3=py311heffc1b2_1 - mdurl=0.1.0=pyhd8ed1ab_0 @@ -203,7 +203,6 @@ dependencies: - msgpack-python=1.0.7=py311hd03642b_0 - multidict=6.0.4=py311he2be06e_1 - multimethod=1.9.1=pyhd8ed1ab_0 - - munch=4.0.0=pyhd8ed1ab_0 - munkres=1.1.4=pyh9f0ad1d_0 - mypy_extensions=1.0.0=pyha770c72_0 - nest-asyncio=1.5.8=pyhd8ed1ab_0 @@ -217,6 +216,7 @@ dependencies: - pillow=10.1.0=py311hb9c5795_0 - pkginfo=1.9.6=pyhd8ed1ab_0 - pkgutil-resolve-name=1.3.10=pyhd8ed1ab_1 + - platformdirs=4.1.0=pyhd8ed1ab_0 - pluggy=1.3.0=pyhd8ed1ab_0 - prometheus_client=0.19.0=pyhd8ed1ab_0 - psutil=5.9.5=py311heffc1b2_1 @@ -239,14 +239,14 @@ dependencies: - pywin32-on-windows=0.1.0=pyh1179c8e_3 - pyxlsb=1.0.10=pyhd8ed1ab_0 - pyyaml=6.0.1=py311heffc1b2_1 - - pyzmq=25.1.1=py311he9c0408_2 + - pyzmq=25.1.2=py311h6727e71_0 - regex=2023.10.3=py311heffc1b2_0 - rfc3986=2.0.0=pyhd8ed1ab_0 - rfc3986-validator=0.1.1=pyh9f0ad1d_0 - rpds-py=0.13.2=py311h94f323b_0 - rtree=1.1.0=py311hd698ff7_0 - ruamel.yaml.clib=0.2.7=py311heffc1b2_2 - - ruff=0.1.6=py311h6fc163c_0 + - ruff=0.1.7=py311h6fc163c_0 - setuptools=68.2.2=pyhd8ed1ab_0 - shellingham=1.5.4=pyhd8ed1ab_0 - simpleeval=0.9.13=pyhd8ed1ab_1 @@ -279,7 +279,7 @@ dependencies: - wcwidth=0.2.12=pyhd8ed1ab_0 - webcolors=1.13=pyhd8ed1ab_0 - webencodings=0.5.1=pyhd8ed1ab_2 - - websocket-client=1.6.4=pyhd8ed1ab_0 + - websocket-client=1.7.0=pyhd8ed1ab_0 - websockets=10.4=py311he2be06e_1 - wheel=0.42.0=pyhd8ed1ab_0 - widgetsnbextension=4.0.9=pyhd8ed1ab_0 @@ -310,7 +310,7 @@ dependencies: - comm=0.1.4=pyhd8ed1ab_0 - coverage=7.3.2=py311heffc1b2_0 - curl=8.4.0=h2d989ff_0 - - fonttools=4.45.1=py311h05b510d_0 + - fonttools=4.46.0=py311h05b510d_0 - gitdb=4.0.11=pyhd8ed1ab_0 - graphql-core=3.2.3=pyhd8ed1ab_0 - grpcio=1.59.2=py311h79dd126_0 @@ -320,7 +320,7 @@ dependencies: - hdf5=1.14.2=nompi_h3aba7b3_100 - html5lib=1.1=pyh9f0ad1d_0 - hypothesis=6.91.0=pyha770c72_0 - - importlib-metadata=6.8.0=pyha770c72_0 + - importlib-metadata=7.0.0=pyha770c72_0 - importlib_resources=6.1.1=pyhd8ed1ab_0 - isodate=0.6.1=pyhd8ed1ab_0 - janus=1.0.0=pyhd8ed1ab_0 @@ -329,6 +329,7 @@ dependencies: - jinja2=3.1.2=pyhd8ed1ab_1 - joblib=1.3.2=pyhd8ed1ab_0 - jsonlines=4.0.0=pyhd8ed1ab_0 + - jupyter_core=5.5.0=py311h267d04e_0 - jupyterlab_pygments=0.3.0=pyhd8ed1ab_0 - latexcodec=2.0.1=pyh9f0ad1d_0 - libcblas=3.9.0=20_osxarm64_openblas @@ -344,7 +345,6 @@ dependencies: - pexpect=4.8.0=pyh1a96a4e_2 - pint=0.22=pyhd8ed1ab_1 - pip=23.3.1=pyhd8ed1ab_0 - - platformdirs=4.0.0=pyhd8ed1ab_0 - poppler=23.11.0=hcdd998b_0 - postgresql=16.1=hc6ab77f_0 - proj=9.3.0=h52fb9d0_2 @@ -359,7 +359,7 @@ dependencies: - python-slugify=8.0.1=pyhd8ed1ab_2 - pyu2f=0.1.5=pyhd8ed1ab_0 - qtpy=2.4.1=pyhd8ed1ab_0 - - referencing=0.31.1=pyhd8ed1ab_0 + - referencing=0.32.0=pyhd8ed1ab_0 - restructuredtext_lint=1.4.0=pyhd8ed1ab_0 - rfc3339-validator=0.1.4=pyhd8ed1ab_0 - rsa=4.9=pyhd8ed1ab_0 @@ -371,6 +371,7 @@ dependencies: - typing_inspect=0.9.0=pyhd8ed1ab_0 - universal_pathlib=0.1.4=pyhd8ed1ab_0 - urllib3=1.26.18=pyhd8ed1ab_0 + - virtualenv=20.25.0=pyhd8ed1ab_0 - watchdog=3.0.0=py311heffc1b2_1 - xerces-c=3.2.4=hd886eac_3 - yarl=1.9.3=py311h05b510d_0 @@ -381,22 +382,21 @@ dependencies: - arrow=1.3.0=pyhd8ed1ab_0 - async-timeout=4.0.3=pyhd8ed1ab_0 - aws-crt-cpp=0.24.7=hba4ac3b_6 - - botocore=1.33.5=pyhd8ed1ab_0 + - botocore=1.33.9=pyhd8ed1ab_0 - branca=0.7.0=pyhd8ed1ab_1 - croniter=2.0.1=pyhd8ed1ab_0 - - cryptography=41.0.5=py311h71175c2_0 + - cryptography=41.0.7=py311h08c85a6_1 - fqdn=1.5.1=pyhd8ed1ab_0 - geotiff=1.7.1=h71398c0_14 - gitpython=3.1.40=pyhd8ed1ab_0 - google-crc32c=1.1.2=py311h533d1a3_5 - - googleapis-common-protos=1.61.0=pyhd8ed1ab_0 + - googleapis-common-protos=1.62.0=pyhd8ed1ab_0 - gql=3.4.1=pyhd8ed1ab_0 - graphql-relay=3.2.0=pyhd8ed1ab_0 - grpcio-health-checking=1.59.2=pyhd8ed1ab_0 - httpcore=1.0.2=pyhd8ed1ab_0 - - importlib_metadata=6.8.0=hd8ed1ab_0 + - importlib_metadata=7.0.0=hd8ed1ab_0 - jsonschema-specifications=2023.11.2=pyhd8ed1ab_0 - - jupyter_core=5.5.0=py311h267d04e_0 - jupyter_server_terminals=0.4.4=pyhd8ed1ab_1 - kealib=1.5.2=h47b5e36_1 - libnetcdf=4.9.2=nompi_hb2fb864_112 @@ -421,32 +421,31 @@ dependencies: - rich=13.7.0=pyhd8ed1ab_0 - sqlalchemy=2.0.23=py311h05b510d_0 - stack_data=0.6.2=pyhd8ed1ab_0 - - starlette=0.32.0.post1=pyhd8ed1ab_0 + - starlette=0.33.0=pyhd8ed1ab_0 - tiledb=2.16.3=he15c4da_3 - ukkonen=1.0.1=py311he4fd1f5_4 - uvicorn=0.24.0.post1=py311h267d04e_0 - - virtualenv=20.24.7=pyhd8ed1ab_0 - watchfiles=0.21.0=py311h94f323b_0 - aiohttp=3.8.6=py311h05b510d_1 - - alembic=1.12.1=pyhd8ed1ab_0 - - arelle-release=2.17.7=pyhd8ed1ab_0 + - alembic=1.13.0=pyhd8ed1ab_0 + - arelle-release=2.18.0=pyhd8ed1ab_0 - argon2-cffi=23.1.0=pyhd8ed1ab_0 - aws-sdk-cpp=1.11.182=h31542fa_7 - bottleneck=1.3.7=py311hb49d859_1 - cachecontrol=0.13.1=pyhd8ed1ab_0 - contourpy=1.2.0=py311hd03642b_0 - - dask-core=2023.11.0=pyhd8ed1ab_0 + - dask-core=2023.12.0=pyhd8ed1ab_0 - dnspython=2.4.2=pyhd8ed1ab_1 - ensureconda=1.4.3=pyhd8ed1ab_0 - - folium=0.15.0=pyhd8ed1ab_0 + - folium=0.15.1=pyhd8ed1ab_0 - google-resumable-media=2.6.0=pyhd8ed1ab_0 - graphene=3.3=pyhd8ed1ab_0 - grpcio-status=1.59.2=pyhd8ed1ab_0 - gtk2=2.24.33=h57013de_2 - h3-py=3.7.6=py311ha891d26_1 - httpx=0.25.2=pyhd8ed1ab_0 - - identify=2.5.32=pyhd8ed1ab_0 - - ipython=8.18.1=pyh31011fe_1 + - identify=2.5.33=pyhd8ed1ab_0 + - ipython=8.18.1=pyh707e725_3 - isoduration=20.11.0=pyhd8ed1ab_0 - jsonschema=4.20.0=pyhd8ed1ab_0 - jupyter_client=8.6.0=pyhd8ed1ab_0 @@ -471,16 +470,16 @@ dependencies: - typeguard=4.1.5=pyhd8ed1ab_1 - typer=0.9.0=pyhd8ed1ab_0 - uvicorn-standard=0.24.0.post1=ha1ab1f8_0 - - boto3=1.33.5=pyhd8ed1ab_0 + - boto3=1.33.9=pyhd8ed1ab_0 - cachecontrol-with-filecache=0.13.1=pyhd8ed1ab_0 - - dagster=1.5.9=pyhd8ed1ab_0 + - dagster=1.5.10=pyhd8ed1ab_0 - datasette=0.64.4=pyhd8ed1ab_1 - doc8=1.1.1=pyhd8ed1ab_0 - email-validator=2.1.0.post1=pyhd8ed1ab_0 - frictionless=4.40.8=pyh6c4a22f_0 - gdal=3.8.0=py311h32a4f3d_6 - geopandas-base=0.14.1=pyha770c72_0 - - google-auth=2.24.0=pyhca7485f_0 + - google-auth=2.25.1=pyhca7485f_0 - gql-with-requests=3.4.1=pyhd8ed1ab_0 - graphviz=9.0.0=h3face73_1 - ipykernel=6.26.0=pyh3cd1d5f_0 @@ -493,14 +492,14 @@ dependencies: - pre-commit=3.5.0=pyha770c72_0 - pydantic-settings=2.1.0=pyhd8ed1ab_1 - requests-oauthlib=1.3.1=pyhd8ed1ab_0 - - scikit-learn=1.3.2=py311ha25ca4d_1 + - scikit-learn=1.3.2=py311ha25ca4d_2 - timezonefinder=6.2.0=py311h05b510d_2 - catalystcoop.ferc_xbrl_extractor=1.3.1=pyhd8ed1ab_0 - conda-lock=2.5.1=pyhd8ed1ab_0 - - dagster-graphql=1.5.9=pyhd8ed1ab_0 - - dagster-postgres=0.21.9=pyhd8ed1ab_1 - - fiona=1.9.5=py311h4760b73_1 - - google-api-core=2.14.0=pyhd8ed1ab_0 + - dagster-graphql=1.5.10=pyhd8ed1ab_0 + - dagster-postgres=0.21.10=pyhd8ed1ab_0 + - fiona=1.9.5=py311h4760b73_2 + - google-api-core=2.15.0=pyhd8ed1ab_0 - google-auth-oauthlib=1.1.0=pyhd8ed1ab_0 - jupyter_console=6.6.3=pyhd8ed1ab_0 - jupyter_events=0.9.0=pyhd8ed1ab_0 @@ -514,23 +513,23 @@ dependencies: - qtconsole-base=5.5.1=pyha770c72_0 - recordlinkage=0.16=pyhd8ed1ab_0 - tabulator=1.53.5=pyhd8ed1ab_0 - - dagster-webserver=1.5.9=pyhd8ed1ab_0 + - dagster-webserver=1.5.10=pyhd8ed1ab_0 - geopandas=0.14.1=pyhd8ed1ab_0 - - google-cloud-core=2.3.3=pyhd8ed1ab_0 + - google-cloud-core=2.4.1=pyhd8ed1ab_0 - libarrow-dataset=14.0.1=had9dd58_3_cpu - libarrow-flight-sql=14.0.1=h660fe36_3_cpu - - nbconvert-core=7.11.0=pyhd8ed1ab_0 + - nbconvert-core=7.12.0=pyhd8ed1ab_0 - tableschema=1.19.3=pyh9f0ad1d_0 - datapackage=1.15.2=pyh44b312d_0 - google-cloud-storage=2.13.0=pyhca7485f_0 - - jupyter_server=2.11.1=pyhd8ed1ab_0 + - jupyter_server=2.12.1=pyhd8ed1ab_0 - libarrow-substrait=14.0.1=h594d712_3_cpu - - nbconvert-pandoc=7.11.0=pyhd8ed1ab_0 - - gcsfs=2023.10.0=pyhd8ed1ab_0 + - nbconvert-pandoc=7.12.0=pyhd8ed1ab_0 + - gcsfs=2023.12.1=pyhd8ed1ab_0 - jupyter-lsp=2.2.1=pyhd8ed1ab_0 - jupyter-resource-usage=1.0.1=pyhd8ed1ab_0 - jupyterlab_server=2.25.2=pyhd8ed1ab_0 - - nbconvert=7.11.0=pyhd8ed1ab_0 + - nbconvert=7.12.0=pyhd8ed1ab_0 - notebook-shim=0.2.3=pyhd8ed1ab_0 - pyarrow=14.0.1=py311h637fcfe_3_cpu - jupyterlab=4.0.9=pyhd8ed1ab_0 diff --git a/pyproject.toml b/pyproject.toml index 7a970bd38b..86a02af819 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,7 +18,7 @@ dependencies = [ "build>=1.0", "catalystcoop.dbfread>=3.0,<3.1", "catalystcoop.ferc-xbrl-extractor>=1.2.0,<2", - "click>=8.1", + "click>=8", "coloredlogs>=14.0", # Dagster requires 14.0 "conda-lock>=2.5.1", "coverage>=7", @@ -119,17 +119,12 @@ keywords = [ ] [project.scripts] -metadata_to_rst = "pudl.convert.metadata_to_rst:main" -epacems_to_parquet = "pudl.convert.epacems_to_parquet:main" +metadata_to_rst = "pudl.convert.metadata_to_rst:metadata_to_rst" ferc_to_sqlite = "pudl.ferc_to_sqlite.cli:main" -pudl_datastore = "pudl.workspace.datastore:main" -pudl_etl = "pudl.cli.etl:main" -pudl_setup = "pudl.workspace.setup_cli:main" -state_demand = "pudl.analysis.state_demand:main" -pudl_check_fks = "pudl.etl.check_foreign_keys:main" -# pudl_territories currently blows up memory usage to 100+ GB. -# See https://github.com/catalyst-cooperative/pudl/issues/1174 -# pudl_territories = "pudl.analysis.service_territory:main" +pudl_datastore = "pudl.workspace.datastore:pudl_datastore" +pudl_etl = "pudl.etl.cli:pudl_etl" +pudl_check_fks = "pudl.etl.check_foreign_keys:pudl_check_fks" +pudl_service_territories = "pudl.analysis.service_territory:pudl_service_territories" [project.urls] "Homepage" = "https://catalyst.coop/pudl/" @@ -268,16 +263,19 @@ filterwarnings = [ "once:The behavior of DataFrame concatenation with empty or all-NA entries is deprecated.:FutureWarning", ] -addopts = "--verbose --pdbcls=IPython.terminal.debugger:TerminalPdb" +markers = [ + "slow: marks tests as slow (deselect with '-m \"not slow\"')", +] +# Oddly, despite the use of --cov-config=pyproject.toml here, pytest does not seem to +# pick up the source directories specified in the [tool.coverage.run] section below. +# (though it *does* pick up the omit parameters!). This means we need to specify the +# source directories we want to collect coverage twice, and keep the two in sync. +addopts = "--verbose --pdbcls=IPython.terminal.debugger:TerminalPdb --cov-append --cov-config=pyproject.toml --cov-report=xml --cov=src/pudl --cov=test/integration --cov=test/unit" log_format = "%(asctime)s [%(levelname)8s] %(name)s:%(lineno)s %(message)s" log_date_format = "%Y-%m-%d %H:%M:%S" log_cli = "true" log_cli_level = "DEBUG" -doctest_optionflags = [ - "NORMALIZE_WHITESPACE", - "IGNORE_EXCEPTION_DETAIL", - "ELLIPSIS", -] +doctest_optionflags = ["NORMALIZE_WHITESPACE", "IGNORE_EXCEPTION_DETAIL", "ELLIPSIS"] [tool.conda-lock] channels = ["conda-forge", "defaults"] @@ -300,3 +298,31 @@ prettier = ">=3.0" python = ">=3.11,<3.12" sqlite = ">=3.43" curl = ">=8.4.0" + +[tool.coverage.run] +# See note above on need to specify separate sources for pytest-coverage and coverage. +source = ["src/pudl/", "test/integration/", "test/unit/"] +omit = [ + # Never hit by integration tests: + "src/pudl/validate.py", +] + +[tool.coverage.report] +precision = 1 +sort = "miss" +skip_empty = true +exclude_lines = [ + # Have to re-enable the standard pragma + "pragma: no cover", + + # Don't complain if tests don't hit defensive assertion code: + "raise AssertionError", + "raise NotImplementedError", + + # Don't complain if non-runnable code isn't run: + "if 0:", + "if __name__ == .__main__.:", + + # Stuff that's not expected to run normally... + "logger.debug", +] diff --git a/src/pudl/__init__.py b/src/pudl/__init__.py index a578d8f855..0496f353e7 100644 --- a/src/pudl/__init__.py +++ b/src/pudl/__init__.py @@ -4,7 +4,6 @@ from . import ( analysis, - cli, convert, etl, extract, diff --git a/src/pudl/__main__.py b/src/pudl/__main__.py index bf9e0ad091..8f238a831d 100644 --- a/src/pudl/__main__.py +++ b/src/pudl/__main__.py @@ -2,7 +2,7 @@ import sys -import pudl.cli +from pudl.etl.cli import pudl_etl if __name__ == "__main__": - sys.exit(pudl.cli.etl.main()) + sys.exit(pudl_etl()) diff --git a/src/pudl/analysis/service_territory.py b/src/pudl/analysis/service_territory.py index 322868ee29..6e5c50f3a1 100644 --- a/src/pudl/analysis/service_territory.py +++ b/src/pudl/analysis/service_territory.py @@ -5,16 +5,17 @@ the historical spatial extent of utility and balancing area territories. Output the resulting geometries for use in other applications. """ -import argparse import math +import pathlib import sys from collections.abc import Iterable from typing import Literal +import click import geopandas as gpd import pandas as pd import sqlalchemy as sa -from dagster import AssetKey, AssetsDefinition, Field, asset +from dagster import AssetsDefinition, Field, asset from matplotlib import pyplot as plt import pudl @@ -255,22 +256,41 @@ def get_territory_geometries( ) -def _save_geoparquet(gdf, entity_type, dissolve, limit_by_state): - # For filenames based on input args: - dissolved = "" - if dissolve: - dissolved = "_dissolved" - else: - # States & counties only remain at this point if we didn't dissolve - for col in ("county_id_fips", "state_id_fips"): - # pandas.NA values are not compatible with Parquet Strings yet. - gdf[col] = gdf[col].fillna("") - limited = "" - if limit_by_state: - limited = "_limited" - # Save the geometries to a GeoParquet file - fn = f"{entity_type}_geom{limited+dissolved}.pq" - gdf.to_parquet(fn, index=False) +def _save_geoparquet( + gdf: gpd.GeoDataFrame, + entity_type: Literal["util", "ba"], + dissolve: bool, + limit_by_state: bool, + output_dir: pathlib.Path | None = None, +) -> None: + """Save utility or balancing authority service territory geometries to GeoParquet. + + In order to prevent the geometry data from exceeding the 2GB maximum size of an + Arrow object, we need to keep the row groups small. Sort the dataframe by the + primary key columns to minimize the number of values in any row group. Output + filename is constructed based on input arguments. + + Args: + gdf: GeoDataframe containing utility or balancing authority geometries. + entity_type: short string indicating whether we're outputting utility or + balancing authority geometries. + dissolve: Wether the individual county geometries making up the service + territories have been merged together. Used to construct filename. + limit_by_state: Whether service territories have been limited to include only + counties in states where the utilities reported sales. Used to construct + filename. + output_dir: Path to the directory where the GeoParquet file will be written. + + """ + entity_name = "balancing_authority" if entity_type == "ba" else "utility" + dissolved = "_dissolved" if dissolve else "" + limited = "_limited" if limit_by_state else "" + if output_dir is None: + output_dir = pathlib.Path.cwd() + file_path = output_dir / f"{entity_name}_geometry{limited}{dissolved}.parquet" + gdf.sort_values(["report_date", f"{entity_name}_id_eia"]).to_parquet( + file_path, row_group_size=512, compression="snappy", index=False + ) def compile_geoms( @@ -282,8 +302,10 @@ def compile_geoms( census_counties: pd.DataFrame, entity_type: Literal["ba", "util"], save_format: Literal["geoparquet", "geodataframe", "dataframe"], + output_dir: pathlib.Path | None = None, dissolve: bool = False, limit_by_state: bool = True, + years: list[int] = [], ): """Compile all available utility or balancing authority geometries. @@ -293,11 +315,22 @@ def compile_geoms( balancing authority, with geometries available at the county level. """ logger.info( - "Compiling %s geometries with dissolve=%s and limit_by_state=%s.", - entity_type, - dissolve, - limit_by_state, + f"Compiling {entity_type} geometries with {dissolve=}, {limit_by_state=}, " + f"and {years=}." ) + if save_format == "geoparquet" and output_dir is None: + raise ValueError("No output_dir provided while writing geoparquet.") + + if years: + + def _limit_years(df: pd.DataFrame) -> pd.DataFrame: + return df[df.report_date.dt.year.isin(years)] + + balancing_authority_eia861 = _limit_years(balancing_authority_eia861) + balancing_authority_assn_eia861 = _limit_years(balancing_authority_assn_eia861) + denorm_utilities_eia = _limit_years(denorm_utilities_eia) + service_territory_eia861 = _limit_years(service_territory_eia861) + utility_assn_eia861 = _limit_years(utility_assn_eia861) utilids_all_eia = utility_ids_all_eia( denorm_utilities_eia, service_territory_eia861 @@ -325,17 +358,13 @@ def compile_geoms( dissolve=dissolve, ) if save_format == "geoparquet": - if dissolve: - # States & counties only remain at this point if we didn't dissolve - for col in ("county_id_fips", "state_id_fips"): - # pandas.NA values are not compatible with Parquet Strings yet. - geom[col] = geom[col].fillna("") - - _save_geoparquet( # To do: update to use new io manager. + # TODO[dagster]: update to use IO Manager. + _save_geoparquet( geom, entity_type=entity_type, dissolve=dissolve, limit_by_state=limit_by_state, + output_dir=output_dir, ) elif save_format == "dataframe": geom = pd.DataFrame(geom.drop(columns="geometry")) @@ -424,25 +453,25 @@ def dagster_compile_geoms( ################################################################################ # Functions for visualizing the service territory geometries ################################################################################ -def plot_historical_territory(gdf, id_col, id_val): +def plot_historical_territory( + gdf: gpd.GeoDataFrame, + id_col: str, + id_val: str | int, +) -> None: """Plot all the historical geometries defined for the specified entity. This is useful for exploring how a particular entity's service territory has evolved over time, or for identifying individual missing or inaccurate territories. Args: - gdf (geopandas.GeoDataFrame): A geodataframe containing geometries pertaining - electricity planning areas. Can be broken down by county FIPS code, or - have a single record containing a geometry for each combination of - report_date and the column being used to select planning areas (see - below). - id_col (str): The label of a column in gdf that identifies the planning area - to be visualized, like utility_id_eia, balancing_authority_id_eia, or - balancing_authority_code_eia. - id_val (str or int): The value identifying the - - Returns: - None + gdf: A geodataframe containing geometries pertaining electricity planning areas. + Can be broken down by county FIPS code, or have a single record containing a + geometry for each combination of report_date and the column being used to + select planning areas (see below). + id_col: The label of a column in gdf that identifies the planning area to be + visualized, like ``utility_id_eia``, ``balancing_authority_id_eia``, or + ``balancing_authority_code_eia``. + id_val: The ID of the entity whose territory should be plotted. """ if id_col not in gdf.columns: raise ValueError(f"The input id_col {id_col} doesn't exist in this GDF.") @@ -483,11 +512,11 @@ def plot_historical_territory(gdf, id_col, id_val): def plot_all_territories( - gdf, - report_date, - respondent_type=("balancing_authority", "utility"), - color="black", - alpha=0.25, + gdf: gpd.GeoDataFrame, + report_date: str, + respondent_type: str | Iterable[str] = ("balancing_authority", "utility"), + color: str = "black", + alpha: float = 0.25, ): """Plot all of the planning areas of a given type for a given report date. @@ -496,16 +525,15 @@ def plot_all_territories( entangled with the FERC 714 data. Args: - gdf (geopandas.GeoDataFrame): GeoDataFrame containing planning area - geometries, organized by respondent_id_ferc714 and report_date. - - report_date (datetime): A Datetime indicating what year's planning - areas should be displayed. - respondent_type (str or iterable): Type of respondent whose planning + gdf: GeoDataFrame containing planning area geometries, organized by + ``respondent_id_ferc714`` and ``report_date``. + report_date: A string representing a datetime that indicates what year's + planning areas should be displayed. + respondent_type: Type of respondent whose planning areas should be displayed. Either "utility" or "balancing_authority" or an iterable collection containing both. - color (str): Color to use for the planning areas. - alpha (float): Transparency to use for the planning areas. + color: Color to use for the planning areas. + alpha: Transparency to use for the planning areas. Returns: matplotlib.axes.Axes @@ -536,77 +564,161 @@ def plot_all_territories( ################################################################################ -# Functions that provide a CLI to the service territory module +# Provide a CLI for generating service territories ################################################################################ -def parse_command_line(argv): - """Parse script command line arguments. See the -h option. +@click.command( + context_settings={"help_option_names": ["-h", "--help"]}, +) +@click.option( + "--entity-type", + type=click.Choice(["util", "ba"]), + default="util", + show_default=True, + help=( + "What type of entity's service territories should be generated: Utility " + "(util) or Balancing Authority (ba)?" + ), +) +@click.option( + "--limit-by-state/--no-limit-by-state", + default=False, + help=( + "Limit service territories to including only counties located in states where " + "the utility or balancing authority also reported electricity sales in EIA-861 " + "in the year that the geometry pertains to. In theory a utility could serve a " + "county, but not sell any electricity there, but that seems like an unusual " + "situation." + ), + show_default=True, +) +@click.option( + "--year", + "-y", + "years", + type=click.IntRange(min=2001), + default=[], + multiple=True, + help=( + "Limit service territories generated to those from the given year. This can " + "dramatically reduce the memory and CPU intensity of the geospatial " + "operations. Especially useful for testing. Option can be used multiple times " + "toselect multiple years." + ), +) +@click.option( + "--dissolve/--no-dissolve", + default=True, + help=( + "Dissolve county level geometries to the utility or balancing authority " + "boundaries. The dissolve operation may take several minutes and is quite " + "memory intensive, but results in significantly smaller files, in which each " + "record contains the whole geometry of a utility or balancing authority. The " + "un-dissolved geometries use many records to describe each service territory, " + "with each record containing the geometry of a single constituent county." + ), + show_default=True, +) +@click.option( + "--output-dir", + "-o", + type=click.Path( + exists=True, + writable=True, + dir_okay=True, + file_okay=False, + resolve_path=True, + path_type=pathlib.Path, + ), + default=pathlib.Path.cwd(), + show_default=True, + help=( + "Path to the directory where the service territory geometries should be saved. " + "Defaults to the current working directory. Filenames are constructed based on " + "the other flags provided." + ), +) +@click.option( + "--logfile", + help="If specified, write logs to this file.", + type=click.Path( + exists=False, + resolve_path=True, + path_type=pathlib.Path, + ), +) +@click.option( + "--loglevel", + default="INFO", + type=click.Choice( + ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], case_sensitive=False + ), + show_default=True, +) +def pudl_service_territories( + entity_type: Literal["util", "ba"], + dissolve: bool, + output_dir: pathlib.Path, + limit_by_state: bool, + years: list[int], + logfile: pathlib.Path, + loglevel: str, +): + """Compile historical utility and balancing area service territory geometries. - Args: - argv (list): command line arguments including caller file name. + This script produces GeoParquet files describing the historical service territories + of utilities and balancing authorities based on data reported in the EIA Form 861 + and county geometries from the US Census DP1 geodatabase. - Returns: - dict: A dictionary mapping command line arguments to their values. - """ - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "-d", - "--dissolve", - dest="dissolve", - action="store_true", - default=False, - help="Dissolve county level geometries to utility or balancing authorities", - ) - parser.add_argument( - "--logfile", - default=None, - type=str, - help="If specified, write logs to this file.", - ) - parser.add_argument( - "--loglevel", - help="Set logging level (DEBUG, INFO, WARNING, ERROR, or CRITICAL).", - default="INFO", - ) - return parser.parse_args(argv[1:]) + See: https://geoparquet.org/ for more on the GeoParquet file format. + Usage examples: -def main(): - """Compile historical utility and balancing area territories.""" + pudl_service_territories --entity-type ba --dissolve --limit-by-state + pudl_service_territories --entity-type util + """ # Display logged output from the PUDL package: + pudl.logging_helpers.configure_root_logger(logfile=logfile, loglevel=loglevel) - args = parse_command_line(sys.argv) - pudl.logging_helpers.configure_root_logger( - logfile=args.logfile, loglevel=args.loglevel + pudl_engine = sa.create_engine(PudlPaths().pudl_db) + # Load the required US Census DP1 county geometry data: + dp1_engine = PudlPaths().sqlite_db_uri("censusdp1tract") + sql = """ +SELECT + geoid10, + namelsad10, + dp0010001, + shape AS geometry +FROM + county_2010census_dp1; +""" + county_gdf = gpd.read_postgis( + sql, + con=dp1_engine, + geom_col="geometry", + crs="EPSG:4326", ) - pudl_engine = sa.create_engine(PudlPaths().pudl_db) - # Load the US Census DP1 county data: - county_gdf = pudl.etl.defs.load_asset_value(AssetKey("county_censusdp1")) - - kwargs_dicts = [ - {"entity_type": "util", "limit_by_state": False}, - {"entity_type": "util", "limit_by_state": True}, - {"entity_type": "ba", "limit_by_state": True}, - {"entity_type": "ba", "limit_by_state": False}, - ] - - for kwargs in kwargs_dicts: - _ = compile_geoms( - balancing_authority_eia861=pd.read_sql( - "balancing_authority_eia861", pudl_engine - ), - balancing_authority_assn_eia861=pd.read_sql( - "balancing_authority_assn_eia861", pudl_engine - ), - denorm_utilities_eia=pd.read_sql(AssetKey("denorm_utilities_eia")), - service_territory_eia861=pd.read_sql(AssetKey("service_territory_eia861")), - utility_assn_eia861=pd.read_sql("utility_assn_eia861", pudl_engine), - census_counties=county_gdf, - dissolve=args.dissolve, - save_format="geoparquet", - **kwargs, - ) + _ = compile_geoms( + balancing_authority_eia861=pd.read_sql( + "balancing_authority_eia861", + pudl_engine, + ), + balancing_authority_assn_eia861=pd.read_sql( + "balancing_authority_assn_eia861", + pudl_engine, + ), + denorm_utilities_eia=pd.read_sql("denorm_utilities_eia", pudl_engine), + service_territory_eia861=pd.read_sql("service_territory_eia861", pudl_engine), + utility_assn_eia861=pd.read_sql("utility_assn_eia861", pudl_engine), + census_counties=county_gdf, + dissolve=dissolve, + save_format="geoparquet", + output_dir=output_dir, + entity_type=entity_type, + limit_by_state=limit_by_state, + years=years, + ) if __name__ == "__main__": - sys.exit(main()) + sys.exit(pudl_service_territories()) diff --git a/src/pudl/analysis/state_demand.py b/src/pudl/analysis/state_demand.py index bcec43c064..936ab0effd 100644 --- a/src/pudl/analysis/state_demand.py +++ b/src/pudl/analysis/state_demand.py @@ -1,4 +1,4 @@ -"""Predict state-level electricity demand. +"""Estimate historical hourly state-level electricity demand. Using hourly electricity demand reported at the balancing authority and utility level in the FERC 714, and service territories for utilities and balancing autorities inferred @@ -15,27 +15,19 @@ manual and could certainly be improved, but overall the results seem reasonable. Additional predictive spatial variables will be required to obtain more granular electricity demand estimates (e.g. at the county level). - -Currently the script takes no arguments and simply runs a predefined analysis across all -states and all years for which both EIA 861 and FERC 714 data are available, and outputs -the results as a CSV in PUDL_DIR/local/state-demand/demand.csv """ -import argparse import datetime -import sys from collections.abc import Iterable from typing import Any import geopandas as gpd -import matplotlib.pyplot as plt import numpy as np import pandas as pd -from dagster import AssetKey, AssetOut, Field, asset, multi_asset +from dagster import AssetOut, Field, asset, multi_asset import pudl.analysis.timeseries_cleaning import pudl.logging_helpers import pudl.output.pudltabl -import pudl.workspace.setup from pudl.metadata.dfs import POLITICAL_SUBDIVISIONS logger = pudl.logging_helpers.get_logger(__name__) @@ -592,7 +584,8 @@ def predicted_state_hourly_demand( Args: imputed_hourly_demand_ferc714: Hourly demand timeseries, with columns - `respondent_id_ferc714`, report `year`, `utc_datetime`, and `demand_mwh`. + ``respondent_id_ferc714``, report ``year``, ``utc_datetime``, and + ``demand_mwh``. county_censusdp1: The county layer of the Census DP1 shapefile. fipsified_respondents_ferc714: Annual respondents with the county FIPS IDs for their service territories. @@ -600,9 +593,8 @@ def predicted_state_hourly_demand( scaled to match these totals. Returns: - Dataframe with columns - `state_id_fips`, `utc_datetime`, `demand_mwh`, and - (if `state_totals` was provided) `scaled_demand_mwh`. + Dataframe with columns ``state_id_fips``, ``utc_datetime``, ``demand_mwh``, and + (if ``state_totals`` was provided) ``scaled_demand_mwh``. """ # Get config mean_overlaps = context.op_config["mean_overlaps"] @@ -665,233 +657,3 @@ def predicted_state_hourly_demand( # Sum demand by state by matching UTC time fields = [x for x in ["demand_mwh", "scaled_demand_mwh"] if x in df] return df.groupby(["state_id_fips", "utc_datetime"], as_index=False)[fields].sum() - - -def plot_demand_timeseries( - a: pd.DataFrame, - b: pd.DataFrame = None, - window: int = 168, - title: str = None, - path: str = None, -) -> None: - """Make a timeseries plot of predicted and reference demand. - - Args: - a: Predicted demand with columns `utc_datetime` and any of - `demand_mwh` (in grey) and `scaled_demand_mwh` (in orange). - b: Reference demand with columns `utc_datetime` and `demand_mwh` (in red). - window: Width of window (in rows) to use to compute rolling means, - or `None` to plot raw values. - title: Plot title. - path: Plot path. If provided, the figure is saved to file and closed. - """ - plt.figure(figsize=(16, 8)) - # Plot predicted - for field, color in [("demand_mwh", "grey"), ("scaled_demand_mwh", "orange")]: - if field not in a: - continue - y = a[field] - if window: - y = y.rolling(window).mean() - plt.plot( - a["utc_datetime"], y, color=color, alpha=0.5, label=f"Predicted ({field})" - ) - # Plot expected - if b is not None: - y = b["demand_mwh"] - if window: - y = y.rolling(window).mean() - plt.plot( - b["utc_datetime"], y, color="red", alpha=0.5, label="Reference (demand_mwh)" - ) - if title: - plt.title(title) - plt.ylabel("Demand (MWh)") - plt.legend() - if path: - plt.savefig(path, bbox_inches="tight") - plt.close() - - -def plot_demand_scatter( - a: pd.DataFrame, - b: pd.DataFrame, - title: str = None, - path: str = None, -) -> None: - """Make a scatter plot comparing predicted and reference demand. - - Args: - a: Predicted demand with columns `utc_datetime` and any of - `demand_mwh` (in grey) and `scaled_demand_mwh` (in orange). - b: Reference demand with columns `utc_datetime` and `demand_mwh`. - Every element in `utc_datetime` must match the one in `a`. - title: Plot title. - path: Plot path. If provided, the figure is saved to file and closed. - - Raises: - ValueError: Datetime columns do not match. - """ - if not a["utc_datetime"].equals(b["utc_datetime"]): - raise ValueError("Datetime columns do not match") - plt.figure(figsize=(8, 8)) - plt.gca().set_aspect("equal") - plt.axline((0, 0), (1, 1), linestyle=":", color="grey") - for field, color in [("demand_mwh", "grey"), ("scaled_demand_mwh", "orange")]: - if field not in a: - continue - plt.scatter( - b["demand_mwh"], - a[field], - c=color, - s=0.1, - alpha=0.5, - label=f"Prediction ({field})", - ) - if title: - plt.title(title) - plt.xlabel("Reference (MWh)") - plt.ylabel("Predicted (MWh)") - plt.legend() - if path: - plt.savefig(path, bbox_inches="tight") - plt.close() - - -def compare_state_demand( - a: pd.DataFrame, b: pd.DataFrame, scaled: bool = True -) -> pd.DataFrame: - """Compute statistics comparing predicted and reference demand. - - Statistics are computed for each year. - - Args: - a: Predicted demand with columns `utc_datetime` and either - `demand_mwh` (if `scaled=False) or `scaled_demand_mwh` (if `scaled=True`). - b: Reference demand with columns `utc_datetime` and `demand_mwh`. - Every element in `utc_datetime` must match the one in `a`. - - Returns: - Dataframe with columns `year`, - `rmse` (root mean square error), and `mae` (mean absolute error). - - Raises: - ValueError: Datetime columns do not match. - """ - if not a["utc_datetime"].equals(b["utc_datetime"]): - raise ValueError("Datetime columns do not match") - field = "scaled_demand_mwh" if scaled else "demand_mwh" - df = pd.DataFrame( - { - "year": a["utc_datetime"].dt.year, - "diff": a[field] - b["demand_mwh"], - } - ) - return df.groupby(["year"], as_index=False)["diff"].agg( - { - "rmse": lambda x: np.sqrt(np.sum(x**2) / x.size), - "mae": lambda x: np.sum(np.abs(x)) / x.size, - } - ) - - -# --- Parse Command Line Args --- # -def parse_command_line(argv): - """Skeletal command line argument parser to provide a help message.""" - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--logfile", - default=None, - type=str, - help="If specified, write logs to this file.", - ) - parser.add_argument( - "--loglevel", - help="Set logging level (DEBUG, INFO, WARNING, ERROR, or CRITICAL).", - default="INFO", - ) - return parser.parse_args(argv[1:]) - - -# --- Example usage --- # - - -def main(): - """Predict state demand.""" - # --- Parse command line args --- # - args = parse_command_line(sys.argv) - - # --- Connect to PUDL logger --- # - pudl.logging_helpers.configure_root_logger( - logfile=args.logfile, loglevel=args.loglevel - ) - - # --- Connect to PUDL database --- # - - # --- Read in inputs from PUDL + dagster cache --- # - prediction = pudl.etl.defs.load_asset_value( - AssetKey("predicted_state_hourly_demand") - ) - - # --- Export results --- # - - local_dir = pudl.workspace.setup.PudlPaths().data_dir / "local" - ventyx_path = local_dir / "ventyx/state_level_load_2007_2018.csv" - base_dir = local_dir / "state-demand" - base_dir.mkdir(parents=True, exist_ok=True) - demand_path = base_dir / "demand.csv" - stats_path = base_dir / "demand-stats.csv" - timeseries_dir = base_dir / "timeseries" - timeseries_dir.mkdir(parents=True, exist_ok=True) - scatter_dir = base_dir / "scatter" - scatter_dir.mkdir(parents=True, exist_ok=True) - - # Write predicted hourly state demand - prediction.to_csv( - demand_path, index=False, date_format="%Y%m%dT%H", float_format="%.1f" - ) - - # Load Ventyx as reference if available - reference = None - if ventyx_path.exists(): - reference = load_ventyx_hourly_state_demand(ventyx_path) - - # Plots and statistics - stats = [] - for fips in prediction["state_id_fips"].unique(): - state = lookup_state(fips) - # Filter demand by state - a = prediction.query(f"state_id_fips == '{fips}'") - b = None - title = f'{state["fips"]}: {state["name"]} ({state["code"]})' - plot_name = f'{state["fips"]}-{state["name"]}.png' - if reference is not None: - b = reference.query(f"state_id_fips == '{fips}'") - # Save timeseries plot - plot_demand_timeseries( - a, b=b, window=168, title=title, path=timeseries_dir / plot_name - ) - if b is None or b.empty: - continue - # Align predicted and reference demand - a = a.set_index("utc_datetime") - b = b.set_index("utc_datetime") - index = a.index.intersection(b.index) - a = a.loc[index].reset_index() - b = b.loc[index].reset_index() - # Compute statistics - stat = compare_state_demand(a, b, scaled=True) - stat["state_id_fips"] = fips - stats.append(stat) - # Save scatter plot - plot_demand_scatter(a, b=b, title=title, path=scatter_dir / plot_name) - - # Write statistics - if reference is not None: - pd.concat(stats, ignore_index=True).to_csv( - stats_path, index=False, float_format="%.1f" - ) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/src/pudl/cli/__init__.py b/src/pudl/cli/__init__.py deleted file mode 100644 index 2e91185eee..0000000000 --- a/src/pudl/cli/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -"""Holds various command-line interfaces for PUDL.""" - -from . import etl diff --git a/src/pudl/convert/__init__.py b/src/pudl/convert/__init__.py index 02676d1eeb..1e39a45c9e 100644 --- a/src/pudl/convert/__init__.py +++ b/src/pudl/convert/__init__.py @@ -3,16 +3,8 @@ It's often useful to be able to convert entire datasets in bulk from one format to another, both independent of and within the context of the ETL pipeline. This subpackage collects those tools together in one place. - -Currently the tools use a mix of idioms, referring either to a particular -dataset and a particular format, or two formats. Some of them read from the -original raw data as organized by the :mod:`pudl.workspace` package (e.g. -:mod:`pudl.convert.ferc_to_sqlite` or :mod:`pudl.convert.epacems_to_parquet`), -and others convert metadata into RST for use in documentation -(e.g. :mod:`pudl.convert.metadata_to_rst`). """ from . import ( censusdp1tract_to_sqlite, - epacems_to_parquet, metadata_to_rst, ) diff --git a/src/pudl/convert/epacems_to_parquet.py b/src/pudl/convert/epacems_to_parquet.py deleted file mode 100644 index f15d462acc..0000000000 --- a/src/pudl/convert/epacems_to_parquet.py +++ /dev/null @@ -1,162 +0,0 @@ -"""Process raw EPA CEMS data into a Parquet dataset outside of the PUDL ETL. - -This script transforms the raw EPA CEMS data from Zip compressed CSV files into an -Apache Parquet dataset partitioned by year and state. - -Processing the EPA CEMS data requires information that's stored in the main PUDL -database, so to run this script, you must already have a PUDL database available on your -system. -""" -import argparse -import sys -from collections.abc import Callable - -from dagster import ( - DagsterInstance, - Definitions, - JobDefinition, - build_reconstructable_job, - define_asset_job, - execute_job, -) -from dotenv import load_dotenv - -import pudl -from pudl.metadata.classes import DataSource - -logger = pudl.logging_helpers.get_logger(__name__) - - -def parse_command_line(argv): - """Parse command line arguments. See the -h option. - - Args: - argv (str): Command line arguments, including caller filename. - - Returns: - dict: Dictionary of command line arguments and their parsed values. - """ - parser = argparse.ArgumentParser(description=__doc__) - - parser.add_argument( - "-y", - "--years", - nargs="+", - type=int, - help="""Which years of EPA CEMS data should be converted to Apache - Parquet format. Default is all available years, ranging from 1995 to - the present. Note that data is typically incomplete before ~2000.""", - default=DataSource.from_id("epacems").working_partitions["years"], - ) - parser.add_argument( - "-s", - "--states", - nargs="+", - type=str.upper, - help="""Which states EPA CEMS data should be converted to Apache - Parquet format, as a list of two letter US state abbreviations. Default - is everything: all 48 continental US states plus Washington DC.""", - default=DataSource.from_id("epacems").working_partitions["states"], - ) - parser.add_argument( - "-c", - "--clobber", - action="store_true", - help="""Clobber existing parquet files if they exist. If clobber is not - included but the parquet directory already exists the _build will - fail.""", - default=False, - ) - parser.add_argument( - "--gcs-cache-path", - type=str, - help="""Load datastore resources from Google Cloud Storage. - Should be gs://bucket[/path_prefix]""", - ) - parser.add_argument( - "--bypass-local-cache", - action="store_true", - default=False, - help="If enabled, the local file cache for datastore will not be used.", - ) - parser.add_argument( - "--loglevel", - help="Set logging level (DEBUG, INFO, WARNING, ERROR, or CRITICAL).", - default="INFO", - ) - parser.add_argument( - "--logfile", - default=None, - help="If specified, write logs to this file.", - ) - arguments = parser.parse_args(argv[1:]) - return arguments - - -def epacems_job_factory(loglevel: str, logfile: str) -> Callable[[], JobDefinition]: - """Factory for parameterizing a reconstructable epacems job. - - Args: - loglevel: The log level for the job's execution. - logfile: Path to a log file for the job's execution. - - Returns: - The job definition to be executed. - """ - - def get_epacems_job(): - """Create an epacems_job wrapped by to be wrapped by reconstructable.""" - pudl.logging_helpers.configure_root_logger(logfile=logfile, loglevel=loglevel) - return Definitions( - assets=pudl.etl.default_assets, - resources=pudl.etl.default_resources, - jobs=[ - define_asset_job("epacems_job", selection="hourly_emissions_epacems") - ], - ).get_job_def("epacems_job") - - return get_epacems_job - - -def main(): - """Convert zipped EPA CEMS Hourly data to Apache Parquet format.""" - load_dotenv() - args = parse_command_line(sys.argv) - # Display logged output from the PUDL package: - pudl.logging_helpers.configure_root_logger( - logfile=args.logfile, loglevel=args.loglevel - ) - - epacems_reconstructable_job = build_reconstructable_job( - "pudl.convert.epacems_to_parquet", - "epacems_job_factory", - reconstructable_kwargs={"loglevel": args.loglevel, "logfile": args.logfile}, - ) - result = execute_job( - epacems_reconstructable_job, - instance=DagsterInstance.get(), - run_config={ - "resources": { - "dataset_settings": { - "config": {"epacems": {"years": args.years, "states": args.states}} - }, - "datastore": { - "config": { - "gcs_cache_path": args.gcs_cache_path - if args.gcs_cache_path - else "", - }, - }, - }, - }, - ) - - # Workaround to reliably getting full stack trace - if not result.success: - for event in result.all_events: - if event.event_type_value == "STEP_FAILURE": - raise Exception(event.event_specific_data.error) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/src/pudl/convert/metadata_to_rst.py b/src/pudl/convert/metadata_to_rst.py index 4e92453a09..372407330c 100644 --- a/src/pudl/convert/metadata_to_rst.py +++ b/src/pudl/convert/metadata_to_rst.py @@ -1,8 +1,9 @@ """Export PUDL table and field metadata to RST for use in documentation.""" -import argparse +import pathlib import sys -from pathlib import Path + +import click import pudl.logging_helpers from pudl.metadata.classes import Package @@ -11,64 +12,81 @@ logger = pudl.logging_helpers.get_logger(__name__) -def parse_command_line(argv): - """Parse command line arguments. See the -h option. - - Args: - argv (str): Command line arguments, including caller filename. +@click.command( + context_settings={"help_option_names": ["-h", "--help"]}, +) +@click.option( + "--skip", + "-s", + help=( + "Name of a table that should be skipped and excluded from RST output. " + "Use this option multiple times to skip multiple tables." + ), + type=str, + default=[], + multiple=True, +) +@click.option( + "--output", + "-o", + type=click.Path(), + default=None, + help="Path to which the RST output should be written. Defaults to STDOUT.", +) +@click.option( + "--docs-dir", + "-d", + type=click.Path( + exists=True, + dir_okay=True, + file_okay=False, + resolve_path=True, + path_type=pathlib.Path, + writable=True, + ), + default=pathlib.Path().cwd() / "docs", + help=( + "Path to the PUDL repository docs directory. " + "Must exist and be writable. Defaults to ./docs/" + ), +) +@click.option( + "--logfile", + help="If specified, write logs to this file.", + type=click.Path( + exists=False, + resolve_path=True, + path_type=pathlib.Path, + ), +) +@click.option( + "--loglevel", + default="INFO", + type=click.Choice( + ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], case_sensitive=False + ), +) +def metadata_to_rst( + skip: list[str], + output: pathlib.Path, + docs_dir: pathlib.Path, + logfile: pathlib.Path, + loglevel: str, +): + """Export PUDL table and field metadata to RST for use in documentation. - Returns: - dict: Dictionary of command line arguments and their parsed values. + metadata_to_rst -s bad_table1 -s bad_table2 -d ./pudl/docs -o ./datadict.rst """ - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--skip", - help="List of table names that should be skipped and excluded from RST output.", - nargs="*", - default=[], - ) - parser.add_argument( - "-o", - "--output", - help="Path to the file where the RST output should be written.", - default=False, - ) - parser.add_argument( - "--docs_dir", - help="Path to docs directory.", - type=lambda x: Path(x).resolve(), - default=Path().cwd() / "docs", - ) - parser.add_argument( - "--logfile", - default=None, - type=str, - help="If specified, write logs to this file.", - ) - parser.add_argument( - "--loglevel", - help="Set logging level (DEBUG, INFO, WARNING, ERROR, or CRITICAL).", - default="INFO", - ) - arguments = parser.parse_args(argv[1:]) - return arguments - - -def main(): - """Run conversion from json to rst.""" - args = parse_command_line(sys.argv) - pudl.logging_helpers.configure_root_logger( - logfile=args.logfile, loglevel=args.loglevel - ) + pudl.logging_helpers.configure_root_logger(logfile=logfile, loglevel=loglevel) - logger.info(f"Exporting PUDL metadata to: {args.output}") - resource_ids = [rid for rid in sorted(RESOURCE_METADATA) if rid not in args.skip] + logger.info(f"Exporting PUDL metadata to: {output}") + resource_ids = [rid for rid in sorted(RESOURCE_METADATA) if rid not in skip] package = Package.from_resource_ids(resource_ids=tuple(sorted(resource_ids))) # Sort fields within each resource by name: for resource in package.resources: resource.schema.fields = sorted(resource.schema.fields, key=lambda x: x.name) - package.to_rst(docs_dir=args.docs_dir, path=args.output) + package.to_rst(docs_dir=docs_dir, path=output) if __name__ == "__main__": - sys.exit(main()) + sys.exit(metadata_to_rst()) diff --git a/src/pudl/etl/__init__.py b/src/pudl/etl/__init__.py index bb8fcddc1a..ce9f2cc4a2 100644 --- a/src/pudl/etl/__init__.py +++ b/src/pudl/etl/__init__.py @@ -22,6 +22,7 @@ from . import ( check_foreign_keys, + cli, eia_bulk_elec_assets, epacems_assets, glue_assets, diff --git a/src/pudl/etl/check_foreign_keys.py b/src/pudl/etl/check_foreign_keys.py index e28dcebe4c..4639fb5e61 100644 --- a/src/pudl/etl/check_foreign_keys.py +++ b/src/pudl/etl/check_foreign_keys.py @@ -1,18 +1,8 @@ -"""A command line interface (CLI) to check foreign key constraints in the PUDL database. - -Assets are executed once their upstream dependencies have been executed. However, this -order is non deterministic because they are executed in parallel. This means the order -that tables are loaded into ``pudl.sqlite`` will not satisfy foreign key constraints. - -Foreign key constraints on ``pudl.sqlite`` are disabled so dagster can load tables into -the database without a foreign key error being raised. However, foreign key constraints -can be evaluated after all of the data has been loaded into the database. To check the -constraints, run the ``pudl_check_fks`` cli command once the data has been loaded into -``pudl.sqlite``. -""" -import argparse +"""Check that foreign key constraints in the PUDL database are respected.""" +import pathlib import sys +import click from dagster import build_init_resource_context from dotenv import load_dotenv @@ -22,39 +12,43 @@ logger = pudl.logging_helpers.get_logger(__name__) -def parse_command_line(argv): - """Parse script command line arguments. See the -h option. - - Args: - argv (list): command line arguments including caller file name. - - Returns: - dict: A dictionary mapping command line arguments to their values. +@click.command( + context_settings={"help_option_names": ["-h", "--help"]}, +) +@click.option( + "--logfile", + help="If specified, write logs to this file.", + type=click.Path( + exists=False, + resolve_path=True, + path_type=pathlib.Path, + ), +) +@click.option( + "--loglevel", + default="INFO", + type=click.Choice( + ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], case_sensitive=False + ), +) +def pudl_check_fks(logfile: pathlib.Path, loglevel: str): + """Check that foreign key constraints in the PUDL database are respected. + + Dagster manages the dependencies between various assets in our ETL pipeline, + attempting to materialize tables only after their upstream dependencies have been + satisfied. However, this order is non deterministic because they are executed in + parallel, and doesn't necessarily correspond to the foreign-key constraints within + the database, so durint the ETL we disable foreign key constraints within + ``pudl.sqlite``. + + However, we still expect foreign key constraints to be satisfied once all of the + tables have been loaded, so we check that they are valid after the ETL has + completed. This script runs the same check. """ - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--logfile", - default=None, - help="If specified, write logs to this file.", - ) - parser.add_argument( - "--loglevel", - help="Set logging level (DEBUG, INFO, WARNING, ERROR, or CRITICAL).", - default="INFO", - ) - arguments = parser.parse_args(argv[1:]) - return arguments - - -def main(): - """Parse command line and check PUDL foreign key constraints.""" load_dotenv() - args = parse_command_line(sys.argv) # Display logged output from the PUDL package: - pudl.logging_helpers.configure_root_logger( - logfile=args.logfile, loglevel=args.loglevel - ) + pudl.logging_helpers.configure_root_logger(logfile=logfile, loglevel=loglevel) context = build_init_resource_context() io_manager = pudl_sqlite_io_manager(context) @@ -63,7 +57,8 @@ def main(): logger.info(f"Checking foreign key constraints in {database_path}") io_manager.check_foreign_keys() + return 0 if __name__ == "__main__": - sys.exit(main()) + sys.exit(pudl_check_fks()) diff --git a/src/pudl/cli/etl.py b/src/pudl/etl/cli.py similarity index 56% rename from src/pudl/cli/etl.py rename to src/pudl/etl/cli.py index 9972ff1ff3..cc226e9cc4 100644 --- a/src/pudl/cli/etl.py +++ b/src/pudl/etl/cli.py @@ -1,20 +1,9 @@ -"""A command line interface (CLI) to the main PUDL ETL functionality. - -This script cordinates the PUDL ETL process, based on parameters provided via a YAML -settings file. - -If the settings for a dataset has empty parameters (meaning there are no years or tables -included), no outputs will be generated. See :doc:`/dev/run_the_etl` for details. - -The output SQLite and Parquet files will be stored in ``PUDL_OUTPUT``. To -setup your default ``PUDL_INPUT`` and ``PUDL_OUTPUT`` directories see -``pudl_setup --help``. -""" - -import argparse +"""A command line interface (CLI) to the main PUDL ETL functionality.""" +import pathlib import sys from collections.abc import Callable +import click import fsspec from dagster import ( DagsterInstance, @@ -26,49 +15,13 @@ ) import pudl +from pudl.helpers import get_dagster_execution_config from pudl.settings import EpaCemsSettings, EtlSettings from pudl.workspace.setup import PudlPaths logger = pudl.logging_helpers.get_logger(__name__) -def parse_command_line(argv): - """Parse script command line arguments. See the -h option. - - Args: - argv (list): command line arguments including caller file name. - - Returns: - dict: A dictionary mapping command line arguments to their values. - """ - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - dest="settings_file", type=str, default="", help="path to ETL settings file." - ) - parser.add_argument( - "--logfile", - default=None, - help="If specified, write logs to this file.", - ) - parser.add_argument( - "--gcs-cache-path", - type=str, - help="Load datastore resources from Google Cloud Storage. Should be gs://bucket[/path_prefix]", - ) - parser.add_argument( - "--loglevel", - help="Set logging level (DEBUG, INFO, WARNING, ERROR, or CRITICAL).", - default="INFO", - ) - parser.add_argument( - "--max-concurrent", - help="Set the max number of processes dagster can launch. Defaults to use the number of CPUs on the machine.", - default=0, - ) - arguments = parser.parse_args(argv[1:]) - return arguments - - def pudl_etl_job_factory( logfile: str | None = None, loglevel: str = "INFO", process_epacems: bool = True ) -> Callable[[], JobDefinition]: @@ -105,16 +58,63 @@ def get_pudl_etl_job(): return get_pudl_etl_job -def main(): - """Parse command line and initialize PUDL DB.""" - args = parse_command_line(sys.argv) - +@click.command( + context_settings={"help_option_names": ["-h", "--help"]}, +) +@click.argument( + "etl_settings_yml", + type=click.Path( + exists=True, + dir_okay=False, + resolve_path=True, + path_type=pathlib.Path, + ), +) +@click.option( + "--dagster-workers", + default=0, + type=int, + help="Max number of processes Dagster can launch. Defaults to the number of CPUs.", +) +@click.option( + "--gcs-cache-path", + type=str, + help=( + "Load cached inputs from Google Cloud Storage if possible. This is usually " + "much faster and more reliable than downloading from Zenodo directly. The " + "path should be a URL of the form gs://bucket[/path_prefix]. Internally we use " + "gs://internal-zenodo-cache.catalyst.coop. A public cache is available at " + "gs://zenodo-cache.catalyst.coop but requires GCS authentication and a billing " + "project to pay data egress costs." + ), +) +@click.option( + "--logfile", + help="If specified, write logs to this file.", + type=click.Path( + exists=False, + resolve_path=True, + path_type=pathlib.Path, + ), +) +@click.option( + "--loglevel", + default="INFO", + type=click.Choice( + ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], case_sensitive=False + ), +) +def pudl_etl( + etl_settings_yml: pathlib.Path, + dagster_workers: int, + gcs_cache_path: str, + logfile: pathlib.Path, + loglevel: str, +): + """Use Dagster to run the PUDL ETL, as specified by the file ETL_SETTINGS_YML.""" # Display logged output from the PUDL package: - pudl.logging_helpers.configure_root_logger( - logfile=args.logfile, loglevel=args.loglevel - ) - - etl_settings = EtlSettings.from_yaml(args.settings_file) + pudl.logging_helpers.configure_root_logger(logfile=logfile, loglevel=loglevel) + etl_settings = EtlSettings.from_yaml(etl_settings_yml) dataset_settings_config = etl_settings.datasets.model_dump() process_epacems = True @@ -127,36 +127,30 @@ def main(): dataset_settings_config["epacems"] = EpaCemsSettings().model_dump() pudl_etl_reconstructable_job = build_reconstructable_job( - "pudl.cli.etl", + "pudl.etl.cli", "pudl_etl_job_factory", reconstructable_kwargs={ - "loglevel": args.loglevel, - "logfile": args.logfile, + "loglevel": loglevel, + "logfile": logfile, "process_epacems": process_epacems, }, ) - result = execute_job( - pudl_etl_reconstructable_job, - instance=DagsterInstance.get(), - run_config={ - "execution": { + run_config = { + "resources": { + "dataset_settings": {"config": dataset_settings_config}, + "datastore": { "config": { - "multiprocess": { - "max_concurrent": int(args.max_concurrent), - }, - } - }, - "resources": { - "dataset_settings": {"config": dataset_settings_config}, - "datastore": { - "config": { - "gcs_cache_path": args.gcs_cache_path - if args.gcs_cache_path - else "", - }, + "gcs_cache_path": gcs_cache_path, }, }, }, + } + run_config.update(get_dagster_execution_config(dagster_workers)) + + result = execute_job( + pudl_etl_reconstructable_job, + instance=DagsterInstance.get(), + run_config=run_config, ) # Workaround to reliably getting full stack trace @@ -177,4 +171,4 @@ def main(): if __name__ == "__main__": - sys.exit(main()) + sys.exit(pudl_etl()) diff --git a/src/pudl/extract/eia923.py b/src/pudl/extract/eia923.py index a55b7cceea..ea391bfc1c 100644 --- a/src/pudl/extract/eia923.py +++ b/src/pudl/extract/eia923.py @@ -25,11 +25,7 @@ def __init__(self, *args, **kwargs): ds (:class:datastore.Datastore): Initialized datastore. """ self.METADATA = excel.Metadata("eia923") - # There's an issue with the EIA-923 archive for 2018 which prevents this table - # from being extracted currently. When we update to a new DOI this problem will - # probably fix itself. See comments on this issue: - # https://github.com/catalyst-cooperative/pudl/issues/2448 - self.BLACKLISTED_PAGES = ["plant_frame", "emissions_control"] + self.BLACKLISTED_PAGES = ["plant_frame"] self.cols_added = [] super().__init__(*args, **kwargs) @@ -105,11 +101,7 @@ def get_dtypes(page, **partition): "raw_eia923__generation_fuel", "raw_eia923__generator", "raw_eia923__stocks", - # There's an issue with the EIA-923 archive for 2018 which prevents this table - # from being extracted currently. When we update to a new DOI this problem will - # probably fix itself. See comments on this issue: - # https://github.com/catalyst-cooperative/pudl/issues/2448 - # "raw_emissions_control_eia923", + "raw_eia923__emissions_control", ) @@ -140,9 +132,4 @@ def extract_eia923(context, eia923_raw_dfs): return ( Output(output_name=table_name, value=df) for table_name, df in eia923_raw_dfs.items() - # There's an issue with the EIA-923 archive for 2018 which prevents this table - # from being extracted currently. When we update to a new DOI this problem will - # probably fix itself. See comments on this issue: - # https://github.com/catalyst-cooperative/pudl/issues/2448 - if table_name != "raw_eia923__emissions_control" ) diff --git a/src/pudl/ferc_to_sqlite/cli.py b/src/pudl/ferc_to_sqlite/cli.py index 5754ee97df..f6b1d72f54 100644 --- a/src/pudl/ferc_to_sqlite/cli.py +++ b/src/pudl/ferc_to_sqlite/cli.py @@ -1,14 +1,10 @@ -"""A script for cloning the FERC Form 1 database into SQLite. - -This script generates a SQLite database that is a clone/mirror of the original -FERC Form1 database. We use this cloned database as the starting point for the -main PUDL ETL process. The underlying work in the script is being done in -:mod:`pudl.extract.ferc1`. -""" -import argparse +"""A script using Dagster to convert FERC data fom DBF and XBRL to SQLite databases.""" +import pathlib import sys +import time from collections.abc import Callable +import click from dagster import ( DagsterInstance, JobDefinition, @@ -18,68 +14,13 @@ import pudl from pudl import ferc_to_sqlite +from pudl.helpers import get_dagster_execution_config from pudl.settings import EtlSettings # Create a logger to output any messages we might have... logger = pudl.logging_helpers.get_logger(__name__) -def parse_command_line(argv): - """Parse command line arguments. See the -h option. - - Args: - argv (str): Command line arguments, including caller filename. - - Returns: - dict: Dictionary of command line arguments and their parsed values. - """ - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "settings_file", type=str, default="", help="path to YAML settings file." - ) - parser.add_argument( - "--logfile", - default=None, - type=str, - help="If specified, write logs to this file.", - ) - parser.add_argument( - "-c", - "--clobber", - action="store_true", - help="""Clobber existing sqlite database if it exists. If clobber is - not included but the sqlite databse already exists the build will - fail.""", - default=False, - ) - parser.add_argument( - "-b", - "--batch-size", - default=50, - type=int, - help="Specify number of XBRL instances to be processed at a time (defaults to 50)", - ) - parser.add_argument( - "-w", - "--workers", - default=None, - type=int, - help="Specify number of worker processes for parsing XBRL filings.", - ) - parser.add_argument( - "--gcs-cache-path", - type=str, - help="Load datastore resources from Google Cloud Storage. Should be gs://bucket[/path_prefix]", - ) - parser.add_argument( - "--loglevel", - help="Set logging level (DEBUG, INFO, WARNING, ERROR, or CRITICAL).", - default="INFO", - ) - arguments = parser.parse_args(argv[1:]) - return arguments - - def ferc_to_sqlite_job_factory( logfile: str | None = None, loglevel: str = "INFO", @@ -89,8 +30,8 @@ def ferc_to_sqlite_job_factory( """Factory for parameterizing a reconstructable ferc_to_sqlite job. Args: - loglevel: The log level for the job's execution. logfile: Path to a log file for the job's execution. + loglevel: The log level for the job's execution. enable_xbrl: if True, include XBRL data processing in the job. enable_dbf: if True, include DBF data processing in the job. @@ -122,54 +63,143 @@ def get_ferc_to_sqlite_job(): return get_ferc_to_sqlite_job -def main(): # noqa: C901 - """Clone the FERC Form 1 FoxPro database into SQLite.""" - args = parse_command_line(sys.argv) - +@click.command( + name="ferc_to_sqlite", + context_settings={"help_option_names": ["-h", "--help"]}, +) +@click.argument( + "etl_settings_yml", + type=click.Path( + exists=True, + dir_okay=False, + resolve_path=True, + path_type=pathlib.Path, + ), +) +@click.option( + "-b", + "--batch-size", + type=int, + default=50, + help="Number of XBRL instances to be processed at a time.", +) +@click.option( + "--clobber/--no-clobber", + type=bool, + default=False, + help=( + "Clobber existing FERC SQLite databases if they exist. If clobber is not " + "specified but the SQLite database already exists the run will fail." + ), +) +@click.option( + "-w", + "--workers", + type=int, + default=0, + help=( + "Number of worker processes to use when parsing XBRL filings. " + "Defaults to using the number of CPUs." + ), +) +@click.option( + "--dagster-workers", + type=int, + default=0, + help=( + "Set the max number of processes that dagster can launch. " + "If set to 1, in-process serial executor will be used. If set to 0, " + "dagster will saturate available CPUs (this is the default)." + ), +) +@click.option( + "--gcs-cache-path", + type=str, + help=( + "Load cached inputs from Google Cloud Storage if possible. This is usually " + "much faster and more reliable than downloading from Zenodo directly. The " + "path should be a URL of the form gs://bucket[/path_prefix]. Internally we use " + "gs://internal-zenodo-cache.catalyst.coop. A public cache is available at " + "gs://zenodo-cache.catalyst.coop but requires GCS authentication and a billing " + "project to pay data egress costs." + ), +) +@click.option( + "--logfile", + type=click.Path( + exists=False, + resolve_path=True, + path_type=pathlib.Path, + ), + help="If specified, write logs to this file.", +) +@click.option( + "--loglevel", + type=click.Choice( + ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], case_sensitive=False + ), + default="INFO", +) +def main( + etl_settings_yml: pathlib.Path, + batch_size: int, + workers: int, + dagster_workers: int, + clobber: bool, + gcs_cache_path: str, + logfile: pathlib.Path, + loglevel: str, +): + """Use Dagster to convert FERC data fom DBF and XBRL to SQLite databases. + + Reads settings specifying which forms and years to convert from ETL_SETTINGS_YML. + + Also produces JSON versions of XBRL taxonomies and datapackage descriptors which + annotate the XBRL derived SQLite databases. + """ # Display logged output from the PUDL package: - pudl.logging_helpers.configure_root_logger( - logfile=args.logfile, loglevel=args.loglevel - ) + pudl.logging_helpers.configure_root_logger(logfile=logfile, loglevel=loglevel) - etl_settings = EtlSettings.from_yaml(args.settings_file) + etl_settings = EtlSettings.from_yaml(etl_settings_yml) ferc_to_sqlite_reconstructable_job = build_reconstructable_job( "pudl.ferc_to_sqlite.cli", "ferc_to_sqlite_job_factory", - reconstructable_kwargs={"loglevel": args.loglevel, "logfile": args.logfile}, + reconstructable_kwargs={"loglevel": loglevel, "logfile": logfile}, ) - - result = execute_job( - ferc_to_sqlite_reconstructable_job, - instance=DagsterInstance.get(), - run_config={ - "resources": { - "ferc_to_sqlite_settings": { - "config": etl_settings.ferc_to_sqlite_settings.model_dump() - }, - "datastore": { - "config": { - "gcs_cache_path": args.gcs_cache_path - if args.gcs_cache_path - else "", - }, - }, + run_config = { + "resources": { + "ferc_to_sqlite_settings": { + "config": etl_settings.ferc_to_sqlite_settings.model_dump() }, - "ops": { - "xbrl2sqlite": { - "config": { - "workers": args.workers, - "batch_size": args.batch_size, - "clobber": args.clobber, - }, - }, - "dbf2sqlite": { - "config": {"clobber": args.clobber}, + "datastore": { + "config": {"gcs_cache_path": gcs_cache_path}, + }, + }, + "ops": { + "xbrl2sqlite": { + "config": { + "workers": workers, + "batch_size": batch_size, + "clobber": clobber, }, }, + "dbf2sqlite": { + "config": {"clobber": clobber}, + }, }, + } + run_config.update(get_dagster_execution_config(dagster_workers)) + + start_time = time.time() + result = execute_job( + ferc_to_sqlite_reconstructable_job, + instance=DagsterInstance.get(), + run_config=run_config, raise_on_error=True, ) + end_time = time.time() + logger.info(f"FERC to SQLite job completed in {end_time - start_time} seconds.") # Workaround to reliably getting full stack trace if not result.success: diff --git a/src/pudl/helpers.py b/src/pudl/helpers.py index e9f576db5a..4e1fb73f5d 100644 --- a/src/pudl/helpers.py +++ b/src/pudl/helpers.py @@ -1796,6 +1796,41 @@ def scale_by_ownership( return gens +def get_dagster_execution_config(num_workers: int = 0): + """Get the dagster execution config for a given number of workers. + + If num_workers is 0, then the dagster execution config will not include + any limits. With num_workesr set to 1, we will use in-process serial + executor, otherwise multi-process executor with maximum of num_workers + will be used. + + Args: + num_workers: The number of workers to use for the dagster execution config. + If 0, then the dagster execution config will not include a multiprocess + executor. + + Returns: + A dagster execution config. + """ + if not num_workers: + return {} + if num_workers == 1: + return { + "execution": { + "config": { + "in_process": {}, + }, + }, + } + return { + "execution": { + "config": { + "multiprocess": {"max_concurrent": num_workers}, + }, + }, + } + + def assert_cols_areclose( df: pd.DataFrame, a_cols: list[str], diff --git a/src/pudl/metadata/resources/ferc1_eia_record_linkage.py b/src/pudl/metadata/resources/ferc1_eia_record_linkage.py index e1a5f89032..c60ecedf3f 100644 --- a/src/pudl/metadata/resources/ferc1_eia_record_linkage.py +++ b/src/pudl/metadata/resources/ferc1_eia_record_linkage.py @@ -23,8 +23,8 @@ Because generators are often owned by multiple utilities, another dimension of this plant part table involves generating two records for each owner: one for the portion of the plant part they own and one for the plant part as a whole. The -portion records are labeled in the "ownership_record_type" column as "owned" -and the total records are labeled as "total". +portion records are labeled in the ``ownership_record_type`` column as ``owned`` +and the total records are labeled as ``total``. This table includes A LOT of duplicative information about EIA plants. It is primarily meant for use as an input into the record linkage between FERC1 plants and EIA.""", diff --git a/src/pudl/output/ferc1.py b/src/pudl/output/ferc1.py index 5f95bb600b..3594701306 100644 --- a/src/pudl/output/ferc1.py +++ b/src/pudl/output/ferc1.py @@ -70,8 +70,8 @@ ], group_metric_tolerances=GroupMetricTolerances( ungrouped=MetricTolerances( - error_frequency=0.014, - relative_error_magnitude=0.04, + error_frequency=0.011, + relative_error_magnitude=0.004, null_calculated_value_frequency=1.0, ), report_year=MetricTolerances( @@ -1074,6 +1074,27 @@ class NodeId(NamedTuple): plant_function: str | pandas_NAType +class OffByFactoid(NamedTuple): + """A calculated factoid which is off by one other factoid. + + A factoid where a sizeable majority of utilities are using a non-standard and + non-reported calculation to generate it. These calculated factoids are either + missing one factoid, or include an additional factoid not included in the FERC + metadata. Thus, the calculations are 'off by' this factoid. + """ + + table_name: str + xbrl_factoid: str + utility_type: str | pandas_NAType + plant_status: str | pandas_NAType + plant_function: str | pandas_NAType + table_name_off_by: str + xbrl_factoid_off_by: str + utility_type_off_by: str | pandas_NAType + plant_status_off_by: str | pandas_NAType + plant_function_off_by: str | pandas_NAType + + @asset def _out_ferc1__explosion_tags(table_dimensions_ferc1) -> pd.DataFrame: """Grab the stored tables of tags and add inferred dimension.""" @@ -1159,9 +1180,10 @@ def _aggregatable_dimension_tags( def exploded_table_asset_factory( root_table: str, - table_names_to_explode: list[str], + table_names: list[str], seed_nodes: list[NodeId], group_metric_checks: GroupMetricChecks, + off_by_facts: list[OffByFactoid], io_manager_key: str | None = None, ) -> AssetsDefinition: """Create an exploded table based on a set of related input tables.""" @@ -1172,7 +1194,7 @@ def exploded_table_asset_factory( ), "_out_ferc1__explosion_tags": AssetIn("_out_ferc1__explosion_tags"), } - ins |= {table_name: AssetIn(table_name) for table_name in table_names_to_explode} + ins |= {table_name: AssetIn(table_name) for table_name in table_names} @asset(name=f"exploded_{root_table}", ins=ins, io_manager_key=io_manager_key) def exploded_tables_asset( @@ -1189,6 +1211,7 @@ def exploded_tables_asset( "metadata_xbrl_ferc1", "calculation_components_xbrl_ferc1", "_out_ferc1__explosion_tags", + "off_by_facts", ] } return Exploder( @@ -1199,6 +1222,7 @@ def exploded_tables_asset( seed_nodes=seed_nodes, tags=tags, group_metric_checks=group_metric_checks, + off_by_facts=off_by_facts, ).boom(tables_to_explode=tables_to_explode) return exploded_tables_asset @@ -1214,7 +1238,7 @@ def create_exploded_table_assets() -> list[AssetsDefinition]: explosion_args = [ { "root_table": "income_statement_ferc1", - "table_names_to_explode": [ + "table_names": [ "income_statement_ferc1", "depreciation_amortization_summary_ferc1", "electric_operating_expenses_ferc1", @@ -1232,10 +1256,11 @@ def create_exploded_table_assets() -> list[AssetsDefinition]: plant_function=pd.NA, ), ], + "off_by_facts": [], }, { "root_table": "balance_sheet_assets_ferc1", - "table_names_to_explode": [ + "table_names": [ "balance_sheet_assets_ferc1", "utility_plant_summary_ferc1", "plant_in_service_ferc1", @@ -1253,10 +1278,48 @@ def create_exploded_table_assets() -> list[AssetsDefinition]: plant_function=pd.NA, ) ], + "off_by_facts": [ + OffByFactoid( + "utility_plant_summary_ferc1", + "utility_plant_in_service_classified_and_property_under_capital_leases", + "electric", + pd.NA, + pd.NA, + "utility_plant_summary_ferc1", + "utility_plant_in_service_completed_construction_not_classified", + "electric", + pd.NA, + pd.NA, + ), + OffByFactoid( + "utility_plant_summary_ferc1", + "utility_plant_in_service_classified_and_property_under_capital_leases", + "electric", + pd.NA, + pd.NA, + "utility_plant_summary_ferc1", + "utility_plant_in_service_property_under_capital_leases", + "electric", + pd.NA, + pd.NA, + ), + OffByFactoid( + "utility_plant_summary_ferc1", + "depreciation_utility_plant_in_service", + "electric", + pd.NA, + pd.NA, + "utility_plant_summary_ferc1", + "amortization_of_other_utility_plant_utility_plant_in_service", + "electric", + pd.NA, + pd.NA, + ), + ], }, { "root_table": "balance_sheet_liabilities_ferc1", - "table_names_to_explode": [ + "table_names": [ "balance_sheet_liabilities_ferc1", "retained_earnings_ferc1", ], @@ -1272,6 +1335,7 @@ def create_exploded_table_assets() -> list[AssetsDefinition]: plant_function=pd.NA, ) ], + "off_by_facts": [], }, ] return [exploded_table_asset_factory(**kwargs) for kwargs in explosion_args] @@ -1292,6 +1356,7 @@ def __init__( seed_nodes: list[NodeId], tags: pd.DataFrame = pd.DataFrame(), group_metric_checks: GroupMetricChecks = GroupMetricChecks(), + off_by_facts: list[OffByFactoid] = None, ): """Instantiate an Exploder class. @@ -1310,6 +1375,7 @@ def __init__( self.calculation_components_xbrl_ferc1 = calculation_components_xbrl_ferc1 self.seed_nodes = seed_nodes self.tags = tags + self.off_by_facts = off_by_facts @cached_property def exploded_calcs(self: Self): @@ -1360,7 +1426,7 @@ def exploded_calcs(self: Self): .sort_index() .reset_index() ) - + calc_explode = self.add_sizable_minority_corrections_to_calcs(calc_explode) ############################################################################## # Everything below here is error checking / debugging / temporary fixes dupes = calc_explode.duplicated(subset=parent_cols + calc_cols, keep=False) @@ -1394,6 +1460,39 @@ def exploded_calcs(self: Self): return calc_explode + def add_sizable_minority_corrections_to_calcs( + self: Self, exploded_calcs: pd.DataFrame + ) -> pd.DataFrame: + """Add correction calculation records for the sizable fuck up utilities.""" + if not self.off_by_facts: + return exploded_calcs + facts_to_fix = ( + pd.DataFrame(self.off_by_facts) + .rename(columns={col: f"{col}_parent" for col in NodeId._fields}) + .assign( + xbrl_factoid=( + lambda x: x.xbrl_factoid_parent + + "_off_by_" + + x.xbrl_factoid_off_by + + "_correction" + ), + weight=1, + is_total_to_subdimensions_calc=False, + # theses fixes rn are only from inter-table calcs. + # if they weren't we'd need to check within the group of + # the parent fact like in process_xbrl_metadata_calculations + is_within_table_calc=False, + ) + .drop(columns=["xbrl_factoid_off_by"]) + ) + facts_to_fix.columns = facts_to_fix.columns.str.removesuffix("_off_by") + return pd.concat( + [ + exploded_calcs, + facts_to_fix[exploded_calcs.columns].astype(exploded_calcs.dtypes), + ] + ) + @cached_property def exploded_meta(self: Self) -> pd.DataFrame: """Combine a set of interrelated table's metatada for use in :class:`Exploder`. @@ -1495,7 +1594,7 @@ def calculation_forest(self: Self) -> "XbrlCalculationForestFerc1": ) @cached_property - def other_dimensions(self: Self) -> list[str]: + def dimensions(self: Self) -> list[str]: """Get all of the column names for the other dimensions.""" return pudl.transform.ferc1.other_dimensions(table_names=self.table_names) @@ -1542,6 +1641,11 @@ def value_col(self: Self) -> str: value_col = list(set(value_cols))[0] return value_col + @property + def calc_idx(self: Self) -> list[str]: + """Primary key columns for calculations in this explosion.""" + return [col for col in list(NodeId._fields) if col in self.exploded_pks] + def boom(self: Self, tables_to_explode: dict[str, pd.DataFrame]) -> pd.DataFrame: """Explode a set of nested tables. @@ -1558,14 +1662,14 @@ def boom(self: Self, tables_to_explode: dict[str, pd.DataFrame]) -> pd.DataFrame """ exploded = ( self.initial_explosion_concatenation(tables_to_explode) + .pipe(self.calculate_intertable_non_total_calculations) + .pipe(self.add_sizable_minority_corrections) .pipe(self.reconcile_intertable_calculations) .pipe(self.calculation_forest.leafy_data, value_col=self.value_col) ) # Identify which columns should be kept in the output... # TODO: Define schema for the tables explicitly. - cols_to_keep = list( - set(self.exploded_pks + self.other_dimensions + [self.value_col]) - ) + cols_to_keep = list(set(self.exploded_pks + self.dimensions + [self.value_col])) if ("utility_type" in cols_to_keep) and ( "utility_type_other" in exploded.columns ): @@ -1632,6 +1736,122 @@ def initial_explosion_concatenation( ) return exploded + def calculate_intertable_non_total_calculations( + self, exploded: pd.DataFrame + ) -> pd.DataFrame: + """Calculate the inter-table non-total calculable xbrl_factoids.""" + # add the abs_diff column for the calculated fields + calculated_df = pudl.transform.ferc1.calculate_values_from_components( + calculation_components=self.exploded_calcs[ + ~self.exploded_calcs.is_within_table_calc + & ~self.exploded_calcs.is_total_to_subdimensions_calc + ], + data=exploded, + calc_idx=self.calc_idx, + value_col=self.value_col, + ) + return calculated_df + + def add_sizable_minority_corrections( + self: Self, calculated_df: pd.DataFrame + ) -> pd.DataFrame: + """Identify and fix the utilities that report calcs off-by one other fact. + + We noticed that there are a sizable minority of utilities that report some + calculated values with a different set of child subcomponents. Our tooling + for these calculation expects all utilities to report in the same manner. + So we have identified the handful of worst calculable ``xbrl_factiod`` + offenders :attr:`self.off_by_facts`. This method identifies data corrections + for those :attr:`self.off_by_facts` and adds them into the exploded data table. + + The data corrections are identified by calculating the absolute difference + between the reported value and calculable value from the standard set of + subcomponents (via :func:`pudl.transform.ferc1.calculate_values_from_components`) + and finding the child factiods that have the same value as the absolute difference. + This indicates that the calculable parent factiod is off by that cooresponding + child fact. + + Relatedly, :meth:`add_sizable_minority_corrections_to_calcs` adds these + :attr:`self.off_by_facts` to :attr:`self.exploded_calcs`. + """ + if not self.off_by_facts: + return calculated_df + + off_by = pd.DataFrame(self.off_by_facts) + not_close = ( + calculated_df[~np.isclose(calculated_df.abs_diff, 0)] + .set_index(list(NodeId._fields)) + # grab the parent side of the off_by facts + .loc[off_by.set_index(list(NodeId._fields)).index.unique()] + .reset_index() + ) + off_by_fact = ( + calculated_df[calculated_df.row_type_xbrl != "correction"] + .set_index(list(NodeId._fields)) + # grab the child side of the off_by facts + .loc[ + off_by.set_index( + [f"{col}_off_by" for col in list(NodeId._fields)] + ).index.unique() + ] + .reset_index() + ) + # Idenfity the calculations with one missing other fact + # when the diff is the same as the value of another fact, then that + # calculated value could have been perfect with the addition of the + # missing facoid. + data_corrections = ( + pd.merge( + left=not_close, + right=off_by_fact, + left_on=["report_year", "utility_id_ferc1", "abs_diff"], + right_on=["report_year", "utility_id_ferc1", self.value_col], + how="inner", + suffixes=("", "_off_by"), + ) + # use a dict/kwarg for assign so we can dynamically set the name of value_col + # the correction value is the diff - not the abs_diff from not_close or value_col + # from off_by_fact bc often the utility included a fact so this diff is negative + .assign( + **{ + "xbrl_factoid": ( + lambda x: x.xbrl_factoid + + "_off_by_" + + x.xbrl_factoid_off_by + + "_correction" + ), + self.value_col: lambda x: x["diff"], + "row_type_xbrl": "correction", + } + )[ + # drop all the _off_by and calc cols + list(NodeId._fields) + + ["report_year", "utility_id_ferc1", self.value_col, "row_type_xbrl"] + ] + ) + bad_utils = data_corrections.utility_id_ferc1.unique() + logger.info( + f"Adding {len(data_corrections)} from {len(bad_utils)} utilities that report differently." + ) + dtype_calced = { + col: dtype + for (col, dtype) in calculated_df.dtypes.items() + if col in data_corrections.columns + } + corrected = pd.concat( + [calculated_df, data_corrections.astype(dtype_calced)] + ).set_index(self.calc_idx) + # now we are going to re-calculate just the fixed facts + fixed_facts = corrected.loc[off_by.set_index(self.calc_idx).index.unique()] + unfixed_facts = corrected.loc[ + corrected.index.difference(off_by.set_index(self.calc_idx).index.unique()) + ].reset_index() + fixed_facts = self.calculate_intertable_non_total_calculations( + exploded=fixed_facts.drop(columns=["calculated_value"]).reset_index() + ) + + return pd.concat([unfixed_facts, fixed_facts.astype(unfixed_facts.dtypes)]) + def reconcile_intertable_calculations( self: Self, exploded: pd.DataFrame ) -> pd.DataFrame: @@ -1655,18 +1875,10 @@ def reconcile_intertable_calculations( f"{self.root_table}: Reconcile inter-table calculations: " f"{list(calculations_intertable.xbrl_factoid.unique())}." ) - calc_idx = [col for col in list(NodeId._fields) if col in self.exploded_pks] logger.info("Checking inter-table, non-total to subtotal calcs.") - calculated_df = pudl.transform.ferc1.calculate_values_from_components( - calculation_components=calculations_intertable[ - ~calculations_intertable.is_total_to_subdimensions_calc - ], - data=exploded, - calc_idx=calc_idx, - value_col=self.value_col, - ) + # we've added calculated fields via calculate_intertable_non_total_calculations calculated_df = pudl.transform.ferc1.check_calculation_metrics( - calculated_df=calculated_df, group_metric_checks=self.group_metric_checks + calculated_df=exploded, group_metric_checks=self.group_metric_checks ) calculated_df = pudl.transform.ferc1.add_corrections( calculated_df=calculated_df, @@ -1679,8 +1891,8 @@ def reconcile_intertable_calculations( calculation_components=calculations_intertable[ calculations_intertable.is_total_to_subdimensions_calc ], - data=exploded, - calc_idx=calc_idx, + data=exploded.drop(columns="calculated_value"), + calc_idx=self.calc_idx, value_col=self.value_col, ) subtotal_calcs = pudl.transform.ferc1.check_calculation_metrics( diff --git a/src/pudl/package_data/eia923/file_map.csv b/src/pudl/package_data/eia923/file_map.csv index ecd1d35d59..89ff347691 100644 --- a/src/pudl/package_data/eia923/file_map.csv +++ b/src/pudl/package_data/eia923/file_map.csv @@ -11,4 +11,4 @@ plant_frame,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,EIA923_Schedules_2_3_4_5_2011_Final_Re puerto_rico,-1,-1,-1,-1,-1,-1,-1,eia923December2008.xls,EIA923 SCHEDULES 2_3_4_5 M Final 2009 REVISED 05252011.XLS,EIA923 SCHEDULES 2_3_4_5 Final 2010.xls,EIA923_Schedules_2_3_4_5_2011_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2012_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_2013_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2014_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2015_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2016_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2017_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2018_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2019_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2020_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2021_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2022_Final.xlsx,EIA923_Schedules_2_3_4_5_M_08_2023_19OCT2023.xlsx stocks,f906920y2001.xls,f906920y2002.xls,f906920_2003.xls,f906920_2004.xls,f906920_2005.xls,f906920_2006.xls,f906920_2007.xls,eia923December2008.xls,EIA923 SCHEDULES 2_3_4_5 M Final 2009 REVISED 05252011.XLS,EIA923 SCHEDULES 2_3_4_5 Final 2010.xls,EIA923_Schedules_2_3_4_5_2011_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2012_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_2013_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2014_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2015_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2016_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2017_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2018_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2019_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2020_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2021_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2022_Final.xlsx,EIA923_Schedules_2_3_4_5_M_08_2023_19OCT2023.xlsx plant_frame_puerto_rico,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,EIA923_Schedules_2_3_4_5_M_12_2019_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2020_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2021_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2022_Final.xlsx,EIA923_Schedules_2_3_4_5_M_08_2023_19OCT2023.xlsx -emissions_control,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,EIA923_Schedule_8_Annual_Environmental_Information_2012_Final_Revision.xlsx,EIA923_Schedule_8_PartsA-D_EnvData_2013_Final_Revision.xlsx,EIA923_Schedule_8_Annual_Environmental_Information_2014_Final_Revision.xlsx,EIA923_Schedule_8_Annual_Environmental_Information_2015_Final_Revision.xlsx,EIA923_Schedule_8_Annual_Environmental_Information_2016_Final_Revision.xlsx,EIA923_Schedule_8_Annual_Envir_Infor_2017_Final.xlsx,EIA923_Schedule_8_Annual_Environmental_Information_2018_Final.xlsx,EIA923_Schedule_8_Annual_Environmental_Information_2019_Final_Revision.xlsx,EIA923_Schedule_8_Annual_Environmental_Information_2020_Final_Revision.xlsx,EIA923_Schedule_8_Annual_Environmental_Information_2021_Final_Revision.xlsx,EIA923_Schedule_8_Annual_Environmental_Information_2022_Early_Release.xlsx,-1 +emissions_control,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,EIA923_Schedule_8_Annual_Environmental_Information_2012_Final_Revision.xlsx,EIA923_Schedule_8_PartsA-D_EnvData_2013_Final_Revision.xlsx,EIA923_Schedule_8_Annual_Environmental_Information_2014_Final_Revision.xlsx,EIA923_Schedule_8_Annual_Environmental_Information_2015_Final_Revision.xlsx,EIA923_Schedule_8_Annual_Environmental_Information_2016_Final_Revision.xlsx,EIA923_Schedule_8_Annual_Envir_Infor_2017_Final.xlsx,EIA923_Schedule_8_Annual_Environmental_Information_2018_Final.xlsx,EIA923_Schedule_8_Annual_Environmental_Information_2019_Final_Revision.xlsx,EIA923_Schedule_8_Annual_Environmental_Information_2020_Final_Revision.xlsx,EIA923_Schedule_8_Annual_Environmental_Information_2021_Final_Revision.xlsx,EIA923_Schedule_8_Annual_Environmental_Information_2022_Final.xlsx,-1 diff --git a/src/pudl/settings.py b/src/pudl/settings.py index 82c399a6df..6c5205e03b 100644 --- a/src/pudl/settings.py +++ b/src/pudl/settings.py @@ -18,7 +18,6 @@ from pydantic_settings import BaseSettings import pudl -import pudl.workspace.setup from pudl.metadata.classes import DataSource from pudl.workspace.datastore import Datastore, ZenodoDoi diff --git a/src/pudl/transform/ferc1.py b/src/pudl/transform/ferc1.py index b8b9843974..92d716ff69 100644 --- a/src/pudl/transform/ferc1.py +++ b/src/pudl/transform/ferc1.py @@ -1151,7 +1151,8 @@ def calculate_values_from_components( {value_col: "float64", "calculated_value": "float64"} ) calculated_df = calculated_df.assign( - abs_diff=lambda x: abs(x[value_col] - x.calculated_value), + diff=lambda x: x[value_col] - x.calculated_value, + abs_diff=lambda x: abs(x["diff"]), rel_diff=lambda x: np.where( (x[value_col] != 0.0), abs(x.abs_diff / x[value_col]), diff --git a/src/pudl/workspace/__init__.py b/src/pudl/workspace/__init__.py index 778e7224bc..f709ba9a97 100644 --- a/src/pudl/workspace/__init__.py +++ b/src/pudl/workspace/__init__.py @@ -7,4 +7,4 @@ These tools are available both as a library module, and via a command line interface installed as an entrypoint script called ``pudl_datastore``. """ -from . import datastore, resource_cache, setup, setup_cli +from . import datastore, resource_cache, setup diff --git a/src/pudl/workspace/datastore.py b/src/pudl/workspace/datastore.py index e1043945a0..809e490e7a 100644 --- a/src/pudl/workspace/datastore.py +++ b/src/pudl/workspace/datastore.py @@ -1,8 +1,8 @@ """Datastore manages file retrieval for PUDL datasets.""" -import argparse import hashlib import io import json +import pathlib import re import sys import zipfile @@ -12,6 +12,7 @@ from typing import Annotated, Any, Self from urllib.parse import ParseResult, urlparse +import click import datapackage import requests from google.auth.exceptions import DefaultCredentialsError @@ -27,7 +28,6 @@ logger = pudl.logging_helpers.get_logger(__name__) -PUDL_YML = Path.home() / ".pudl.yml" ZenodoDoi = Annotated[ str, StringConstraints( @@ -184,6 +184,7 @@ class ZenodoDoiSettings(BaseSettings): ferc60: ZenodoDoi = "10.5281/zenodo.8326695" ferc714: ZenodoDoi = "10.5281/zenodo.8326694" phmsagas: ZenodoDoi = "10.5281/zenodo.8346646" + model_config = SettingsConfigDict(env_prefix="pudl_zenodo_doi_", env_file=".env") @@ -411,114 +412,26 @@ def get_zipfile_file_names(self, zip_file: zipfile.ZipFile): return zipfile.ZipFile.namelist(zip_file) -class ParseKeyValues(argparse.Action): - """Transforms k1=v1,k2=v2,... - - into dict(k1=v1, k2=v2, ...). - """ - - def __call__(self, parser, namespace, values, option_string=None): - """Parses the argument value into dict.""" - d = getattr(namespace, self.dest, {}) - if isinstance(values, str): - values = [values] - for val in values: - for kv in val.split(","): - k, v = kv.split("=") - d[k] = v - setattr(namespace, self.dest, d) - - -def parse_command_line(): - """Collect the command line arguments.""" - known_datasets = "\n".join( - [f" - {x}" for x in ZenodoFetcher().get_known_datasets()] - ) - - dataset_msg = f""" -Available Datasets: -{known_datasets}""" - - parser = argparse.ArgumentParser( - description="Download and cache ETL source data from Zenodo.", - epilog=dataset_msg, - formatter_class=argparse.RawTextHelpFormatter, - ) - - parser.add_argument( - "--dataset", - help="Download the specified dataset only. See below for available options. " - "The default is to download all datasets, which may take hours depending on " - "network speed.", - ) - parser.add_argument( - "--validate", - help="Validate locally cached datapackages, but don't download anything.", - action="store_true", - default=False, - ) - parser.add_argument( - "--loglevel", - help="Set logging level (DEBUG, INFO, WARNING, ERROR, or CRITICAL).", - default="INFO", - ) - parser.add_argument( - "--logfile", - default=None, - type=str, - help="If specified, write logs to this file.", - ) - parser.add_argument( - "--quiet", - help="Do not send logging messages to stdout.", - action="store_true", - default=False, - ) - parser.add_argument( - "--gcs-cache-path", - type=str, - help="""Load datastore resources from Google Cloud Storage. Should be gs://bucket[/path_prefix]. -The main zenodo cache bucket is gs://zenodo-cache.catalyst.coop. -If specified without --bypass-local-cache, the local cache will be populated from the GCS cache. -If specified with --bypass-local-cache, the GCS cache will be populated by Zenodo.""", - ) - parser.add_argument( - "--bypass-local-cache", - action="store_true", - default=False, - help="""If enabled, the local file cache for datastore will not be used.""", - ) - parser.add_argument( - "--partition", - default={}, - action=ParseKeyValues, - metavar="KEY=VALUE,...", - help="Only retrieve resources matching these conditions.", - ) - parser.add_argument( - "--list-partitions", - action="store_true", - default=False, - help="List available partition keys and values for each dataset.", - ) - - return parser.parse_args() - - def print_partitions(dstore: Datastore, datasets: list[str]) -> None: """Prints known partition keys and its values for each of the datasets.""" for single_ds in datasets: - parts = dstore.get_datapackage_descriptor(single_ds).get_partitions() + partitions = dstore.get_datapackage_descriptor(single_ds).get_partitions() - print(f"\nPartitions for {single_ds} ({dstore.get_doi(single_ds)}):") - for pkey in sorted(parts): - print(f' {pkey}: {", ".join(str(x) for x in sorted(parts[pkey]))}') - if not parts: + print(f"\nPartitions for {single_ds} ({ZenodoFetcher().get_doi(single_ds)}):") + for partition_key in sorted(partitions): + # try-except required because ferc2 has parts with heterogenous types that + # therefore can't be sorted: [1, 2, None] + try: + parts = sorted(partitions[partition_key]) + except TypeError: + parts = partitions[partition_key] + print(f' {partition_key}: {", ".join(str(x) for x in parts)}') + if not partitions: print(" -- no known partitions --") def validate_cache( - dstore: Datastore, datasets: list[str], args: argparse.Namespace + dstore: Datastore, datasets: list[str], partition: dict[str, str] ) -> None: """Validate elements in the datastore cache. @@ -529,7 +442,7 @@ def validate_cache( num_invalid = 0 descriptor = dstore.get_datapackage_descriptor(single_ds) for res, content in dstore.get_resources( - single_ds, cached_only=True, **args.partition + single_ds, cached_only=True, **partition ): try: num_total += 1 @@ -546,49 +459,185 @@ def validate_cache( def fetch_resources( - dstore: Datastore, datasets: list[str], args: argparse.Namespace + dstore: Datastore, + datasets: list[str], + partition: dict[str, int | str], + gcs_cache_path: str, + bypass_local_cache: bool, ) -> None: """Retrieve all matching resources and store them in the cache.""" for single_ds in datasets: for res, contents in dstore.get_resources( - single_ds, skip_optimally_cached=True, **args.partition + single_ds, skip_optimally_cached=True, **partition ): logger.info(f"Retrieved {res}.") # If the gcs_cache_path is specified and we don't want # to bypass the local cache, populate the local cache. - if args.gcs_cache_path and not args.bypass_local_cache: + if gcs_cache_path and not bypass_local_cache: dstore._cache.add(res, contents) -def main(): - """Cache datasets.""" - args = parse_command_line() +def _parse_key_values( + ctx: click.core.Context, + param: click.Option, + values: str, +) -> dict[str, str]: + """Parse key-value pairs into a Python dictionary. - pudl.logging_helpers.configure_root_logger( - logfile=args.logfile, loglevel=args.loglevel - ) + Transforms a command line argument of the form: k1=v1,k2=v2,k3=v3... + into: {k1:v1, k2:v2, k3:v3, ...} + """ + out_dict = {} + for val in values: + for key_value in val.split(","): + key, value = key_value.split("=") + out_dict[key] = value + return out_dict + + +@click.command( + context_settings={"help_option_names": ["-h", "--help"]}, +) +@click.option( + "--dataset", + "-d", + type=click.Choice(ZenodoFetcher().get_known_datasets()), + default=list(ZenodoFetcher().get_known_datasets()), + multiple=True, + help=( + "Specifies what dataset to work with. The default is to download all datasets. " + "Note that downloading all datasets may take hours depending on network speed. " + "This option may be applied multiple times to specify multiple datasets." + ), +) +@click.option( + "--validate", + is_flag=True, + default=False, + help="Validate the contents of locally cached data, but don't download anything.", +) +@click.option( + "--list-partitions", + help=( + "List the available partition keys and values for each dataset specified " + "using the --dataset argument, or all datasets if --dataset is not used." + ), + is_flag=True, + default=False, +) +@click.option( + "--partition", + "-p", + multiple=True, + help=( + "Only operate on dataset partitions matching these conditions. The argument " + "should have the form: key1=val1,key2=val2,... Conditions are combined with " + "a boolean AND, functionally meaning each key can only appear once. " + "If a key is repeated, only the last value is used. " + "So state=ca,year=2022 will retrieve all California data for 2022, and " + "state=ca,year=2021,year=2022 will also retrieve California data for 2022, " + "while state=ca by itself will retrieve all years of California data." + ), + callback=_parse_key_values, +) +@click.option( + "--bypass-local-cache", + is_flag=True, + default=False, + help=( + "If enabled, locally cached data will not be used. Instead, a new copy will be " + "downloaded from Zenodo or the GCS cache if specified." + ), +) +@click.option( + "--gcs-cache-path", + type=str, + help=( + "Load cached inputs from Google Cloud Storage if possible. This is usually " + "much faster and more reliable than downloading from Zenodo directly. The " + "path should be a URL of the form gs://bucket[/path_prefix]. Internally we use " + "gs://internal-zenodo-cache.catalyst.coop. A public cache is available at " + "gs://zenodo-cache.catalyst.coop but requires GCS authentication and a billing " + "project to pay data egress costs." + ), +) +@click.option( + "--logfile", + help="If specified, write logs to this file.", + type=click.Path( + exists=False, + resolve_path=True, + path_type=pathlib.Path, + ), +) +@click.option( + "--loglevel", + default="INFO", + type=click.Choice( + ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], case_sensitive=False + ), +) +def pudl_datastore( + dataset: list[str], + validate: bool, + list_partitions: bool, + partition: dict[str, int | str], + gcs_cache_path: str, + bypass_local_cache: bool, + logfile: pathlib.Path, + loglevel: str, +): + """Manage the raw data inputs to the PUDL data processing pipeline. + + Download all the raw FERC Form 2 data: + + pudl_datastore --dataset ferc2 + + Download the raw FERC Form 2 data only for 2021 + + pudl_datastore --dataset ferc2 --partition year=2021 + + Re-download the raw FERC Form 2 data for 2021 even if you already have it: + + pudl_datastore --dataset ferc2 --partition year=2021 --bypass-local-cache + + Validate all California EPA CEMS data in the local datastore: + + pudl_datastore --dataset epacems --validate --partition state=ca + + List the available partitions in the EIA-860 and EIA-923 datasets: + + pudl_datastore --dataset eia860 --dataset eia923 --list-partitions + """ + pudl.logging_helpers.configure_root_logger(logfile=logfile, loglevel=loglevel) cache_path = None - if not args.bypass_local_cache: + if not bypass_local_cache: cache_path = PudlPaths().input_dir dstore = Datastore( - gcs_cache_path=args.gcs_cache_path, + gcs_cache_path=gcs_cache_path, local_cache_path=cache_path, ) - datasets = [args.dataset] if args.dataset else dstore.get_known_datasets() - - if args.partition: - logger.info(f"Only retrieving resources for partition: {args.partition}") + if partition: + logger.info(f"Only considering resource partitions: {partition}") - if args.list_partitions: - print_partitions(dstore, datasets) - elif args.validate: - validate_cache(dstore, datasets, args) + if list_partitions: + print_partitions(dstore, dataset) + elif validate: + validate_cache(dstore, dataset, partition) else: - fetch_resources(dstore, datasets, args) + fetch_resources( + dstore=dstore, + datasets=dataset, + partition=partition, + gcs_cache_path=gcs_cache_path, + bypass_local_cache=bypass_local_cache, + ) + + return 0 if __name__ == "__main__": - sys.exit(main()) + sys.exit(pudl_datastore()) diff --git a/src/pudl/workspace/setup.py b/src/pudl/workspace/setup.py index 4337eb69e0..d0ca40c14e 100644 --- a/src/pudl/workspace/setup.py +++ b/src/pudl/workspace/setup.py @@ -1,11 +1,9 @@ """Tools for setting up and managing PUDL workspaces.""" -import importlib.resources import os -import pathlib -import shutil from pathlib import Path +from typing import Self -from pydantic import DirectoryPath, NewPath +from pydantic import DirectoryPath, NewPath, model_validator from pydantic_settings import BaseSettings, SettingsConfigDict import pudl.logging_helpers @@ -26,6 +24,12 @@ class PudlPaths(BaseSettings): pudl_output: PotentialDirectoryPath model_config = SettingsConfigDict(env_file=".env", extra="ignore") + @model_validator(mode="after") + def create_directories(self: Self): + """Create PUDL input and output directories if they don't already exist.""" + self.input_dir.mkdir(parents=True, exist_ok=True) + self.output_dir.mkdir(parents=True, exist_ok=True) + @property def input_dir(self) -> Path: """Path to PUDL input directory.""" @@ -85,75 +89,3 @@ def set_path_overrides( os.environ["PUDL_INPUT"] = input_dir if output_dir: os.environ["PUDL_OUTPUT"] = output_dir - - -def init(clobber=False): - """Set up a new PUDL working environment based on the user settings. - - Args: - clobber (bool): if True, replace existing files. If False (the default) - do not replace existing files. - - Returns: - None - """ - # Create tmp directory - tmp_dir = PudlPaths().data_dir / "tmp" - tmp_dir.mkdir(parents=True, exist_ok=True) - - # These are files that may exist in the package_data directory, but that - # we do not want to deploy into a user workspace: - ignore_files = ["__init__.py", ".gitignore"] - - # TODO(janrous): perhaps we don't need to do this? - # Make a settings directory in the workspace, and deploy settings files: - settings_dir = PudlPaths().settings_dir - settings_dir.mkdir(parents=True, exist_ok=True) - settings_pkg = "pudl.package_data.settings" - deploy(settings_pkg, settings_dir, ignore_files, clobber=clobber) - - # Make output directory: - PudlPaths().output_dir.mkdir(parents=True, exist_ok=True) - # TODO(rousik): it might make sense to turn this into a method of - # PudlPaths object and to move this to settings.py from this module. - # Unclear whether deployment of settings files makes much sense. - - -def deploy( - pkg_path: str, - deploy_dir: pathlib.Path, - ignore_files: list[str], - clobber: bool = False, -) -> None: - """Deploy all files from a package_data directory into a workspace. - - Args: - pkg_path: Dotted module path to the subpackage inside of package_data containing - the resources to be deployed. - deploy_dir: Directory on the filesystem to which the files within pkg_path - should be deployed. - ignore_files: List of filenames (strings) that may be present in the pkg_path - subpackage, but that should be ignored. - clobber: if True, replace existing copies of the files that are being deployed - from pkg_path to deploy_dir. If False, do not replace existing files. - - Returns: - None - """ - files = [ - path - for path in importlib.resources.files(pkg_path).iterdir() - if path.is_file() and path.name not in ignore_files - ] - for file in files: - dest_file = pathlib.Path(deploy_dir, file) - if pathlib.Path.exists(dest_file): - if clobber: - logger.info(f"CLOBBERING existing file at {dest_file}.") - else: - logger.info(f"Skipping existing file at {dest_file}") - continue - - pkg_source = importlib.resources.files(pkg_path) / file - with importlib.resources.as_file(pkg_source) as f: - shutil.copy(f, dest_file) diff --git a/src/pudl/workspace/setup_cli.py b/src/pudl/workspace/setup_cli.py deleted file mode 100644 index 66001bde29..0000000000 --- a/src/pudl/workspace/setup_cli.py +++ /dev/null @@ -1,121 +0,0 @@ -"""Set up a well-organized PUDL data management workspace. - -This script creates a well-defined directory structure for use by the PUDL -package, and copies several example settings files and Jupyter notebooks into -it to get you started. If the command is run without any arguments, it will -create this workspace in your current directory. - -It's also possible to specify different input and output directories, which is -useful if you want to use a single PUDL data store (which may contain many GB -of data) to support several different workspaces. See the --pudl_in and ---pudl_out options. - -By default the script will not overwrite existing files. If you want it to -replace existing files use the --clobber option. - -The directory structure set up for PUDL looks like this: - -PUDL_DIR - └── settings - -PUDL_INPUT - ├── censusdp1tract - ├── eia860 - ├── eia860m - ├── eia861 - ├── eia923 - ... - ├── epacems - ├── ferc1 - ├── ferc714 - └── tmp - -PUDL_OUTPUT - ├── ferc1_dbf.sqlite - ├── ferc1_xbrl.sqlite - ... - ├── pudl.sqlite - └── hourly_emissions_cems.parquet - -Initially, the directories in the data store will be empty. The pudl_datastore or -pudl_etl commands will download data from public sources and organize it for -you there by source. -""" -import argparse -import pathlib -import sys - -import pudl -from pudl.workspace.setup import PudlPaths - -logger = pudl.logging_helpers.get_logger(__name__) - - -def initialize_parser(): - """Parse command line arguments for the pudl_setup script.""" - parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter - ) - parser.add_argument( - "--pudl_in", - "-i", - type=str, - dest="pudl_in", - help="""Directory where the PUDL input data should be located.""", - ) - parser.add_argument( - "--pudl_out", - "-o", - type=str, - dest="pudl_out", - help="""Directory where the PUDL outputs, notebooks, and example - settings files should be located.""", - ) - parser.add_argument( - "--clobber", - "-c", - action="store_true", - help="""Replace existing settings files, notebooks, etc. with fresh - copies of the examples distributed with the PUDL Python package. This - will also update your default PUDL workspace, if you have one.""", - default=False, - ) - parser.add_argument( - "--logfile", - default=None, - type=str, - help="If specified, write logs to this file.", - ) - parser.add_argument( - "--loglevel", - help="Set logging level (DEBUG, INFO, WARNING, ERROR, or CRITICAL).", - default="INFO", - ) - return parser - - -def main(): - """Set up a new default PUDL workspace.""" - # Display logged output from the PUDL package: - - parser = initialize_parser() - args = parser.parse_args(sys.argv[1:]) - pudl.logging_helpers.configure_root_logger( - logfile=args.logfile, loglevel=args.loglevel - ) - - if args.pudl_in: - pudl_in = pathlib.Path(args.pudl_in).expanduser().resolve() - if not pathlib.Path.is_dir(pudl_in): - raise FileNotFoundError(f"Directory not found: {pudl_in}") - PudlPaths.set_path_overrides(input_dir=pudl_in) - if args.pudl_out: - pudl_out = pathlib.Path(args.pudl_out).expanduser().resolve() - if not pathlib.Path.is_dir(pudl_out): - raise FileNotFoundError(f"Directory not found: {pudl_out}") - PudlPaths.set_path_overrides(output_dir=pudl_out) - pudl.workspace.setup.init(clobber=args.clobber) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/test/conftest.py b/test/conftest.py index a09dc516f0..d2a8acbf14 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -11,9 +11,8 @@ import sqlalchemy as sa from dagster import build_init_resource_context, materialize_to_memory -import pudl from pudl import resources -from pudl.cli.etl import pudl_etl_job_factory +from pudl.etl.cli import pudl_etl_job_factory from pudl.extract.ferc1 import raw_xbrl_metadata_json from pudl.ferc_to_sqlite.cli import ferc_to_sqlite_job_factory from pudl.io_managers import ( @@ -364,7 +363,6 @@ def configure_paths_for_tests(tmp_path_factory, request): output_dir=str(Path(out_tmp).resolve()), ) logger.info(f"Using temporary PUDL_OUTPUT: {out_tmp}") - pudl.workspace.setup.init() @pytest.fixture(scope="session") diff --git a/test/integration/console_scripts_test.py b/test/integration/console_scripts_test.py index 41eedf931d..a21007eb34 100644 --- a/test/integration/console_scripts_test.py +++ b/test/integration/console_scripts_test.py @@ -1,20 +1,101 @@ """Test the PUDL console scripts from within PyTest.""" +from pathlib import Path -import importlib.metadata - +import geopandas as gpd import pytest +import sqlalchemy as sa -# Obtain a list of all deployed entry point scripts to test: -PUDL_SCRIPTS = [ - ep.name - for ep in importlib.metadata.entry_points(group="console_scripts") - if ep.value.startswith("pudl") -] +@pytest.mark.parametrize( + "command", + [ + # Force the download of one small partition from Zenodo + "pudl_datastore --dataset ferc60 --bypass-local-cache --partition year=2020", + # Force the download of one small partition from GCS + "pudl_datastore -d ferc60 --bypass-local-cache --gcs-cache-path gs://zenodo-cache.catalyst.coop --partition year=2020", + # Exercise the datastore validation code + "pudl_datastore -d ferc60 --validate --partition year=2020", + # Ensure that all data source partitions are legible + "pudl_datastore --list-partitions --gcs-cache-path gs://zenodo-cache.catalyst.coop", + ], +) +@pytest.mark.script_launch_mode("inprocess") +def test_pudl_datastore(script_runner, command: str): + """CLI tests specific to the pudl_datastore script.""" + runner_args = command.split(" ") + ret = script_runner.run(runner_args, print_result=True) + assert ret.success -@pytest.mark.parametrize("script_name", PUDL_SCRIPTS) + +@pytest.mark.parametrize( + "command,filename,expected_cols", + [ + ( + "pudl_service_territories --entity-type ba -y 2022 --limit-by-state --no-dissolve -o ", + "balancing_authority_geometry_limited.parquet", + { + "area_km2", + "balancing_authority_id_eia", + "county", + "county_id_fips", + "county_name_census", + "geometry", + "population", + "report_date", + "state", + "state_id_fips", + }, + ), + ( + "pudl_service_territories --entity-type ba -y 2021 -y 2022 --no-dissolve -o ", + "balancing_authority_geometry.parquet", + { + "area_km2", + "balancing_authority_id_eia", + "county", + "county_id_fips", + "county_name_census", + "geometry", + "population", + "report_date", + "state", + "state_id_fips", + }, + ), + ( + "pudl_service_territories --entity-type util -y 2022 --dissolve -o ", + "utility_geometry_dissolved.parquet", + { + "area_km2", + "geometry", + "population", + "report_date", + "utility_id_eia", + }, + ), + ], +) @pytest.mark.script_launch_mode("inprocess") -def test_pudl_scripts(script_runner, script_name): - """Run each console script in --help mode for testing.""" - ret = script_runner.run([script_name, "--help"], print_result=False) +def test_pudl_service_territories( + script_runner, + command: str, + tmp_path: Path, + filename: str, + expected_cols: set[str], + pudl_engine: sa.Engine, +): + """CLI tests specific to the pudl_service_territories script. + + Depends on the ``pudl_engine`` fixture to ensure that the censusdp1tract.sqlite + database has been generated, since that data is required for the script to run. + """ + out_path = tmp_path / filename + assert not out_path.exists() + command += str(tmp_path) + ret = script_runner.run(command.split(" "), print_result=True) assert ret.success + assert out_path.exists() + assert out_path.is_file() + gdf = gpd.read_parquet(out_path) + assert set(gdf.columns) == expected_cols + assert not gdf.empty diff --git a/test/unit/console_scripts_test.py b/test/unit/console_scripts_test.py new file mode 100644 index 0000000000..d28a9874df --- /dev/null +++ b/test/unit/console_scripts_test.py @@ -0,0 +1,22 @@ +"""Test the PUDL console scripts from within PyTest.""" + +import importlib.metadata + +import pytest + +# Obtain a list of all deployed entry point scripts to test: +PUDL_SCRIPTS = [ + ep.name + for ep in importlib.metadata.entry_points(group="console_scripts") + if ep.value.startswith("pudl") +] + + +@pytest.mark.parametrize("script_name", PUDL_SCRIPTS) +@pytest.mark.script_launch_mode("inprocess") +def test_pudl_scripts(script_runner, script_name): + """Run each console script in --help mode for testing.""" + ret = script_runner.run([script_name, "--help"], print_result=False) + assert ret.success + ret = script_runner.run([script_name, "-h"], print_result=False) + assert ret.success diff --git a/test/unit/io_managers_test.py b/test/unit/io_managers_test.py index eb407246c9..5cb8c863cd 100644 --- a/test/unit/io_managers_test.py +++ b/test/unit/io_managers_test.py @@ -242,6 +242,7 @@ def test_pudl_sqlite_io_manager_delete_stmt(fake_pudl_sqlite_io_manager_fixture) assert len(returned_df) == 1 +@pytest.mark.slow def test_migrations_match_metadata(tmp_path, monkeypatch): """If you create a `PudlSQLiteIOManager` that points at a non-existing `pudl.sqlite` - it will initialize the DB based on the `package`. @@ -422,7 +423,10 @@ def test_ferc_xbrl_sqlite_io_manager_dedupes(mocker, tmp_path): ) -@hypothesis.settings(print_blob=True, deadline=400) +# ridiculous deadline - dataframe generation is always slow and sometimes +# *very* slow +@pytest.mark.slow +@hypothesis.settings(print_blob=True, deadline=2_000) @hypothesis.given(example_schema.strategy(size=3)) def test_filter_for_freshest_data(df): # XBRL context is the identifying metadata for reported values diff --git a/test/unit/settings_test.py b/test/unit/settings_test.py index d2495be82d..c2034f7695 100644 --- a/test/unit/settings_test.py +++ b/test/unit/settings_test.py @@ -281,6 +281,7 @@ def test_partitions_with_json_normalize(pudl_etl_settings): ) +@pytest.mark.slow def test_partitions_for_datasource_table(pudl_etl_settings): """Test whether or not we can make the datasource table.""" ds = Datastore(local_cache_path=PudlPaths().data_dir)