Merge branch 'dev' into ndefries/flusurv-new-endpoint

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 4.1.15
+current_version = 4.1.30
 commit = False
 tag = False
 
@@ -11,8 +11,4 @@ tag = False
 
 [bumpversion:file:src/client/packaging/npm/package.json]
 
-[bumpversion:file:src/client/packaging/pypi/setup.py]
-
-[bumpversion:file:src/client/packaging/pypi/delphi_epidata/__init__.py]
-
 [bumpversion:file:dev/local/setup.cfg]

.git-blame-ignore-revs

@@ -20,3 +20,5 @@ b9ceb400d9248c8271e8342275664ac5524e335d
 07ed83e5768f717ab0f9a62a9209e4e2cffa058d
 # style(black): format wiki acquisition
 923852eafa86b8f8b182d499489249ba8f815843
+# lint: trailing whitespace changes
+81179c5f144b8f25421e799e823e18cde43c84f9

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,4 +1,4 @@
-closes|addresses <!--list issues closed or partially-addressed by this PR -->
+addresses issue(s) #ISSUE <!--list issue(s) associated with this PR -->
 
 ### Summary:
 

.github/workflows/create-release.yml

@@ -4,7 +4,7 @@ on:
   workflow_dispatch:
     inputs:
       versionName:
-        description: 'Semantic Version Number (i.e., 5.5.0 or patch, minor, major, prepatch, preminor, premajor, prerelease)'
+        description: 'Semantic Version Number (i.e., 5.5.0 or patch, minor, major)'
         required: true
         default: patch
 
@@ -27,11 +27,43 @@ jobs:
           python-version: 3.8
       - name: Change version number
         id: version
+        # See this issue for explanation and testing:
+        # https://github.com/cmu-delphi/delphi-epidata/pull/1473
         run: |
           python -m pip install bump2version
-          echo -n "::set-output name=next_tag::"
-          bump2version --list ${{ github.event.inputs.versionName }} | grep new_version | sed -r s,"^.*=",,
-      - name: Create pull request into prod
+          if [[ ${{ github.event.inputs.versionName }} =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+            # use given version number
+            NEXT_TAG="${{ github.event.inputs.versionName }}"
+          elif [[ ${{ github.event.inputs.versionName }} =~ ^(major|minor|patch)$ ]]; then
+            # calculate new version number based on given tag
+            NEXT_TAG=$(bump2version --dry-run --list ${{ github.event.inputs.versionName }} | grep ^new_version | sed -r s,"^.*=",,)
+          else
+            echo "\nInvalid version name: ${{ github.event.inputs.versionName }}"
+            exit 1
+          fi
+          # apply given or calculated version number
+          bump2version --new-version $NEXT_TAG _ignored_arg_
+          # save version number for later
+          echo "next_tag=$NEXT_TAG" >> $GITHUB_OUTPUT
+      - name: Get Python client file changes
+        id: changed-py
+        uses: dorny/paths-filter@v3
+        with:
+            base: "main"
+            ref: "dev"
+            filters: |
+              python_client_changed:
+                - src/client/delphi_epidata.py
+                - src/client/packaging/pypi/**
+      - name: Bump Python versions if client files changed
+        if: steps.changed-py.outputs.python_client_changed == 'true'
+        # _ignored_arg_ below is required because of tool quirk
+        # https://github.com/c4urself/bump2version/issues/22
+        run: |
+          cd src/client/packaging/pypi
+          # this is using the new literal version number (steps.version.outputs.next_tag) output from the bump2version step above, not the workflow input (github.event.inputs.versionName) which can be relative
+          bump2version --allow-dirty --new-version ${{ steps.version.outputs.next_tag }} _ignored_arg_
+      - name: Create pull request into main
         uses: peter-evans/create-pull-request@v3
         with:
           branch: release/delphi-epidata-${{ steps.version.outputs.next_tag }}

.github/workflows/missing_signals.yaml

@@ -21,7 +21,7 @@ jobs:
         run: python scripts/report_missing_covidcast_meta.py
       - name: Upload Missing Artifact
         if: failure()
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: missing_db_signals.csv
           path:  missing_db_signals.csv

.github/workflows/performance-tests-one-time.yml

@@ -23,14 +23,6 @@ jobs:
       max_time: ${{ steps.output.outputs.max_time }}
       requests_per_sec: ${{ steps.output.outputs.requests_per_sec }}
     steps:
-      - name: Set up WireGuard
-        uses: egor-tensin/[email protected]
-        with:
-          endpoint: '${{ secrets.WG_PERF_ENDPOINT }}'
-          endpoint_public_key: '${{ secrets.WG_PERF_ENDPOINT_PUBLIC_KEY }}'
-          ips: '${{ secrets.WG_PERF_IPS }}'
-          allowed_ips: '${{ secrets.WG_PERF_ALLOWED_IPS }}'
-          private_key: '${{ secrets.WG_PERF_PRIVATE_KEY }}'
       - name: Clean files from previous runs
         uses: AutoModality/action-clean@v1
       - name: Check out repository
@@ -101,7 +93,7 @@ jobs:
               write_float('requests_per_sec', final_line[9])
 
       - name: Archive results as artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: locust-output
           path: |

.github/workflows/performance-tests.yml

@@ -25,14 +25,6 @@ jobs:
       max_time: ${{ steps.output.outputs.max_time }}
       requests_per_sec: ${{ steps.output.outputs.requests_per_sec }}
     steps:
-      - name: Set up WireGuard
-        uses: egor-tensin/[email protected]
-        with:
-          endpoint: '${{ secrets.WG_PERF_ENDPOINT }}'
-          endpoint_public_key: '${{ secrets.WG_PERF_ENDPOINT_PUBLIC_KEY }}'
-          ips: '${{ secrets.WG_PERF_IPS }}'
-          allowed_ips: '${{ secrets.WG_PERF_ALLOWED_IPS }}'
-          private_key: '${{ secrets.WG_PERF_PRIVATE_KEY }}'
       - name: Clean files from previous runs
         uses: AutoModality/action-clean@v1
       - name: Check out repository
@@ -109,7 +101,7 @@ jobs:
               write_float('requests_per_sec', final_line[9])
 
       - name: Archive results as artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: locust-output
           path: |

.github/workflows/release-helper.yml

@@ -13,6 +13,29 @@ jobs:
         if: github.repository_owner != 'cmu-delphi'
         run: exit 1
 
+  client_changed:
+    needs: correct_repository
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 2
+      - name: Get Python client file changes
+        id: changed-py
+        uses: dorny/paths-filter@v3
+        with:
+          # when base and ref are the same, changes are detected against the most recent commit before the push.
+          # see https://github.com/dorny/paths-filter#usage for details.
+          base: 'main'
+          ref: 'main'
+          filters: |
+            python_client_changed:
+              - src/client/delphi_epidata.py
+              - src/client/packaging/pypi/**
+    outputs:
+      python_client_changed: ${{ steps.changed-py.outputs.python_client_changed }}
+
   create_release:
     needs: correct_repository
     runs-on: ubuntu-latest
@@ -29,8 +52,8 @@ jobs:
         id: extract_version
         run: |
           python -m pip install bump2version
-          echo -n "::set-output name=version::"
-          bump2version --dry-run --list patch | grep ^current_version | sed -r s,"^.*=",,
+          VERSION=$(bump2version --dry-run --list patch | grep ^current_version | sed -r s,"^.*=",,)
+          echo "version=$VERSION" >> $GITHUB_OUTPUT
       - name: Create Release
         id: create_release
         uses: release-drafter/release-drafter@v5
@@ -45,7 +68,8 @@ jobs:
       tag_name: ${{ steps.create_release.outputs.tag_name }}
 
   release_python_client:
-    needs: create_release
+    needs: [client_changed, create_release]
+    if: needs.client_changed.outputs.python_client_changed == 'true'
     runs-on: ubuntu-latest
     steps:
       - name: Check out code
@@ -57,15 +81,15 @@ jobs:
       - name: Install build dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install wheel twine
+          pip install build wheel twine
       - name: Prepare package
         run: |
           cp src/client/*.py src/client/packaging/pypi/delphi_epidata/
       - name: Create release
         working-directory: src/client/packaging/pypi
         run: |
-          python setup.py sdist bdist_wheel
-      - uses: actions/upload-artifact@v2
+          python -m build --sdist --wheel
+      - uses: actions/upload-artifact@v4
         with:
           name: delphi_epidata_py
           path: src/client/packaging/pypi/dist/*.tar.gz
@@ -80,8 +104,8 @@ jobs:
         with:
           user: __token__
           password: ${{ secrets.DELPHI_PYPI_PROD_TOKEN }}
-          packages_dir: src/client/packaging/pypi/dist/
-          skip_existing: true
+          packages-dir: src/client/packaging/pypi/dist/
+          skip-existing: true
           # repository_url: https://test.pypi.org/legacy/
 
   release_js_client:
@@ -95,7 +119,7 @@ jobs:
         uses: actions/checkout@v2
       - uses: actions/setup-node@v2
         with:
-          node-version: '16.x'
+          node-version: "16.x"
       - name: Cache Node.js modules
         uses: actions/cache@v2
         with:
@@ -108,7 +132,7 @@ jobs:
       - run: npm pack
       - name: Rename to a different name
         run: for f in *.tgz; do mv "$f" "$(echo "$f" | sed s/delphi_epidata-/delphi_epidata_js-/)"; done
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v4
         with:
           name: delphi_epidata_js
           path: src/client/packaging/npm/*.tgz

.github/workflows/update_gdocs_data.yml

@@ -21,7 +21,10 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-pipd-
       - name: Install Dependencies
-        run: pip install -r requirements.dev.txt
+        run: |
+          pip -V
+          python -m pip install pip==22.0.2
+          pip install -r requirements.dev.txt
       - name: Update Docs
         run: inv update-gdoc
       - name: Create pull request into dev

.gitignore

@@ -11,3 +11,35 @@ __pycache__/
 /node_modules
 .mypy_cache
 /missing_db_signals.csv
+
+
+
+##Added by Tina to avoid tracking R studio config files
+# History files
+.Rhistory
+.Rapp.history
+
+# Session Data files
+.RData
+.RDataTmp
+
+# User-specific files
+.Ruserdata
+
+# RStudio files
+.Rproj.user/
+
+# OAuth2 token, see https://github.com/hadley/httr/releases/tag/v0.3
+.httr-oauth
+
+# knitr and R markdown default cache directories
+*_cache/
+/cache/
+
+# R Environment Variables
+.Renviron
+
+# RStudio Connect folder
+rsconnect/
+
+*.Rproj

dev/local/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = Delphi Development
-version = 4.1.15
+version = 4.1.30
 
 [options]
 packages =
@@ -24,6 +24,7 @@ packages =
     delphi.epidata.acquisition.twtr
     delphi.epidata.acquisition.wiki
     delphi.epidata.client
+    delphi.epidata.common
     delphi.epidata.server
     delphi.epidata.server.admin
     delphi.epidata.server.admin.templates

docs/Gemfile.lock

@@ -205,27 +205,28 @@ GEM
       rb-fsevent (~> 0.10, >= 0.10.3)
       rb-inotify (~> 0.9, >= 0.9.10)
     mercenary (0.3.6)
-    mini_portile2 (2.8.1)
+    mini_portile2 (2.8.6)
     minima (2.5.1)
       jekyll (>= 3.5, < 5.0)
       jekyll-feed (~> 0.9)
       jekyll-seo-tag (~> 2.1)
     minitest (5.17.0)
     multipart-post (2.1.1)
-    nokogiri (1.14.3)
-      mini_portile2 (~> 2.8.0)
+    nokogiri (1.16.5)
+      mini_portile2 (~> 2.8.2)
       racc (~> 1.4)
     octokit (4.20.0)
       faraday (>= 0.9)
       sawyer (~> 0.8.0, >= 0.5.3)
     pathutil (0.16.2)
       forwardable-extended (~> 2.6)
     public_suffix (4.0.6)
-    racc (1.6.2)
+    racc (1.7.3)
     rb-fsevent (0.10.4)
     rb-inotify (0.10.1)
       ffi (~> 1.0)
-    rexml (3.2.5)
+    rexml (3.2.8)
+      strscan (>= 3.0.9)
     rouge (3.26.0)
     ruby-enum (0.9.0)
       i18n
@@ -242,6 +243,7 @@ GEM
       faraday (> 0.8, < 2.0)
     simpleidn (0.2.1)
       unf (~> 0.1.4)
+    strscan (3.1.0)
     terminal-table (1.8.0)
       unicode-display_width (~> 1.1, >= 1.1.1)
     thread_safe (0.3.6)

docs/_config.yml

@@ -45,7 +45,7 @@ footer_content: "Copyright &copy; 2020, <a href='https://delphi.cmu.edu'>Delphi
 
 heading_anchors: true
 
-
+favicon_ico: "favicon.ico"
 # Exclude from processing.
 # The following items will not be processed, by default. Create a custom list
 # to override the default setting.

docs/about_delphi.md

@@ -0,0 +1,15 @@
+---
+title: About Delphi
+nav_order: 60
+has_children: false
+---
+
+# About The Delphi Group
+
+The Delphi Group's mission is to develop the theory and practice of epidemic detection, tracking and forecasting, and their use in decision making, both public and private.
+
+Our vision is to make this technology as useful as weather forecasting is today. 
+
+Find out more about the Delphi Research Group on our <a href="https://delphi.cmu.edu" target="_blank">offical webpage</a>.
+Please read our <a href="https://delphi.cmu.edu/blog" target="_blank">blog</a> for additional news, findings, and musings from the team.
+

docs/api/README.md

@@ -1,14 +1,15 @@
 ---
 title: Other Endpoints (COVID-19 and Other Diseases)
-nav_order: 3
 has_children: true
+nav_order: 3
 ---
 
-# Epidata API (Other Diseases)
+# Other Endpoints (COVID-19 and Other Diseases)
 
 This is the home of [Delphi](https://delphi.cmu.edu/)'s epidemiological data
 API for tracking epidemics such as influenza, dengue, and norovirus. Note that
-our work on COVID-19 is described in the [COVIDcast Epidata API documentation](covidcast.md).
+additional data, including most COVID-19 signals, is available in the
+[main Epidata API (formerly known as COVIDcast)](covidcast.md).
 
 ## Table of Contents
 {: .no_toc .text-delta}
@@ -207,4 +208,4 @@ To access our Epidata API, visit our [Epidata API Client Libraries)](client_libr
 
 # Related Work
 
-Please visit our [related works](related_work.md) page for more information.
+Please visit our [related works](../related_work.md) page for more information.