Merge branch 'dev' into AK-132-Geo-based-access

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 4.1.9
+current_version = 4.1.15
 commit = False
 tag = False
 

.env.example

@@ -4,3 +4,6 @@ FLASK_SECRET=abc
 #API_KEY_REQUIRED_STARTING_AT=2021-07-30
 API_KEY_ADMIN_PASSWORD=abc
 API_KEY_REGISTER_WEBHOOK_TOKEN=abc
+
+# Sentry
+# If setting a Sentry DSN, note that the URL should NOT be quoted!

.git-blame-ignore-revs

@@ -1,7 +1,5 @@
 # style(black): format cdc acquisition
 980b0b7e80c7923b79e14fee620645e680785703
-# style(black): format covidcast_nowcast acquisition
-9e6ff16f599e8feec34a08dd1bddbc5eae347b55
 # style(black): format ecdc acquisition
 d1141d904da4e62992b97c92d5caebd8fadffd42
 # style(black): format flusurv acquisition

.github/workflows/ci.yaml

@@ -75,7 +75,7 @@ jobs:
 
       - name: Start delphi_web_epidata
         run: |
-          docker run --rm -d -p 10080:80 --env "MODULE_NAME=delphi.epidata.server.main" --env "SQLALCHEMY_DATABASE_URI=mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata" --env "FLASK_SECRET=abc" --env "FLASK_PREFIX=/epidata" --env "REDIS_HOST=delphi_redis" --env "REDIS_PASSWORD=1234" --env "API_KEY_REGISTER_WEBHOOK_TOKEN=abc" --env "API_KEY_ADMIN_PASSWORD=test_admin_password" --network delphi-net --name delphi_web_epidata delphi_web_epidata
+          docker run --rm -d -p 10080:80 --env "MODULE_NAME=delphi.epidata.server.main" --env "SQLALCHEMY_DATABASE_URI=mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata" --env "FLASK_SECRET=abc" --env "FLASK_PREFIX=/epidata" --env "REDIS_HOST=delphi_redis" --env "REDIS_PASSWORD=1234" --env "API_KEY_REGISTER_WEBHOOK_TOKEN=abc" --env "API_KEY_ADMIN_PASSWORD=test_admin_password" --env "TESTING_MODE=True" --network delphi-net --name delphi_web_epidata delphi_web_epidata
           docker ps
 
       - name: Run Unit Tests

.github/workflows/performance-tests-one-time.yml

@@ -0,0 +1,108 @@
+name: One-time performance testing - 8th November 2023
+
+# Run "At every 30th minute on day-of-month 8 in November"
+on:
+  schedule:
+    - cron: '*/30 * 8 11 *'
+
+# Add some extra perms to comment on a PR
+permissions:
+  pull-requests: write
+  contents: read
+
+jobs:
+  run-perftests:
+    # Run this on Delphi's self-hosted runner
+    runs-on: self-hosted
+    outputs:
+      request_count: ${{ steps.output.outputs.request_count }}
+      failure_count: ${{ steps.output.outputs.failure_count }}
+      med_time: ${{ steps.output.outputs.med_time }}
+      avg_time: ${{ steps.output.outputs.avg_time }}
+      min_time: ${{ steps.output.outputs.min_time }}
+      max_time: ${{ steps.output.outputs.max_time }}
+      requests_per_sec: ${{ steps.output.outputs.requests_per_sec }}
+    steps:
+      - name: Set up WireGuard
+        uses: egor-tensin/[email protected]
+        with:
+          endpoint: '${{ secrets.WG_PERF_ENDPOINT }}'
+          endpoint_public_key: '${{ secrets.WG_PERF_ENDPOINT_PUBLIC_KEY }}'
+          ips: '${{ secrets.WG_PERF_IPS }}'
+          allowed_ips: '${{ secrets.WG_PERF_ALLOWED_IPS }}'
+          private_key: '${{ secrets.WG_PERF_PRIVATE_KEY }}'
+      - name: Clean files from previous runs
+        uses: AutoModality/action-clean@v1
+      - name: Check out repository
+        uses: actions/checkout@v3
+      - name: Set up repository # mimics install.sh in the README except that delphi is cloned from the PR rather than main
+        run: |
+          cd ..
+          rm -rf driver
+          mkdir -p driver/repos/delphi
+          cd driver/repos/delphi
+          git clone https://github.com/cmu-delphi/operations
+          git clone https://github.com/cmu-delphi/utils
+          git clone https://github.com/cmu-delphi/flu-contest
+          git clone https://github.com/cmu-delphi/nowcast
+          cd ../../
+
+          cd ..
+          cp -R delphi-epidata driver/repos/delphi/delphi-epidata
+          cd -
+
+          ln -s repos/delphi/delphi-epidata/dev/local/Makefile
+      - name: Build & run epidata
+        run: |
+          cd ../driver
+          sudo make web sql="${{ secrets.DB_CONN_STRING }}" rate_limit="999999/second"
+          sudo make redis
+      - name: Check out delphi-admin
+        uses: actions/checkout@v3
+        with:
+          repository: cmu-delphi/delphi-admin
+          token: ${{ secrets.CMU_DELPHI_DEPLOY_MACHINE_PAT }}
+          path: delphi-admin
+      - name: Build & run Locust
+        continue-on-error: true # sometimes ~2-5 queries fail, we shouldn't end the run if that's the case
+        env:
+          PERFTEST_API_KEY: ${{secrets.PERFTEST_API_KEY}}
+        run: |
+          cd delphi-admin/load-testing/locust
+          docker build -t locust .
+          export CSV=v4-requests-small.csv
+          touch output_stats.csv && chmod 666 output_stats.csv
+          touch output_stats_history.csv && chmod 666 output_stats_history.csv
+          touch output_failures.csv && chmod 666 output_failures.csv
+          touch output_exceptions.csv && chmod 666 output_exceptions.csv
+          docker run --net=host -v $PWD:/mnt/locust -e CSV="/mnt/locust/${CSV}" locust -f /mnt/locust/v4.py --host http://127.0.0.1:10080/ --users 10 --spawn-rate 1 --headless -i "$(cat ${CSV} | wc -l)" --csv=/mnt/locust/output
+      - name: Produce output for summary
+        id: output
+        uses: jannekem/run-python-script-action@v1
+        with:
+          script: |
+            import os
+
+            def write_string(name, value):
+              with open(os.environ['GITHUB_OUTPUT'], 'a') as fh:
+                print(f'{name}={value}', file=fh)
+
+            def write_float(name, value):
+              write_string(name, "{:.2f}".format(float(value)))
+
+            with open("delphi-admin/load-testing/locust/output_stats.csv", "r", encoding="utf-8", errors="ignore") as scraped:
+              final_line = scraped.readlines()[-1].split(",")
+              write_string('request_count', final_line[2])
+              write_string('failure_count', final_line[3])
+              write_float('med_time', final_line[4])
+              write_float('avg_time', final_line[5])
+              write_float('min_time', final_line[6])
+              write_float('max_time', final_line[7])
+              write_float('requests_per_sec', final_line[9])
+
+      - name: Archive results as artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: locust-output
+          path: |
+            delphi-admin/load-testing/locust/output_*.csv

.github/workflows/performance-tests.yml

@@ -33,6 +33,8 @@ jobs:
           ips: '${{ secrets.WG_PERF_IPS }}'
           allowed_ips: '${{ secrets.WG_PERF_ALLOWED_IPS }}'
           private_key: '${{ secrets.WG_PERF_PRIVATE_KEY }}'
+      - name: Clean files from previous runs
+        uses: AutoModality/action-clean@v1
       - name: Check out repository
         uses: actions/checkout@v3
       # Previous step checks out default branch, so we check out the pull request's branch
@@ -44,6 +46,7 @@ jobs:
       - name: Set up repository # mimics install.sh in the README except that delphi is cloned from the PR rather than main
         run: |
           cd ..
+          rm -rf driver
           mkdir -p driver/repos/delphi
           cd driver/repos/delphi
           git clone https://github.com/cmu-delphi/operations
@@ -60,7 +63,7 @@ jobs:
       - name: Build & run epidata
         run: |
           cd ../driver
-          sudo make web sql="${{ secrets.DB_CONN_STRING }}"
+          sudo make web sql="${{ secrets.DB_CONN_STRING }}" rate_limit="999999/second"
           sudo make redis
       - name: Check out delphi-admin
         uses: actions/checkout@v3
@@ -70,6 +73,8 @@ jobs:
           path: delphi-admin
       - name: Build & run Locust
         continue-on-error: true # sometimes ~2-5 queries fail, we shouldn't end the run if that's the case
+        env:
+          PERFTEST_API_KEY: ${{secrets.PERFTEST_API_KEY}}
         run: |
           cd delphi-admin/load-testing/locust
           docker build -t locust .

deploy.json

@@ -208,15 +208,6 @@
       "add-header-comment": true
     },
 
-    "// acquisition - covidcast_nowcast",
-    {
-      "type": "move",
-      "src": "src/acquisition/covidcast_nowcast/",
-      "dst": "[[package]]/acquisition/covidcast_nowcast/",
-      "match": "^.*\\.(py)$",
-      "add-header-comment": true
-    },
-
     "// maintenance",
     {
       "type": "move",

dev/local/Makefile

@@ -37,6 +37,7 @@
 #   test=         Only runs tests in the directories provided here, e.g.
 #                 repos/delphi/delphi-epidata/tests/acquisition/covidcast
 #   sql=          Overrides the default SQL connection string.
+#   rate_limit=   Overrides the default rate limit for API requests.
 
 
 # Set optional argument defaults
@@ -56,6 +57,14 @@ else
 	sqlalchemy_uri:=mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata
 endif
 
+ifdef rate_limit
+# Notation found here: https://flask-limiter.readthedocs.io/en/stable/#rate-limit-string-notation
+	rate_limit_settings:=--env "RATE_LIMIT=$(rate_limit)"
+else
+# Default behavior is to set the rate limit to "5/hour" for API key tests via this environment variable
+	rate_limit_settings:=--env "TESTING_MODE=True"
+endif
+
 SHELL:=/bin/sh
 
 # Get the Makefile's absolute path: https://stackoverflow.com/a/324782/4784655
@@ -68,6 +77,7 @@ LOG_REDIS:=delphi_redis_instance_$(NOW).log
 WEB_CONTAINER_ID:=$(shell docker ps -q --filter 'name=delphi_web_epidata')
 DATABASE_CONTAINER_ID:=$(shell docker ps -q --filter 'name=delphi_database_epidata')
 REDIS_CONTAINER_ID:=$(shell docker ps -q --filter 'name=delphi_redis')
+ENV_FILE:=repos/delphi/delphi-epidata/.env
 
 M1=
 ifeq ($(shell uname -smp), Darwin arm64 arm)
@@ -95,15 +105,18 @@ web:
 	@# Run the web server
 	@# MODULE_NAME specifies the location of the `app` variable, the actual WSGI application object to run.
 	@# see https://github.com/tiangolo/meinheld-gunicorn-docker#module_name
+	@touch $(ENV_FILE)
 	@docker run --rm -p 127.0.0.1:10080:80 \
 		$(M1) \
+		--env-file $(ENV_FILE) \
 		--env "MODULE_NAME=delphi.epidata.server.main" \
 		--env "SQLALCHEMY_DATABASE_URI=$(sqlalchemy_uri)" \
 		--env "FLASK_SECRET=abc" --env "FLASK_PREFIX=/epidata" --env "LOG_DEBUG" \
 		--env "REDIS_HOST=delphi_redis" \
 		--env "REDIS_PASSWORD=1234" \
 		--env "API_KEY_ADMIN_PASSWORD=test_admin_password" \
 		--env "API_KEY_REGISTER_WEBHOOK_TOKEN=abc" \
+		$(rate_limit_settings) \
 		--network delphi-net --name delphi_web_epidata \
 		delphi_web_epidata >$(LOG_WEB) 2>&1 &
 

dev/local/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = Delphi Development
-version = 4.1.9
+version = 4.1.15
 
 [options]
 packages =
@@ -13,7 +13,6 @@ packages =
     delphi.epidata.acquisition.covid_hosp.state_daily
     delphi.epidata.acquisition.covid_hosp.state_timeseries
     delphi.epidata.acquisition.covidcast
-    delphi.epidata.acquisition.covidcast_nowcast
     delphi.epidata.acquisition.ecdc
     delphi.epidata.acquisition.flusurv
     delphi.epidata.acquisition.fluview

devops/Dockerfile

@@ -7,7 +7,6 @@ FROM tiangolo/meinheld-gunicorn:python3.8
 LABEL org.opencontainers.image.source=https://github.com/cmu-delphi/delphi-epidata
 
 COPY ./devops/gunicorn_conf.py /app
-COPY ./devops/start_wrapper.sh /
 RUN mkdir -p /app/delphi/epidata
 COPY ./src/server /app/delphi/epidata/server
 COPY ./src/common /app/delphi/epidata/common
@@ -18,7 +17,6 @@ COPY requirements.api.txt /app/requirements_also.txt
 RUN ln -s -f /usr/share/zoneinfo/America/New_York /etc/localtime \
       && rm -rf /app/delphi/epidata/__pycache__ \
       && chmod -R o+r /app/delphi/epidata \
-      && chmod 755 /start_wrapper.sh \
       && pip install --no-cache-dir -r /tmp/requirements.txt -r requirements_also.txt
 # the file /tmp/requirements.txt is created in the parent docker definition.  (see:
 #   https://github.com/tiangolo/meinheld-gunicorn-docker/blob/master/docker-images/python3.8.dockerfile#L5 )
@@ -28,4 +26,4 @@ RUN ln -s -f /usr/share/zoneinfo/America/New_York /etc/localtime \
 ENV PYTHONUNBUFFERED 1
 
 ENTRYPOINT [ "/entrypoint.sh" ]
-CMD [ "/start_wrapper.sh" ]
+CMD [ "/start.sh" ]

docs/epidata_development.md

@@ -388,3 +388,13 @@ The command above maps two local directories into the container:
 - `/repos/delphi/delphi-epidata/src`: Just the source code, which forms the
   container's `delphi.epidata` python package.
 
+## instrumentation with Sentry
+
+Delphi uses [Sentry](https://sentry.io/welcome/) in production for debugging, APM, and other observability purposes. You can instrument your local environment if you want to take advantage of Sentry's features during the development process. In most cases this option is available to internal Delphi team members only.
+
+The bare minimum to set up instrumentation is to supply the DSN for the [epidata-api](https://cmu-delphi.sentry.io/projects/epidata-api/?project=4506123377442816) Sentry project to the application environment.
+
+- You can get the DSN from the Sentry [project's keys config](https://cmu-delphi.sentry.io/settings/projects/epidata-api/keys/), or by asking someone in the prodsys, DevOps, or sysadmin space.
+- Once you have the DSN, add it to your local `.env` file and rebuild your containers to start sending telemetry to Sentry.
+
+Additional internal documentation for Sentry can be found [here](https://bookstack.delphi.cmu.edu/books/systems-handbook/page/sentry).

docs/new_endpoint_tutorial.md

@@ -123,11 +123,9 @@ Here's what we add to each client:
     def fluview_meta():
       """Fetch FluView metadata."""
       # Set up request
-      params = {
-        'endpoint': 'fluview_meta',
-      }
+      params = {}
       # Make the API call
-      return Epidata._request(params)
+      return Epidata._request("fluview_meta", params)
     ```
 
 - [`delphi_epidata.R`](https://github.com/cmu-delphi/delphi-epidata/blob/dev/src/client/delphi_epidata.R)

docs/symptom-survey/publications.md

@@ -26,10 +26,24 @@ Pandemic"](https://www.pnas.org/topic/548) in *PNAS*:
 
 Research publications using the survey data include:
 
+- de Vries, M., Kim, J.Y. & Han, H. (2023). [The unequal landscape of civic
+  opportunity in America](https://doi.org/10.1038/s41562-023-01743-1). *Nature
+  Human Behavior*. 
+- E. Tuzhilina, T. J. Hastie, D. J. McDonald, J. K. Tay & R. Tibshirani (2023).
+  [Smooth multi-period forecasting with application to prediction of COVID-19
+  cases](https://doi.org/10.1080/10618600.2023.2285337). *Journal of Computational
+  and Graphical Statistics*.
+- W. Dempsey (2023). [Addressing selection bias and measurement error in
+  COVID-19 case count data using auxiliary information](https://doi.org/10.1214/23-AOAS1744).
+  *Annals of Applied Statistics* 17 (4), 2903-2923.
+- Ma, M.Z., Chen, S.X. (2023). [Beyond the surface: accounting for confounders
+  in understanding the link between collectivism and COVID-19 pandemic in the
+  United States](https://doi.org/10.1186/s12889-023-16384-2). *BMC Public
+  Health* 23, 1513.
 - C.K. Ettman, E. Badillo Goicoechea, and E.A. Stuart (2023). [Evolution of
   depression and anxiety over the COVID-19 pandemic and across demographic
   groups in a large sample of U.S. adults](https://doi.org/10.1016/j.focus.2023.100140).
-  *AJPM Focus*.
+  *AJPM Focus* 2 (4), 100140.
 - M. Rubinstein, Z. Branson, and E.H. Kennedy (2023). [Heterogeneous
   interventional effects with multiple mediators: Semiparametric and
   nonparametric approaches](https://doi.org/10.1515/jci-2022-0070). *Journal of
@@ -62,7 +76,7 @@ Research publications using the survey data include:
 - Rönn MM, Menzies NA, Salomon JA (2023). [Vaccination and voting patterns in
   the United States: analysis of COVID-19 and flu surveys from 2010 to
   2022](https://doi.org/10.1016/j.amepre.2023.03.001). *American Journal of
-  Preventive Medicine.*
+  Preventive Medicine* 65 (3), 458-466.
 - Taube JC, Susswein Z, Bansal S (2023). [Spatiotemporal Trends in Self-Reported
   Mask-Wearing Behavior in the United States: Analysis of a Large
   Cross-sectional Survey](https://doi.org/10.2196/42128). *JMIR Public Health

integrations/acquisition/covid_hosp/facility/test_scenarios.py

@@ -28,7 +28,7 @@ def setUp(self):
     self.test_utils = UnitTestUtils(__file__)
 
     # use the local instance of the Epidata API
-    Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php'
+    Epidata.BASE_URL = 'http://delphi_web_epidata/epidata'
     Epidata.auth = ('epidata', 'key')
 
     # use the local instance of the epidata database