From e59aeac2969081e13c4f71c853745a80ee9c0b82 Mon Sep 17 00:00:00 2001 From: Rostyslav Zatserkovnyi Date: Tue, 19 Sep 2023 15:32:08 +0300 Subject: [PATCH 01/55] Remove usage of PHP alias in the Python client --- .../covid_hosp/facility/test_scenarios.py | 2 +- .../covid_hosp/state_daily/test_scenarios.py | 2 +- .../state_timeseries/test_scenarios.py | 2 +- .../covidcast/test_covidcast_meta_caching.py | 6 +- .../covidcast/test_csv_uploading.py | 17 ++- .../covidcast_nowcast/test_csv_uploading.py | 2 +- integrations/client/test_delphi_epidata.py | 41 ++++--- integrations/client/test_nowcast.py | 4 +- integrations/server/test_covid_hosp.py | 2 +- integrations/server/test_covidcast.py | 35 +++--- integrations/server/test_covidcast_meta.py | 5 +- integrations/server/test_covidcast_nowcast.py | 5 +- integrations/server/test_fluview.py | 2 +- integrations/server/test_fluview_meta.py | 2 +- src/client/delphi_epidata.py | 109 +++++++----------- 15 files changed, 108 insertions(+), 128 deletions(-) diff --git a/integrations/acquisition/covid_hosp/facility/test_scenarios.py b/integrations/acquisition/covid_hosp/facility/test_scenarios.py index c6c51e2f5..44ee3572d 100644 --- a/integrations/acquisition/covid_hosp/facility/test_scenarios.py +++ b/integrations/acquisition/covid_hosp/facility/test_scenarios.py @@ -28,7 +28,7 @@ def setUp(self): self.test_utils = UnitTestUtils(__file__) # use the local instance of the Epidata API - Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php' + Epidata.BASE_URL = 'http://delphi_web_epidata/epidata' Epidata.auth = ('epidata', 'key') # use the local instance of the epidata database diff --git a/integrations/acquisition/covid_hosp/state_daily/test_scenarios.py b/integrations/acquisition/covid_hosp/state_daily/test_scenarios.py index 2054d19c8..8636295bc 100644 --- a/integrations/acquisition/covid_hosp/state_daily/test_scenarios.py +++ b/integrations/acquisition/covid_hosp/state_daily/test_scenarios.py @@ -32,7 +32,7 @@ def setUp(self): self.test_utils = UnitTestUtils(__file__) # use the local instance of the Epidata API - Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php' + Epidata.BASE_URL = 'http://delphi_web_epidata/epidata' Epidata.auth = ('epidata', 'key') # use the local instance of the epidata database diff --git a/integrations/acquisition/covid_hosp/state_timeseries/test_scenarios.py b/integrations/acquisition/covid_hosp/state_timeseries/test_scenarios.py index 8565b8e7f..46bdeebcd 100644 --- a/integrations/acquisition/covid_hosp/state_timeseries/test_scenarios.py +++ b/integrations/acquisition/covid_hosp/state_timeseries/test_scenarios.py @@ -28,7 +28,7 @@ def setUp(self): self.test_utils = UnitTestUtils(__file__) # use the local instance of the Epidata API - Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php' + Epidata.BASE_URL = 'http://delphi_web_epidata/epidata' Epidata.auth = ('epidata', 'key') # use the local instance of the epidata database diff --git a/integrations/acquisition/covidcast/test_covidcast_meta_caching.py b/integrations/acquisition/covidcast/test_covidcast_meta_caching.py index 6e4c6378f..f0a31518d 100644 --- a/integrations/acquisition/covidcast/test_covidcast_meta_caching.py +++ b/integrations/acquisition/covidcast/test_covidcast_meta_caching.py @@ -22,7 +22,7 @@ ) # use the local instance of the Epidata API -BASE_URL = 'http://delphi_web_epidata/epidata/api.php' +BASE_URL = 'http://delphi_web_epidata/epidata' class CovidcastMetaCacheTests(unittest.TestCase): @@ -69,8 +69,8 @@ def tearDown(self): @staticmethod def _make_request(): - params = {'endpoint': 'covidcast_meta', 'cached': 'true'} - response = requests.get(Epidata.BASE_URL, params=params, auth=Epidata.auth) + params = {'cached': 'true'} + response = requests.get(f"{Epidata.BASE_URL}/covidcast_meta", params=params, auth=Epidata.auth) response.raise_for_status() return response.json() diff --git a/integrations/acquisition/covidcast/test_csv_uploading.py b/integrations/acquisition/covidcast/test_csv_uploading.py index e4c9d881e..9d9157d21 100644 --- a/integrations/acquisition/covidcast/test_csv_uploading.py +++ b/integrations/acquisition/covidcast/test_csv_uploading.py @@ -56,7 +56,7 @@ def setUp(self): secrets.db.epi = ('user', 'pass') # use the local instance of the Epidata API - Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php' + Epidata.BASE_URL = 'http://delphi_web_epidata/epidata' Epidata.auth = ('epidata', 'key') def tearDown(self): @@ -119,7 +119,7 @@ def test_uploading(self): main(args) response = Epidata.covidcast('src-name', signal_name, 'day', 'state', 20200419, '*') - expected_values = pd.concat([values, pd.DataFrame({ "time_value": [20200419] * 3, "signal": [signal_name] * 3, "direction": [None] * 3})], axis=1).rename(columns=uploader_column_rename).to_dict(orient="records") + expected_values = pd.concat([values, pd.DataFrame({ "geo_type": "state", "source": "src-name", "time_type": "day", "time_value": [20200419] * 3, "signal": [signal_name] * 3, "direction": [None] * 3})], axis=1).rename(columns=uploader_column_rename).to_dict(orient="records") expected_response = {'result': 1, 'epidata': self.apply_lag(expected_values), 'message': 'success'} self.assertEqual(response, expected_response) @@ -148,6 +148,9 @@ def test_uploading(self): response = Epidata.covidcast('src-name', signal_name, 'day', 'state', 20200419, '*') expected_values = pd.concat([values, pd.DataFrame({ + "geo_type": "state", + "source": "src-name", + "time_type": "day", "time_value": [20200419] * 3, "signal": [signal_name] * 3, "direction": [None] * 3, @@ -181,7 +184,7 @@ def test_uploading(self): main(args) response = Epidata.covidcast('src-name', signal_name, 'day', 'state', 20200419, '*') - expected_response = {'result': -2, 'message': 'no results'} + expected_response = {'epidata': [], 'result': -2, 'message': 'no results'} self.assertEqual(response, expected_response) self.verify_timestamps_and_defaults() @@ -207,6 +210,9 @@ def test_uploading(self): response = Epidata.covidcast('src-name', signal_name, 'day', 'state', 20200419, '*') expected_values_df = pd.concat([values, pd.DataFrame({ + "geo_type": "state", + "source": "src-name", + "time_type": "day", "time_value": [20200419], "signal": [signal_name], "direction": [None]})], axis=1).rename(columns=uploader_column_rename) @@ -240,6 +246,9 @@ def test_uploading(self): response = Epidata.covidcast('src-name', signal_name, 'day', 'state', 20200419, '*') expected_values = pd.concat([values, pd.DataFrame({ + "geo_type": "state", + "source": "src-name", + "time_type": "day", "time_value": [20200419], "signal": [signal_name], "direction": [None] @@ -270,7 +279,7 @@ def test_uploading(self): main(args) response = Epidata.covidcast('src-name', signal_name, 'day', 'state', 20200419, '*') - expected_response = {'result': -2, 'message': 'no results'} + expected_response = {'epidata': [], 'result': -2, 'message': 'no results'} self.assertEqual(response, expected_response) self.verify_timestamps_and_defaults() diff --git a/integrations/acquisition/covidcast_nowcast/test_csv_uploading.py b/integrations/acquisition/covidcast_nowcast/test_csv_uploading.py index 1299c6144..bf1a0f9a0 100644 --- a/integrations/acquisition/covidcast_nowcast/test_csv_uploading.py +++ b/integrations/acquisition/covidcast_nowcast/test_csv_uploading.py @@ -55,7 +55,7 @@ def setUp(self): secrets.db.epi = ('user', 'pass') # use the local instance of the Epidata API - Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php' + Epidata.BASE_URL = 'http://delphi_web_epidata/epidata' Epidata.auth = ('epidata', 'key') def tearDown(self): diff --git a/integrations/client/test_delphi_epidata.py b/integrations/client/test_delphi_epidata.py index 4ef1fa6a3..7497890c2 100644 --- a/integrations/client/test_delphi_epidata.py +++ b/integrations/client/test_delphi_epidata.py @@ -24,9 +24,9 @@ def fake_epidata_endpoint(func): """This can be used as a decorator to enable a bogus Epidata endpoint to return 404 responses.""" def wrapper(*args): - Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/fake_api.php' + Epidata.BASE_URL = 'http://delphi_web_epidata/fake_epidata' func(*args) - Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php' + Epidata.BASE_URL = 'http://delphi_web_epidata/epidata' return wrapper class DelphiEpidataPythonClientTests(CovidcastBase): @@ -39,7 +39,7 @@ def localSetUp(self): self._db._cursor.execute('update covidcast_meta_cache set timestamp = 0, epidata = "[]"') # use the local instance of the Epidata API - Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php' + Epidata.BASE_URL = 'http://delphi_web_epidata/epidata' Epidata.auth = ('epidata', 'key') # use the local instance of the epidata database @@ -65,8 +65,8 @@ def test_covidcast(self): ) expected = [ - row_latest_issue.as_api_compatibility_row_dict(), - rows[-1].as_api_compatibility_row_dict() + row_latest_issue.as_api_row_dict(), + rows[-1].as_api_row_dict() ] self.assertEqual(response['epidata'], expected) @@ -85,10 +85,10 @@ def test_covidcast(self): expected = [{ rows[0].signal: [ - row_latest_issue.as_api_compatibility_row_dict(ignore_fields=['signal']), + row_latest_issue.as_api_row_dict(ignore_fields=['signal']), ], rows[-1].signal: [ - rows[-1].as_api_compatibility_row_dict(ignore_fields=['signal']), + rows[-1].as_api_row_dict(ignore_fields=['signal']), ], }] @@ -105,7 +105,7 @@ def test_covidcast(self): **self.params_from_row(rows[0]) ) - expected = [row_latest_issue.as_api_compatibility_row_dict()] + expected = [row_latest_issue.as_api_row_dict()] # check result self.assertEqual(response_1, { @@ -120,7 +120,7 @@ def test_covidcast(self): **self.params_from_row(rows[0], as_of=rows[1].issue) ) - expected = [rows[1].as_api_compatibility_row_dict()] + expected = [rows[1].as_api_row_dict()] # check result self.maxDiff=None @@ -137,8 +137,8 @@ def test_covidcast(self): ) expected = [ - rows[0].as_api_compatibility_row_dict(), - rows[1].as_api_compatibility_row_dict() + rows[0].as_api_row_dict(), + rows[1].as_api_row_dict() ] # check result @@ -154,7 +154,7 @@ def test_covidcast(self): **self.params_from_row(rows[0], lag=2) ) - expected = [row_latest_issue.as_api_compatibility_row_dict()] + expected = [row_latest_issue.as_api_row_dict()] # check result self.assertDictEqual(response_3, { @@ -170,7 +170,7 @@ def test_covidcast(self): ) # check result - self.assertEqual(response_1, {'message': 'no results', 'result': -2}) + self.assertEqual(response_1, {'epidata': [], 'message': 'no results', 'result': -2}) @patch('requests.post') @patch('requests.get') @@ -196,7 +196,7 @@ def test_retry_request(self, get): mock_response = MagicMock() mock_response.status_code = 200 get.side_effect = [JSONDecodeError('Expecting value', "", 0), mock_response] - response = Epidata._request(None) + response = Epidata._request(None, "") self.assertEqual(get.call_count, 2) self.assertEqual(response, mock_response.json()) @@ -207,7 +207,7 @@ def test_retry_request(self, get): get.side_effect = [JSONDecodeError('Expecting value', "", 0), JSONDecodeError('Expecting value', "", 0), mock_response] - response = Epidata._request(None) + response = Epidata._request(None, "") self.assertEqual(get.call_count, 2) # 2 from previous test + 2 from this one self.assertEqual(response, {'result': 0, 'message': 'error: Expecting value: line 1 column 1 (char 0)'} @@ -228,7 +228,7 @@ def test_geo_value(self): self._insert_rows(rows) counties = [ - rows[i].as_api_compatibility_row_dict() for i in range(N) + rows[i].as_api_row_dict() for i in range(N) ] def fetch(geo): @@ -336,9 +336,9 @@ def test_async_epidata(self): self._insert_rows(rows) test_output = Epidata.async_epidata([ - self.params_from_row(rows[0], source='covidcast'), - self.params_from_row(rows[1], source='covidcast') - ]*12, batch_size=10) + self.params_from_row(rows[0]), + self.params_from_row(rows[1]) + ]*12, 'covidcast', batch_size=10) responses = [i[0] for i in test_output] # check response is same as standard covidcast call, using 24 calls to test batch sizing self.assertEqual( @@ -354,7 +354,6 @@ def test_async_epidata_fail(self): with pytest.raises(ClientResponseError, match="404, message='NOT FOUND'"): Epidata.async_epidata([ { - 'source': 'covidcast', 'data_source': 'src', 'signals': 'sig', 'time_type': 'day', @@ -362,4 +361,4 @@ def test_async_epidata_fail(self): 'geo_value': '11111', 'time_values': '20200414' } - ]) + ], 'covidcast') diff --git a/integrations/client/test_nowcast.py b/integrations/client/test_nowcast.py index f5124e021..84fc0e080 100644 --- a/integrations/client/test_nowcast.py +++ b/integrations/client/test_nowcast.py @@ -39,7 +39,7 @@ def setUp(self): self.cur = cnx.cursor() # use the local instance of the Epidata API - Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php' + Epidata.BASE_URL = 'http://delphi_web_epidata/epidata' Epidata.auth = ('epidata', 'key') # use the local instance of the epidata database @@ -133,4 +133,4 @@ def test_covidcast_nowcast(self): response = Epidata.covidcast_nowcast( 'src', 'sig1', 'sensor', 'day', 'county', 22222222, '01001') - self.assertEqual(response, {'result': -2, 'message': 'no results'}) + self.assertEqual(response, {'epidata': [], 'result': -2, 'message': 'no results'}) diff --git a/integrations/server/test_covid_hosp.py b/integrations/server/test_covid_hosp.py index 37aa77363..100d961c4 100644 --- a/integrations/server/test_covid_hosp.py +++ b/integrations/server/test_covid_hosp.py @@ -16,7 +16,7 @@ def setUp(self): """Perform per-test setup.""" # use the local instance of the Epidata API - Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php' + Epidata.BASE_URL = 'http://delphi_web_epidata/epidata' Epidata.auth = ('epidata', 'key') # use the local instance of the epidata database diff --git a/integrations/server/test_covidcast.py b/integrations/server/test_covidcast.py index 73787d664..935d7badb 100644 --- a/integrations/server/test_covidcast.py +++ b/integrations/server/test_covidcast.py @@ -23,7 +23,7 @@ def localSetUp(self): def request_based_on_row(self, row: CovidcastTestRow, **kwargs): params = self.params_from_row(row, endpoint='covidcast', **kwargs) # use the local instance of the Epidata API - Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php' + Epidata.BASE_URL = 'http://delphi_web_epidata/epidata' Epidata.auth = ('epidata', 'key') response = Epidata.covidcast(**params) @@ -90,7 +90,7 @@ def test_round_trip(self): # make the request response = self.request_based_on_row(row) - expected = [row.as_api_compatibility_row_dict()] + expected = [row.as_api_row_dict()] self.assertEqual(response, { 'result': 1, @@ -158,11 +158,12 @@ def test_csv_format(self): # This is a hardcoded mess because of api.php. column_order = [ - "geo_value", "signal", "time_value", "direction", "issue", "lag", "missing_value", + "geo_value", "signal", "source", "geo_type", "time_type", + "time_value", "direction", "issue", "lag", "missing_value", "missing_stderr", "missing_sample_size", "value", "stderr", "sample_size" ] expected = ( - row.as_api_compatibility_row_df() + row.as_api_row_df() .assign(direction = None) .to_csv(columns=column_order, index=False) ) @@ -179,7 +180,7 @@ def test_raw_json_format(self): # make the request response = self.request_based_on_row(row, **{'format':'json'}) - expected = [row.as_api_compatibility_row_dict()] + expected = [row.as_api_row_dict()] # assert that the right data came back self.assertEqual(response, expected) @@ -191,13 +192,13 @@ def test_fields(self): row = self._insert_placeholder_set_one() # limit fields - response = self.request_based_on_row(row, **{"fields":"time_value,geo_value"}) + response = self.request_based_on_row(row, **{"fields":"time_value,geo_value,geo_type,source,time_type"}) - expected = row.as_api_compatibility_row_dict() + expected = row.as_api_row_dict() expected_all = { 'result': 1, 'epidata': [{ - k: expected[k] for k in ['time_value', 'geo_value'] + k: expected[k] for k in ['time_value', 'geo_value', 'geo_type', 'source', 'time_type'] }], 'message': 'success', } @@ -206,7 +207,7 @@ def test_fields(self): self.assertEqual(response, expected_all) # limit using invalid fields - response = self.request_based_on_row(row, fields='time_value,geo_value,doesnt_exist') + response = self.request_based_on_row(row, fields='time_value,geo_value,,geo_type,source,time_type,doesnt_exist') # assert that the right data came back (only valid fields) self.assertEqual(response, expected_all) @@ -226,7 +227,7 @@ def test_location_wildcard(self): # insert placeholder data rows = self._insert_placeholder_set_two() - expected = [row.as_api_compatibility_row_dict() for row in rows[:3]] + expected = [row.as_api_row_dict() for row in rows[:3]] # make the request response = self.request_based_on_row(rows[0], geo_value="*") @@ -243,7 +244,7 @@ def test_time_values_wildcard(self): # insert placeholder data rows = self._insert_placeholder_set_three() - expected = [row.as_api_compatibility_row_dict() for row in rows[:3]] + expected = [row.as_api_row_dict() for row in rows[:3]] # make the request response = self.request_based_on_row(rows[0], time_values="*") @@ -261,7 +262,7 @@ def test_issues_wildcard(self): # insert placeholder data rows = self._insert_placeholder_set_five() - expected = [row.as_api_compatibility_row_dict() for row in rows[:3]] + expected = [row.as_api_row_dict() for row in rows[:3]] # make the request response = self.request_based_on_row(rows[0], issues="*") @@ -279,7 +280,7 @@ def test_signal_wildcard(self): # insert placeholder data rows = self._insert_placeholder_set_four() - expected_signals = [row.as_api_compatibility_row_dict() for row in rows[:3]] + expected_signals = [row.as_api_row_dict() for row in rows[:3]] # make the request response = self.request_based_on_row(rows[0], signals="*") @@ -297,7 +298,7 @@ def test_geo_value(self): # insert placeholder data rows = self._insert_placeholder_set_two() - expected = [row.as_api_compatibility_row_dict() for row in rows[:3]] + expected = [row.as_api_row_dict() for row in rows[:3]] def fetch(geo_value): # make the request @@ -335,7 +336,7 @@ def test_location_timeline(self): # insert placeholder data rows = self._insert_placeholder_set_three() - expected_timeseries = [row.as_api_compatibility_row_dict() for row in rows[:3]] + expected_timeseries = [row.as_api_row_dict() for row in rows[:3]] # make the request response = self.request_based_on_row(rows[0], time_values='20000101-20000105') @@ -372,7 +373,7 @@ def test_nullable_columns(self): # make the request response = self.request_based_on_row(row) - expected = row.as_api_compatibility_row_dict() + expected = row.as_api_row_dict() # assert that the right data came back self.assertEqual(response, { @@ -393,7 +394,7 @@ def test_temporal_partitioning(self): # make the request response = self.request_based_on_row(rows[1], time_values="*") - expected = [rows[1].as_api_compatibility_row_dict()] + expected = [rows[1].as_api_row_dict()] # assert that the right data came back self.assertEqual(response, { diff --git a/integrations/server/test_covidcast_meta.py b/integrations/server/test_covidcast_meta.py index d03317c98..857422a41 100644 --- a/integrations/server/test_covidcast_meta.py +++ b/integrations/server/test_covidcast_meta.py @@ -14,7 +14,7 @@ import delphi.operations.secrets as secrets # use the local instance of the Epidata API -BASE_URL = 'http://delphi_web_epidata/epidata/api.php' +BASE_URL = 'http://delphi_web_epidata/epidata' AUTH = ('epidata', 'key') @@ -152,8 +152,7 @@ def _get_id(self): @staticmethod def _fetch(auth=AUTH, **kwargs): params = kwargs.copy() - params['endpoint'] = 'covidcast_meta' - response = requests.get(BASE_URL, params=params, auth=auth) + response = requests.get(f"{BASE_URL}/covidcast_meta", params=params, auth=auth) response.raise_for_status() return response.json() diff --git a/integrations/server/test_covidcast_nowcast.py b/integrations/server/test_covidcast_nowcast.py index 889d962dd..32445afdf 100644 --- a/integrations/server/test_covidcast_nowcast.py +++ b/integrations/server/test_covidcast_nowcast.py @@ -9,7 +9,7 @@ # use the local instance of the Epidata API -BASE_URL = 'http://delphi_web_epidata/epidata/api.php' +BASE_URL = 'http://delphi_web_epidata/epidata' AUTH = ('epidata', 'key') @@ -43,7 +43,7 @@ def tearDown(self): @staticmethod def _make_request(params: dict): - response = requests.get(BASE_URL, params=params, auth=AUTH) + response = requests.get(f"{BASE_URL}/covidcast_nowcast", params=params, auth=AUTH) response.raise_for_status() return response.json() @@ -59,7 +59,6 @@ def test_query(self): self.cnx.commit() # make the request with specified issue date params={ - 'source': 'covidcast_nowcast', 'data_source': 'src', 'signals': 'sig', 'sensor_names': 'sensor', diff --git a/integrations/server/test_fluview.py b/integrations/server/test_fluview.py index c192da637..48d9585fd 100644 --- a/integrations/server/test_fluview.py +++ b/integrations/server/test_fluview.py @@ -18,7 +18,7 @@ def setUpClass(cls): """Perform one-time setup.""" # use the local instance of the Epidata API - Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php' + Epidata.BASE_URL = 'http://delphi_web_epidata/epidata' Epidata.auth = ('epidata', 'key') def setUp(self): diff --git a/integrations/server/test_fluview_meta.py b/integrations/server/test_fluview_meta.py index 1e2cf73e3..6f81c1859 100644 --- a/integrations/server/test_fluview_meta.py +++ b/integrations/server/test_fluview_meta.py @@ -18,7 +18,7 @@ def setUpClass(cls): """Perform one-time setup.""" # use the local instance of the Epidata API - Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php' + Epidata.BASE_URL = 'http://delphi_web_epidata/epidata' Epidata.auth = ('epidata', 'key') def setUp(self): diff --git a/src/client/delphi_epidata.py b/src/client/delphi_epidata.py index fe8dbe51d..ae618e2b1 100644 --- a/src/client/delphi_epidata.py +++ b/src/client/delphi_epidata.py @@ -44,7 +44,7 @@ class Epidata: """An interface to DELPHI's Epidata API.""" # API base url - BASE_URL = "https://api.delphi.cmu.edu/epidata/api.php" + BASE_URL = "https://api.delphi.cmu.edu/epidata" auth = None client_version = _version @@ -68,17 +68,18 @@ def _list(values): @staticmethod @retry(reraise=True, stop=stop_after_attempt(2)) - def _request_with_retry(params): + def _request_with_retry(params, endpoint): """Make request with a retry if an exception is thrown.""" - req = requests.get(Epidata.BASE_URL, params, auth=Epidata.auth, headers=_HEADERS) + request_url = f"{Epidata.BASE_URL}/{endpoint}" + req = requests.get(request_url, params, auth=Epidata.auth, headers=_HEADERS) if req.status_code == 414: - req = requests.post(Epidata.BASE_URL, params, auth=Epidata.auth, headers=_HEADERS) + req = requests.post(request_url, params, auth=Epidata.auth, headers=_HEADERS) # handle 401 and 429 req.raise_for_status() return req @staticmethod - def _request(params): + def _request(params, endpoint): """Request and parse epidata. We default to GET since it has better caching and logging @@ -86,7 +87,7 @@ def _request(params): long and returns a 414. """ try: - result = Epidata._request_with_retry(params) + result = Epidata._request_with_retry(params, endpoint) except Exception as e: return {"result": 0, "message": "error: " + str(e)} if params is not None and "format" in params and params["format"] == "csv": @@ -125,7 +126,6 @@ def fluview(regions, epiweeks, issues=None, lag=None, auth=None): raise EpidataBadRequestException(ISSUES_LAG_EXCLUSIVE) # Set up request params = { - "endpoint": "fluview", "regions": Epidata._list(regions), "epiweeks": Epidata._list(epiweeks), } @@ -136,18 +136,13 @@ def fluview(regions, epiweeks, issues=None, lag=None, auth=None): if auth is not None: params["auth"] = auth # Make the API call - return Epidata._request(params) + return Epidata._request(params, "fluview") # Fetch FluView metadata @staticmethod def fluview_meta(): """Fetch FluView metadata.""" - # Set up request - params = { - "endpoint": "fluview_meta", - } - # Make the API call - return Epidata._request(params) + return Epidata._request({}, "fluview_meta") # Fetch FluView clinical data @staticmethod @@ -160,7 +155,6 @@ def fluview_clinical(regions, epiweeks, issues=None, lag=None): raise EpidataBadRequestException(REGIONS_EPIWEEKS_REQUIRED) # Set up request params = { - "endpoint": "fluview_clinical", "regions": Epidata._list(regions), "epiweeks": Epidata._list(epiweeks), } @@ -169,7 +163,7 @@ def fluview_clinical(regions, epiweeks, issues=None, lag=None): if lag is not None: params["lag"] = lag # Make the API call - return Epidata._request(params) + return Epidata._request(params, "fluview_clinical") # Fetch FluSurv data @staticmethod @@ -182,7 +176,6 @@ def flusurv(locations, epiweeks, issues=None, lag=None): raise EpidataBadRequestException(REGIONS_EPIWEEKS_REQUIRED) # Set up request params = { - "endpoint": "flusurv", "locations": Epidata._list(locations), "epiweeks": Epidata._list(epiweeks), } @@ -191,7 +184,7 @@ def flusurv(locations, epiweeks, issues=None, lag=None): if lag is not None: params["lag"] = lag # Make the API call - return Epidata._request(params) + return Epidata._request(params, "flusurv") # Fetch PAHO Dengue data @staticmethod @@ -204,7 +197,6 @@ def paho_dengue(regions, epiweeks, issues=None, lag=None): raise EpidataBadRequestException(REGIONS_EPIWEEKS_REQUIRED) # Set up request params = { - "endpoint": "paho_dengue", "regions": Epidata._list(regions), "epiweeks": Epidata._list(epiweeks), } @@ -213,7 +205,7 @@ def paho_dengue(regions, epiweeks, issues=None, lag=None): if lag is not None: params["lag"] = lag # Make the API call - return Epidata._request(params) + return Epidata._request(params, "paho_dengue") # Fetch ECDC ILI data @staticmethod @@ -226,7 +218,6 @@ def ecdc_ili(regions, epiweeks, issues=None, lag=None): raise EpidataBadRequestException(REGIONS_EPIWEEKS_REQUIRED) # Set up request params = { - "endpoint": "ecdc_ili", "regions": Epidata._list(regions), "epiweeks": Epidata._list(epiweeks), } @@ -235,7 +226,7 @@ def ecdc_ili(regions, epiweeks, issues=None, lag=None): if lag is not None: params["lag"] = lag # Make the API call - return Epidata._request(params) + return Epidata._request(params, "ecdc_ili") # Fetch KCDC ILI data @staticmethod @@ -248,7 +239,6 @@ def kcdc_ili(regions, epiweeks, issues=None, lag=None): raise EpidataBadRequestException(REGIONS_EPIWEEKS_REQUIRED) # Set up request params = { - "endpoint": "kcdc_ili", "regions": Epidata._list(regions), "epiweeks": Epidata._list(epiweeks), } @@ -257,7 +247,7 @@ def kcdc_ili(regions, epiweeks, issues=None, lag=None): if lag is not None: params["lag"] = lag # Make the API call - return Epidata._request(params) + return Epidata._request(params, "kcdc_ili") # Fetch Google Flu Trends data @staticmethod @@ -268,12 +258,11 @@ def gft(locations, epiweeks): raise EpidataBadRequestException(LOCATIONS_EPIWEEKS_REQUIRED) # Set up request params = { - "endpoint": "gft", "locations": Epidata._list(locations), "epiweeks": Epidata._list(epiweeks), } # Make the API call - return Epidata._request(params) + return Epidata._request(params, "gft") # Fetch Google Health Trends data @staticmethod @@ -286,14 +275,13 @@ def ght(auth, locations, epiweeks, query): ) # Set up request params = { - "endpoint": "ght", "auth": auth, "locations": Epidata._list(locations), "epiweeks": Epidata._list(epiweeks), "query": query, } # Make the API call - return Epidata._request(params) + return Epidata._request(params, "ght") # Fetch HealthTweets data @staticmethod @@ -306,7 +294,6 @@ def twitter(auth, locations, dates=None, epiweeks=None): raise EpidataBadRequestException("exactly one of `dates` and `epiweeks` is required") # Set up request params = { - "endpoint": "twitter", "auth": auth, "locations": Epidata._list(locations), } @@ -315,7 +302,7 @@ def twitter(auth, locations, dates=None, epiweeks=None): if epiweeks is not None: params["epiweeks"] = Epidata._list(epiweeks) # Make the API call - return Epidata._request(params) + return Epidata._request(params, "twitter") # Fetch Wikipedia access data @staticmethod @@ -328,7 +315,6 @@ def wiki(articles, dates=None, epiweeks=None, hours=None, language="en"): raise EpidataBadRequestException("exactly one of `dates` and `epiweeks` is required") # Set up request params = { - "endpoint": "wiki", "articles": Epidata._list(articles), "language": language, } @@ -339,7 +325,7 @@ def wiki(articles, dates=None, epiweeks=None, hours=None, language="en"): if hours is not None: params["hours"] = Epidata._list(hours) # Make the API call - return Epidata._request(params) + return Epidata._request(params, "wiki") # Fetch CDC page hits @staticmethod @@ -350,13 +336,12 @@ def cdc(auth, epiweeks, locations): raise EpidataBadRequestException("`auth`, `epiweeks`, and `locations` are all required") # Set up request params = { - "endpoint": "cdc", "auth": auth, "epiweeks": Epidata._list(epiweeks), "locations": Epidata._list(locations), } # Make the API call - return Epidata._request(params) + return Epidata._request(params, "cdc") # Fetch Quidel data @staticmethod @@ -367,13 +352,12 @@ def quidel(auth, epiweeks, locations): raise EpidataBadRequestException("`auth`, `epiweeks`, and `locations` are all required") # Set up request params = { - "endpoint": "quidel", "auth": auth, "epiweeks": Epidata._list(epiweeks), "locations": Epidata._list(locations), } # Make the API call - return Epidata._request(params) + return Epidata._request(params, "quidel") # Fetch NoroSTAT data (point data, no min/max) @staticmethod @@ -384,13 +368,12 @@ def norostat(auth, location, epiweeks): raise EpidataBadRequestException("`auth`, `location`, and `epiweeks` are all required") # Set up request params = { - "endpoint": "norostat", "auth": auth, "location": location, "epiweeks": Epidata._list(epiweeks), } # Make the API call - return Epidata._request(params) + return Epidata._request(params, "norostat") # Fetch NoroSTAT metadata @staticmethod @@ -401,11 +384,10 @@ def meta_norostat(auth): raise EpidataBadRequestException("`auth` is required") # Set up request params = { - "endpoint": "meta_norostat", "auth": auth, } # Make the API call - return Epidata._request(params) + return Epidata._request(params, "meta_norostat") # Fetch NIDSS flu data @staticmethod @@ -418,7 +400,6 @@ def nidss_flu(regions, epiweeks, issues=None, lag=None): raise EpidataBadRequestException(REGIONS_EPIWEEKS_REQUIRED) # Set up request params = { - "endpoint": "nidss_flu", "regions": Epidata._list(regions), "epiweeks": Epidata._list(epiweeks), } @@ -427,7 +408,7 @@ def nidss_flu(regions, epiweeks, issues=None, lag=None): if lag is not None: params["lag"] = lag # Make the API call - return Epidata._request(params) + return Epidata._request(params, "nidss_flu") # Fetch NIDSS dengue data @staticmethod @@ -438,12 +419,11 @@ def nidss_dengue(locations, epiweeks): raise EpidataBadRequestException(REGIONS_EPIWEEKS_REQUIRED) # Set up request params = { - "endpoint": "nidss_dengue", "locations": Epidata._list(locations), "epiweeks": Epidata._list(epiweeks), } # Make the API call - return Epidata._request(params) + return Epidata._request(params, "nidss_dengue") # Fetch Delphi's forecast @staticmethod @@ -454,12 +434,11 @@ def delphi(system, epiweek): raise EpidataBadRequestException("`system` and `epiweek` are both required") # Set up request params = { - "endpoint": "delphi", "system": system, "epiweek": epiweek, } # Make the API call - return Epidata._request(params) + return Epidata._request(params, "delphi") # Fetch Delphi's digital surveillance sensors @staticmethod @@ -472,7 +451,6 @@ def sensors(auth, names, locations, epiweeks): ) # Set up request params = { - "endpoint": "sensors", "names": Epidata._list(names), "locations": Epidata._list(locations), "epiweeks": Epidata._list(epiweeks), @@ -480,7 +458,7 @@ def sensors(auth, names, locations, epiweeks): if auth is not None: params["auth"] = auth # Make the API call - return Epidata._request(params) + return Epidata._request(params, "sensors") # Fetch Delphi's dengue digital surveillance sensors @staticmethod @@ -493,14 +471,13 @@ def dengue_sensors(auth, names, locations, epiweeks): ) # Set up request params = { - "endpoint": "dengue_sensors", "auth": auth, "names": Epidata._list(names), "locations": Epidata._list(locations), "epiweeks": Epidata._list(epiweeks), } # Make the API call - return Epidata._request(params) + return Epidata._request(params, "dengue_sensors") # Fetch Delphi's wILI nowcast @staticmethod @@ -511,12 +488,11 @@ def nowcast(locations, epiweeks): raise EpidataBadRequestException(REGIONS_EPIWEEKS_REQUIRED) # Set up request params = { - "endpoint": "nowcast", "locations": Epidata._list(locations), "epiweeks": Epidata._list(epiweeks), } # Make the API call - return Epidata._request(params) + return Epidata._request(params, "nowcast") # Fetch Delphi's dengue nowcast @staticmethod @@ -527,18 +503,17 @@ def dengue_nowcast(locations, epiweeks): raise EpidataBadRequestException(REGIONS_EPIWEEKS_REQUIRED) # Set up request params = { - "endpoint": "dengue_nowcast", "locations": Epidata._list(locations), "epiweeks": Epidata._list(epiweeks), } # Make the API call - return Epidata._request(params) + return Epidata._request(params, "dengue_nowcast") # Fetch API metadata @staticmethod def meta(): """Fetch API metadata.""" - return Epidata._request({"endpoint": "meta"}) + return Epidata._request({}, "meta") # Fetch Delphi's COVID-19 Surveillance Streams @staticmethod @@ -568,7 +543,6 @@ def covidcast( raise EpidataBadRequestException(REGIONS_EPIWEEKS_REQUIRED) # Set up request params = { - "endpoint": "covidcast", "data_source": data_source, "signals": Epidata._list(signals), "time_type": time_type, @@ -594,13 +568,13 @@ def covidcast( params["fields"] = kwargs["fields"] # Make the API call - return Epidata._request(params) + return Epidata._request(params, "covidcast") # Fetch Delphi's COVID-19 Surveillance Streams metadata @staticmethod def covidcast_meta(): """Fetch Delphi's COVID-19 Surveillance Streams metadata""" - return Epidata._request({"endpoint": "covidcast_meta"}) + return Epidata._request({}, "covidcast_meta") # Fetch COVID hospitalization data @staticmethod @@ -611,7 +585,6 @@ def covid_hosp(states, dates, issues=None, as_of=None): raise EpidataBadRequestException("`states` and `dates` are both required") # Set up request params = { - "endpoint": "covid_hosp", "states": Epidata._list(states), "dates": Epidata._list(dates), } @@ -620,7 +593,7 @@ def covid_hosp(states, dates, issues=None, as_of=None): if as_of is not None: params["as_of"] = as_of # Make the API call - return Epidata._request(params) + return Epidata._request(params, "covid_hosp_state_timeseries") # Fetch COVID hospitalization data for specific facilities @staticmethod @@ -633,21 +606,20 @@ def covid_hosp_facility(hospital_pks, collection_weeks, publication_dates=None): ) # Set up request params = { - "source": "covid_hosp_facility", "hospital_pks": Epidata._list(hospital_pks), "collection_weeks": Epidata._list(collection_weeks), } if publication_dates is not None: params["publication_dates"] = Epidata._list(publication_dates) # Make the API call - return Epidata._request(params) + return Epidata._request(params, "covid_hosp_facility") # Lookup COVID hospitalization facility identifiers @staticmethod def covid_hosp_facility_lookup(state=None, ccn=None, city=None, zip=None, fips_code=None): """Lookup COVID hospitalization facility identifiers.""" # Set up request - params = {"source": "covid_hosp_facility_lookup"} + params = {} if state is not None: params["state"] = state elif ccn is not None: @@ -663,7 +635,7 @@ def covid_hosp_facility_lookup(state=None, ccn=None, city=None, zip=None, fips_c "one of `state`, `ccn`, `city`, `zip`, or `fips_code` is required" ) # Make the API call - return Epidata._request(params) + return Epidata._request(params, "covid_hosp_facility_lookup") # Fetch Delphi's COVID-19 Nowcast sensors @staticmethod @@ -693,7 +665,6 @@ def covidcast_nowcast( raise EpidataBadRequestException(REGIONS_EPIWEEKS_REQUIRED) # Set up request params = { - "source": "covidcast_nowcast", "data_source": data_source, "signals": Epidata._list(signals), "sensor_names": Epidata._list(sensor_names), @@ -717,15 +688,17 @@ def covidcast_nowcast( params["format"] = kwargs["format"] # Make the API call - return Epidata._request(params) + return Epidata._request(params, "covidcast_nowcast") @staticmethod - def async_epidata(param_list, batch_size=50): + def async_epidata(param_list, endpoint, batch_size=50): """Make asynchronous Epidata calls for a list of parameters.""" + request_url = f"{Epidata.BASE_URL}/{endpoint}" + async def async_get(params, session): """Helper function to make Epidata GET requests.""" - async with session.get(Epidata.BASE_URL, params=params) as response: + async with session.get(request_url, params=params) as response: response.raise_for_status() return await response.json(), params From b5128f09779de310ccd775a8a83edb5205372291 Mon Sep 17 00:00:00 2001 From: Rostyslav Zatserkovnyi Date: Fri, 22 Sep 2023 11:54:16 +0300 Subject: [PATCH 02/55] Post review tweaks --- docs/new_endpoint_tutorial.md | 6 +- integrations/client/test_delphi_epidata.py | 12 ++-- integrations/server/test_covidcast.py | 5 +- .../server/test_covidcast_endpoints.py | 20 +++++- src/client/delphi_epidata.py | 66 +++++++++---------- src/maintenance/signal_dash_data_generator.py | 2 +- 6 files changed, 64 insertions(+), 47 deletions(-) diff --git a/docs/new_endpoint_tutorial.md b/docs/new_endpoint_tutorial.md index 6e6094161..abf8a5fa1 100644 --- a/docs/new_endpoint_tutorial.md +++ b/docs/new_endpoint_tutorial.md @@ -123,11 +123,9 @@ Here's what we add to each client: def fluview_meta(): """Fetch FluView metadata.""" # Set up request - params = { - 'endpoint': 'fluview_meta', - } + params = {} # Make the API call - return Epidata._request(params) + return Epidata._request("fluview_meta", params) ``` - [`delphi_epidata.R`](https://github.com/cmu-delphi/delphi-epidata/blob/dev/src/client/delphi_epidata.R) diff --git a/integrations/client/test_delphi_epidata.py b/integrations/client/test_delphi_epidata.py index 7497890c2..99473f0ac 100644 --- a/integrations/client/test_delphi_epidata.py +++ b/integrations/client/test_delphi_epidata.py @@ -196,7 +196,7 @@ def test_retry_request(self, get): mock_response = MagicMock() mock_response.status_code = 200 get.side_effect = [JSONDecodeError('Expecting value', "", 0), mock_response] - response = Epidata._request(None, "") + response = Epidata._request("") self.assertEqual(get.call_count, 2) self.assertEqual(response, mock_response.json()) @@ -207,7 +207,7 @@ def test_retry_request(self, get): get.side_effect = [JSONDecodeError('Expecting value', "", 0), JSONDecodeError('Expecting value', "", 0), mock_response] - response = Epidata._request(None, "") + response = Epidata._request("") self.assertEqual(get.call_count, 2) # 2 from previous test + 2 from this one self.assertEqual(response, {'result': 0, 'message': 'error: Expecting value: line 1 column 1 (char 0)'} @@ -335,10 +335,10 @@ def test_async_epidata(self): ] self._insert_rows(rows) - test_output = Epidata.async_epidata([ + test_output = Epidata.async_epidata('covidcast', [ self.params_from_row(rows[0]), self.params_from_row(rows[1]) - ]*12, 'covidcast', batch_size=10) + ]*12, batch_size=10) responses = [i[0] for i in test_output] # check response is same as standard covidcast call, using 24 calls to test batch sizing self.assertEqual( @@ -352,7 +352,7 @@ def test_async_epidata(self): @fake_epidata_endpoint def test_async_epidata_fail(self): with pytest.raises(ClientResponseError, match="404, message='NOT FOUND'"): - Epidata.async_epidata([ + Epidata.async_epidata('covidcast', [ { 'data_source': 'src', 'signals': 'sig', @@ -361,4 +361,4 @@ def test_async_epidata_fail(self): 'geo_value': '11111', 'time_values': '20200414' } - ], 'covidcast') + ]) diff --git a/integrations/server/test_covidcast.py b/integrations/server/test_covidcast.py index 935d7badb..4963ca9a8 100644 --- a/integrations/server/test_covidcast.py +++ b/integrations/server/test_covidcast.py @@ -156,7 +156,8 @@ def test_csv_format(self): **{'format':'csv'} ) - # This is a hardcoded mess because of api.php. + # This is a hardcoded mess because of the field ordering constructed here: + # https://github.com/cmu-delphi/delphi-epidata/blob/f7da6598a810be8df5374e3a71512c631c3a14f1/src/server/endpoints/covidcast.py#L83-L93 column_order = [ "geo_value", "signal", "source", "geo_type", "time_type", "time_value", "direction", "issue", "lag", "missing_value", @@ -207,7 +208,7 @@ def test_fields(self): self.assertEqual(response, expected_all) # limit using invalid fields - response = self.request_based_on_row(row, fields='time_value,geo_value,,geo_type,source,time_type,doesnt_exist') + response = self.request_based_on_row(row, fields='time_value,geo_value,geo_type,source,time_type,doesnt_exist') # assert that the right data came back (only valid fields) self.assertEqual(response, expected_all) diff --git a/integrations/server/test_covidcast_endpoints.py b/integrations/server/test_covidcast_endpoints.py index 3ba0af039..e79b6f32f 100644 --- a/integrations/server/test_covidcast_endpoints.py +++ b/integrations/server/test_covidcast_endpoints.py @@ -113,7 +113,25 @@ def test_compatibility(self): with self.subTest("simple"): out = self._fetch("/", signal=first.signal_pair(), geo=first.geo_pair(), time="day:*", is_compatibility=True) - self.assertEqual(len(out["epidata"]), len(rows)) + self.assertEqual(out["epidata"], [row.as_api_compatibility_row_dict() for row in rows]) + + def test_compatibility_restricted_source(self): + """Restricted request at the /api.php endpoint.""" + rows = [CovidcastTestRow.make_default_row(time_value=2020_04_01 + i, value=i, source="quidel") for i in range(10)] + first = rows[0] + self._insert_rows(rows) + + with self.subTest("no_roles"): + out = self._fetch("/", signal=first.signal_pair(), geo=first.geo_pair(), time="day:*", is_compatibility=True) + self.assertTrue("epidata" not in out) + + with self.subTest("no_api_key"): + out = self._fetch("/", auth=None, signal=first.signal_pair(), geo=first.geo_pair(), time="day:*", is_compatibility=True) + self.assertTrue("epidata" not in out) + + with self.subTest("quidel_role"): + out = self._fetch("/", auth=("epidata", "quidel_key"), signal=first.signal_pair(), geo=first.geo_pair(), time="day:*", is_compatibility=True) + self.assertEqual(out["epidata"], [row.as_api_compatibility_row_dict() for row in rows]) def test_trend(self): """Request a signal from the /trend endpoint.""" diff --git a/src/client/delphi_epidata.py b/src/client/delphi_epidata.py index ae618e2b1..e773bdc50 100644 --- a/src/client/delphi_epidata.py +++ b/src/client/delphi_epidata.py @@ -68,7 +68,7 @@ def _list(values): @staticmethod @retry(reraise=True, stop=stop_after_attempt(2)) - def _request_with_retry(params, endpoint): + def _request_with_retry(endpoint, params={}): """Make request with a retry if an exception is thrown.""" request_url = f"{Epidata.BASE_URL}/{endpoint}" req = requests.get(request_url, params, auth=Epidata.auth, headers=_HEADERS) @@ -79,7 +79,7 @@ def _request_with_retry(params, endpoint): return req @staticmethod - def _request(params, endpoint): + def _request(endpoint, params={}): """Request and parse epidata. We default to GET since it has better caching and logging @@ -87,7 +87,7 @@ def _request(params, endpoint): long and returns a 414. """ try: - result = Epidata._request_with_retry(params, endpoint) + result = Epidata._request_with_retry(endpoint, params) except Exception as e: return {"result": 0, "message": "error: " + str(e)} if params is not None and "format" in params and params["format"] == "csv": @@ -136,13 +136,13 @@ def fluview(regions, epiweeks, issues=None, lag=None, auth=None): if auth is not None: params["auth"] = auth # Make the API call - return Epidata._request(params, "fluview") + return Epidata._request("fluview", params) # Fetch FluView metadata @staticmethod def fluview_meta(): """Fetch FluView metadata.""" - return Epidata._request({}, "fluview_meta") + return Epidata._request("fluview_meta") # Fetch FluView clinical data @staticmethod @@ -163,7 +163,7 @@ def fluview_clinical(regions, epiweeks, issues=None, lag=None): if lag is not None: params["lag"] = lag # Make the API call - return Epidata._request(params, "fluview_clinical") + return Epidata._request("fluview_clinical", params) # Fetch FluSurv data @staticmethod @@ -184,7 +184,7 @@ def flusurv(locations, epiweeks, issues=None, lag=None): if lag is not None: params["lag"] = lag # Make the API call - return Epidata._request(params, "flusurv") + return Epidata._request("flusurv", params) # Fetch PAHO Dengue data @staticmethod @@ -205,7 +205,7 @@ def paho_dengue(regions, epiweeks, issues=None, lag=None): if lag is not None: params["lag"] = lag # Make the API call - return Epidata._request(params, "paho_dengue") + return Epidata._request("paho_dengue", params) # Fetch ECDC ILI data @staticmethod @@ -226,7 +226,7 @@ def ecdc_ili(regions, epiweeks, issues=None, lag=None): if lag is not None: params["lag"] = lag # Make the API call - return Epidata._request(params, "ecdc_ili") + return Epidata._request("ecdc_ili", params) # Fetch KCDC ILI data @staticmethod @@ -247,7 +247,7 @@ def kcdc_ili(regions, epiweeks, issues=None, lag=None): if lag is not None: params["lag"] = lag # Make the API call - return Epidata._request(params, "kcdc_ili") + return Epidata._request("kcdc_ili", params) # Fetch Google Flu Trends data @staticmethod @@ -262,7 +262,7 @@ def gft(locations, epiweeks): "epiweeks": Epidata._list(epiweeks), } # Make the API call - return Epidata._request(params, "gft") + return Epidata._request("gft", params) # Fetch Google Health Trends data @staticmethod @@ -281,7 +281,7 @@ def ght(auth, locations, epiweeks, query): "query": query, } # Make the API call - return Epidata._request(params, "ght") + return Epidata._request("ght", params) # Fetch HealthTweets data @staticmethod @@ -302,7 +302,7 @@ def twitter(auth, locations, dates=None, epiweeks=None): if epiweeks is not None: params["epiweeks"] = Epidata._list(epiweeks) # Make the API call - return Epidata._request(params, "twitter") + return Epidata._request("twitter", params) # Fetch Wikipedia access data @staticmethod @@ -325,7 +325,7 @@ def wiki(articles, dates=None, epiweeks=None, hours=None, language="en"): if hours is not None: params["hours"] = Epidata._list(hours) # Make the API call - return Epidata._request(params, "wiki") + return Epidata._request("wiki", params) # Fetch CDC page hits @staticmethod @@ -341,7 +341,7 @@ def cdc(auth, epiweeks, locations): "locations": Epidata._list(locations), } # Make the API call - return Epidata._request(params, "cdc") + return Epidata._request("cdc", params) # Fetch Quidel data @staticmethod @@ -357,7 +357,7 @@ def quidel(auth, epiweeks, locations): "locations": Epidata._list(locations), } # Make the API call - return Epidata._request(params, "quidel") + return Epidata._request("quidel", params) # Fetch NoroSTAT data (point data, no min/max) @staticmethod @@ -373,7 +373,7 @@ def norostat(auth, location, epiweeks): "epiweeks": Epidata._list(epiweeks), } # Make the API call - return Epidata._request(params, "norostat") + return Epidata._request("norostat", params) # Fetch NoroSTAT metadata @staticmethod @@ -387,7 +387,7 @@ def meta_norostat(auth): "auth": auth, } # Make the API call - return Epidata._request(params, "meta_norostat") + return Epidata._request("meta_norostat", params) # Fetch NIDSS flu data @staticmethod @@ -408,7 +408,7 @@ def nidss_flu(regions, epiweeks, issues=None, lag=None): if lag is not None: params["lag"] = lag # Make the API call - return Epidata._request(params, "nidss_flu") + return Epidata._request("nidss_flu", params) # Fetch NIDSS dengue data @staticmethod @@ -423,7 +423,7 @@ def nidss_dengue(locations, epiweeks): "epiweeks": Epidata._list(epiweeks), } # Make the API call - return Epidata._request(params, "nidss_dengue") + return Epidata._request("nidss_dengue", params) # Fetch Delphi's forecast @staticmethod @@ -438,7 +438,7 @@ def delphi(system, epiweek): "epiweek": epiweek, } # Make the API call - return Epidata._request(params, "delphi") + return Epidata._request("delphi", params) # Fetch Delphi's digital surveillance sensors @staticmethod @@ -458,7 +458,7 @@ def sensors(auth, names, locations, epiweeks): if auth is not None: params["auth"] = auth # Make the API call - return Epidata._request(params, "sensors") + return Epidata._request("sensors", params) # Fetch Delphi's dengue digital surveillance sensors @staticmethod @@ -477,7 +477,7 @@ def dengue_sensors(auth, names, locations, epiweeks): "epiweeks": Epidata._list(epiweeks), } # Make the API call - return Epidata._request(params, "dengue_sensors") + return Epidata._request("dengue_sensors", params) # Fetch Delphi's wILI nowcast @staticmethod @@ -492,7 +492,7 @@ def nowcast(locations, epiweeks): "epiweeks": Epidata._list(epiweeks), } # Make the API call - return Epidata._request(params, "nowcast") + return Epidata._request("nowcast", params) # Fetch Delphi's dengue nowcast @staticmethod @@ -507,13 +507,13 @@ def dengue_nowcast(locations, epiweeks): "epiweeks": Epidata._list(epiweeks), } # Make the API call - return Epidata._request(params, "dengue_nowcast") + return Epidata._request("dengue_nowcast", params) # Fetch API metadata @staticmethod def meta(): """Fetch API metadata.""" - return Epidata._request({}, "meta") + return Epidata._request("meta") # Fetch Delphi's COVID-19 Surveillance Streams @staticmethod @@ -568,13 +568,13 @@ def covidcast( params["fields"] = kwargs["fields"] # Make the API call - return Epidata._request(params, "covidcast") + return Epidata._request("covidcast", params) # Fetch Delphi's COVID-19 Surveillance Streams metadata @staticmethod def covidcast_meta(): """Fetch Delphi's COVID-19 Surveillance Streams metadata""" - return Epidata._request({}, "covidcast_meta") + return Epidata._request("covidcast_meta") # Fetch COVID hospitalization data @staticmethod @@ -593,7 +593,7 @@ def covid_hosp(states, dates, issues=None, as_of=None): if as_of is not None: params["as_of"] = as_of # Make the API call - return Epidata._request(params, "covid_hosp_state_timeseries") + return Epidata._request("covid_hosp_state_timeseries", params) # Fetch COVID hospitalization data for specific facilities @staticmethod @@ -612,7 +612,7 @@ def covid_hosp_facility(hospital_pks, collection_weeks, publication_dates=None): if publication_dates is not None: params["publication_dates"] = Epidata._list(publication_dates) # Make the API call - return Epidata._request(params, "covid_hosp_facility") + return Epidata._request("covid_hosp_facility", params) # Lookup COVID hospitalization facility identifiers @staticmethod @@ -635,7 +635,7 @@ def covid_hosp_facility_lookup(state=None, ccn=None, city=None, zip=None, fips_c "one of `state`, `ccn`, `city`, `zip`, or `fips_code` is required" ) # Make the API call - return Epidata._request(params, "covid_hosp_facility_lookup") + return Epidata._request("covid_hosp_facility_lookup", params) # Fetch Delphi's COVID-19 Nowcast sensors @staticmethod @@ -688,10 +688,10 @@ def covidcast_nowcast( params["format"] = kwargs["format"] # Make the API call - return Epidata._request(params, "covidcast_nowcast") + return Epidata._request("covidcast_nowcast", params) @staticmethod - def async_epidata(param_list, endpoint, batch_size=50): + def async_epidata(endpoint, param_list, batch_size=50): """Make asynchronous Epidata calls for a list of parameters.""" request_url = f"{Epidata.BASE_URL}/{endpoint}" diff --git a/src/maintenance/signal_dash_data_generator.py b/src/maintenance/signal_dash_data_generator.py index 6eea06579..b7f1048f5 100644 --- a/src/maintenance/signal_dash_data_generator.py +++ b/src/maintenance/signal_dash_data_generator.py @@ -19,7 +19,7 @@ LOOKBACK_DAYS_FOR_COVERAGE = 56 -BASE_COVIDCAST = covidcast.covidcast.Epidata.BASE_URL[:-len("api.php")] + "covidcast" +BASE_COVIDCAST = covidcast.covidcast.Epidata.BASE_URL + "/covidcast" COVERAGE_URL = f"{BASE_COVIDCAST}/coverage?format=csv&signal={{source}}:{{signal}}&days={LOOKBACK_DAYS_FOR_COVERAGE}" @dataclass From c65095e43bf521ae37936c5bae8863e8c7a02ced Mon Sep 17 00:00:00 2001 From: george haff Date: Fri, 22 Sep 2023 21:15:55 -0400 Subject: [PATCH 03/55] removed 'api.php' from comments where not appropriate --- tests/server/dev_test_granular_sensor_authentication.py | 2 +- tests/server/endpoints/test_nidss_flu.py | 2 -- tests/server/test_exceptions.py | 2 -- tests/server/test_query.py | 2 -- tests/server/test_security.py | 2 +- tests/server/test_validate.py | 2 -- 6 files changed, 2 insertions(+), 10 deletions(-) diff --git a/tests/server/dev_test_granular_sensor_authentication.py b/tests/server/dev_test_granular_sensor_authentication.py index bc742392a..2309c626e 100644 --- a/tests/server/dev_test_granular_sensor_authentication.py +++ b/tests/server/dev_test_granular_sensor_authentication.py @@ -1,4 +1,4 @@ -"""Unit tests for granular sensor authentication in api.php.""" +"""Unit tests for granular sensor authentication.""" # standard library import unittest diff --git a/tests/server/endpoints/test_nidss_flu.py b/tests/server/endpoints/test_nidss_flu.py index bc0723bf3..7ec50a6d2 100644 --- a/tests/server/endpoints/test_nidss_flu.py +++ b/tests/server/endpoints/test_nidss_flu.py @@ -1,5 +1,3 @@ -"""Unit tests for granular sensor authentication in api.php.""" - # standard library import unittest import base64 diff --git a/tests/server/test_exceptions.py b/tests/server/test_exceptions.py index 94cdc34f1..b07fe3ac7 100644 --- a/tests/server/test_exceptions.py +++ b/tests/server/test_exceptions.py @@ -1,5 +1,3 @@ -"""Unit tests for granular sensor authentication in api.php.""" - # standard library import unittest diff --git a/tests/server/test_query.py b/tests/server/test_query.py index ec07d3e8b..95b21a55a 100644 --- a/tests/server/test_query.py +++ b/tests/server/test_query.py @@ -1,5 +1,3 @@ -"""Unit tests for granular sensor authentication in api.php.""" - # standard library import unittest import base64 diff --git a/tests/server/test_security.py b/tests/server/test_security.py index e209d9342..eb8e32700 100644 --- a/tests/server/test_security.py +++ b/tests/server/test_security.py @@ -1,4 +1,4 @@ -"""Unit tests for granular sensor authentication in api.php.""" +"""Unit tests for granular sensor authentication.""" # standard library import unittest diff --git a/tests/server/test_validate.py b/tests/server/test_validate.py index 27ce28672..f06e9e997 100644 --- a/tests/server/test_validate.py +++ b/tests/server/test_validate.py @@ -1,5 +1,3 @@ -"""Unit tests for granular sensor authentication in api.php.""" - # standard library import unittest From 05e2b843c168bb52b1363ab0221b76b173436723 Mon Sep 17 00:00:00 2001 From: Rostyslav Zatserkovnyi Date: Wed, 27 Sep 2023 18:00:16 +0300 Subject: [PATCH 04/55] Refactor? --- integrations/server/test_api_keys.py | 53 +++++++------------- integrations/server/test_signal_dashboard.py | 10 +--- src/common/integration_test_base_class.py | 2 +- 3 files changed, 22 insertions(+), 43 deletions(-) diff --git a/integrations/server/test_api_keys.py b/integrations/server/test_api_keys.py index 418e265ac..0c5be4c0b 100644 --- a/integrations/server/test_api_keys.py +++ b/integrations/server/test_api_keys.py @@ -11,9 +11,8 @@ class APIKeysTets(DelphiTestBase): def localSetUp(self): self.role_name = "cdc" - def _make_request(self, url: str = None, params: dict = {}, auth: tuple = None): - if not url: - url = self.epidata_client.BASE_URL + def _make_request(self, endpoint: str, params: dict = {}, auth: tuple = None): + url = f"{self.epidata_client.BASE_URL}/{endpoint}" response = requests.get(url, params=params, auth=auth) return response @@ -22,13 +21,12 @@ def test_public_route(self): public_route = "http://delphi_web_epidata/epidata/version" status_codes = set() for _ in range(10): - status_codes.add(self._make_request(public_route).status_code) + status_codes.add(self._make_request("version", public_route).status_code) self.assertEqual(status_codes, {200}) def test_no_multiples_data_source(self): """Test requests with no multiples and with provided `data_source` and `signal` as a separate query params.""" params = { - "source": "covidcast", "data_source": "fb-survey", "signal": "smoothed_wcli", "time_type": "day", @@ -38,13 +36,12 @@ def test_no_multiples_data_source(self): } status_codes = set() for _ in range(10): - status_codes.add(self._make_request(params=params).status_code) + status_codes.add(self._make_request("covidcast", params=params).status_code) self.assertEqual(status_codes, {200}) def test_no_multiples_source_signal(self): """Test requests with colon-delimited source-signal param presentation.""" params = { - "source": "covidcast", "signal": "fb-survey:smoothed_wcli", "time_type": "day", "geo_type": "state", @@ -53,13 +50,12 @@ def test_no_multiples_source_signal(self): } status_codes = set() for _ in range(10): - status_codes.add(self._make_request(params=params).status_code) + status_codes.add(self._make_request("covidcast", params=params).status_code) self.assertEqual(status_codes, {200}) def test_multiples_allowed_signal_two_multiples(self): """Test requests with 2 multiples and allowed dashboard signal""" params = { - "source": "covidcast", "signal": "fb-survey:smoothed_wcli", "time_type": "day", "geo_type": "state", @@ -68,13 +64,12 @@ def test_multiples_allowed_signal_two_multiples(self): } status_codes = set() for _ in range(10): - status_codes.add(self._make_request(params=params).status_code) + status_codes.add(self._make_request("covidcast", params=params).status_code) self.assertEqual(status_codes, {200}) def test_multiples_non_allowed_signal(self): """Test requests with 2 multiples and non-allowed dashboard signal""" params = { - "source": "covidcast", "signal": "hospital-admissions:smoothed_adj_covid19_from_claims", "time_type": "day", "geo_type": "state", @@ -83,13 +78,12 @@ def test_multiples_non_allowed_signal(self): } status_codes = set() for _ in range(10): - status_codes.add(self._make_request(params=params).status_code) + status_codes.add(self._make_request("covidcast", params=params).status_code) self.assertEqual(status_codes, {200, 429}) def test_multiples_mixed_allowed_signal_two_multiples(self): """Test requests with 2 multiples and mixed-allowed dashboard signal""" params = { - "source": "covidcast", "signal": "fb-survey:smoothed_wcli,hospital-admissions:smoothed_adj_covid19_from_claims", "time_type": "day", "geo_type": "state", @@ -98,13 +92,12 @@ def test_multiples_mixed_allowed_signal_two_multiples(self): } status_codes = set() for _ in range(10): - status_codes.add(self._make_request(params=params).status_code) + status_codes.add(self._make_request("covidcast", params=params).status_code) self.assertEqual(status_codes, {200, 429}) def test_multiples_allowed_signal_three_multiples(self): """Test requests with 3 multiples and allowed dashboard signal""" params = { - "source": "covidcast", "signal": "fb-survey:smoothed_wcli,fb-survey:smoothed_wcli", "time_type": "day", "geo_type": "state", @@ -113,13 +106,12 @@ def test_multiples_allowed_signal_three_multiples(self): } status_codes = set() for _ in range(10): - status_codes.add(self._make_request(params=params).status_code) + status_codes.add(self._make_request("covidcast", params=params).status_code) self.assertEqual(status_codes, {401}) def test_multiples_mixed_allowed_signal_three_multiples(self): """Test requests with 3 multiples and mixed-allowed dashboard signal""" params = { - "source": "covidcast", "signal": "fb-survey:smoothed_wcli,fb-survey:smoothed_wcli1", "time_type": "day", "geo_type": "state", @@ -128,13 +120,12 @@ def test_multiples_mixed_allowed_signal_three_multiples(self): } status_codes = set() for _ in range(10): - status_codes.add(self._make_request(params=params).status_code) + status_codes.add(self._make_request("covidcast", params=params).status_code) self.assertEqual(status_codes, {401}) def test_multiples_mixed_allowed_signal_api_key(self): """Test requests with 3 multiples and mixed-allowed dashboard signal + valid API Key""" params = { - "source": "covidcast", "signal": "fb-survey:smoothed_wcli,fb-survey:smoothed_wcli1", "time_type": "day", "geo_type": "state", @@ -144,14 +135,13 @@ def test_multiples_mixed_allowed_signal_api_key(self): status_codes = set() for _ in range(10): status_codes.add( - self._make_request(params=params, auth=self.epidata_client.auth).status_code + self._make_request("covidcast", params=params, auth=self.epidata_client.auth).status_code ) self.assertEqual(status_codes, {200}) def test_multiples_allowed_signal_api_key(self): """Test requests with 3 multiples and allowed dashboard signal + valid API Key""" params = { - "source": "covidcast", "signal": "fb-survey:smoothed_wcli,fb-survey:smoothed_wcli", "time_type": "day", "geo_type": "state", @@ -161,14 +151,13 @@ def test_multiples_allowed_signal_api_key(self): status_codes = set() for _ in range(10): status_codes.add( - self._make_request(params=params, auth=self.epidata_client.auth).status_code + self._make_request("covidcast", params=params, auth=self.epidata_client.auth).status_code ) self.assertEqual(status_codes, {200}) def test_no_multiples_allowed_signal_api_key(self): """Test requests with no multiples and allowed dashboard signal + valid API Key""" params = { - "source": "covidcast", "signal": "fb-survey:smoothed_wcli", "time_type": "day", "geo_type": "state", @@ -178,14 +167,13 @@ def test_no_multiples_allowed_signal_api_key(self): status_codes = set() for _ in range(10): status_codes.add( - self._make_request(params=params, auth=self.epidata_client.auth).status_code + self._make_request("covidcast", params=params, auth=self.epidata_client.auth).status_code ) self.assertEqual(status_codes, {200}) def test_no_multiples_allowed_signal_bad_api_key(self): """Test requests with no multiples and allowed dashboard signal + bad API Key""" params = { - "source": "covidcast", "signal": "fb-survey:smoothed_wcli", "time_type": "day", "geo_type": "state", @@ -196,54 +184,51 @@ def test_no_multiples_allowed_signal_bad_api_key(self): for _ in range(10): status_codes.add( self._make_request( - params=params, auth=("bad_key", "bad_email") + "covidcast", params=params, auth=("bad_key", "bad_email") ).status_code ) self.assertEqual(status_codes, {200}) def test_restricted_endpoint_no_key(self): """Test restricted endpoint with no auth key""" - params = {"source": "cdc", "regions": "1as", "epiweeks": "202020"} + params = {"regions": "1as", "epiweeks": "202020"} status_codes = set() for _ in range(10): - status_codes.add(self._make_request(params=params).status_code) + status_codes.add(self._make_request("cdc", params=params).status_code) self.assertEqual(status_codes, {401}) def test_restricted_endpoint_invalid_key(self): """Test restricted endpoint with invalid auth key""" params = { - "source": "cdc", "regions": "1as", "epiweeks": "202020", "auth": "invalid_key", } status_codes = set() for _ in range(10): - status_codes.add(self._make_request(params=params).status_code) + status_codes.add(self._make_request("cdc", params=params).status_code) self.assertEqual(status_codes, {401}) def test_restricted_endpoint_no_roles_key(self): """Test restricted endpoint with no roles key""" params = { - "source": "cdc", "regions": "1as", "epiweeks": "202020", "auth": "key", } status_codes = set() for _ in range(10): - status_codes.add(self._make_request(params=params).status_code) + status_codes.add(self._make_request("cdc", params=params).status_code) self.assertEqual(status_codes, {401}) def test_restricted_endpoint_valid_roles_key(self): """Test restricted endpoint with valid auth key with required role""" params = { - "source": "cdc", "regions": "1as", "epiweeks": "202020", "auth": "cdc_key", } status_codes = set() for _ in range(10): - status_codes.add(self._make_request(params=params).status_code) + status_codes.add(self._make_request("cdc", params=params).status_code) self.assertEqual(status_codes, {200}) diff --git a/integrations/server/test_signal_dashboard.py b/integrations/server/test_signal_dashboard.py index e99a464fd..1aed3eec6 100644 --- a/integrations/server/test_signal_dashboard.py +++ b/integrations/server/test_signal_dashboard.py @@ -25,10 +25,7 @@ def setUp(self) -> None: def test_signal_dashboard_coverage(self): """Basic integration test for signal_dashboard_coverage endpoint""" - params = { - "endpoint": "signal_dashboard_coverage", - } - response = self.epidata_client._request(params=params) + response = self.epidata_client._request("signal_dashboard_coverage") self.assertEqual( response, { @@ -41,10 +38,7 @@ def test_signal_dashboard_coverage(self): def test_signal_dashboard_status(self): """Basic integration test for signal_dashboard_status endpoint""" - params = { - "endpoint": "signal_dashboard_status", - } - response = self.epidata_client._request(params=params) + response = self.epidata_client._request("signal_dashboard_status") self.assertEqual( response, { diff --git a/src/common/integration_test_base_class.py b/src/common/integration_test_base_class.py index 387c0f617..47c9f68e4 100644 --- a/src/common/integration_test_base_class.py +++ b/src/common/integration_test_base_class.py @@ -19,7 +19,7 @@ def __init__(self, methodName: str = "runTest") -> None: self.create_tables_list = [] self.role_name = None self.epidata_client = Epidata - self.epidata_client.BASE_URL = "http://delphi_web_epidata/epidata/api.php" + self.epidata_client.BASE_URL = "http://delphi_web_epidata/epidata" self.epidata_client.auth = ("epidata", "key") def create_key_with_role(self, cur, role_name: str): From 6e7190f96bbf02b7fcd4aca0687fd8466a934840 Mon Sep 17 00:00:00 2001 From: Rostyslav Zatserkovnyi Date: Wed, 27 Sep 2023 19:03:29 +0300 Subject: [PATCH 05/55] Add pypi changelog --- src/client/packaging/pypi/CHANGELOG.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 src/client/packaging/pypi/CHANGELOG.md diff --git a/src/client/packaging/pypi/CHANGELOG.md b/src/client/packaging/pypi/CHANGELOG.md new file mode 100644 index 000000000..c9853bba8 --- /dev/null +++ b/src/client/packaging/pypi/CHANGELOG.md @@ -0,0 +1,19 @@ +# Change Log +All notable changes to this client will be documented in this file. + +The format is based on [Keep a Changelog](http://keepachangelog.com/) +and this project adheres to [Semantic Versioning](http://semver.org/). + +## [4.2.0] - 2023-09-27 + +### Added + +### Changed + +- Modify the signatures for several methods: endpoint no longer needs to be specified in the params dict under the "source" key, but becomes a mandatory parameter for the method: + - `_request(params)` → `_request(endpoint, params={})` + - `_request_with_retry(params)` → `_request_with_retry(endpoint, params={})` + - `async_epidata(param_list, batch_size=50)` → `async_epidata(endpoint, param_list, batch_size=50)` + +### Fixed + \ No newline at end of file From 4513ced99f69557fc6227e24e87a9fa25a74fac4 Mon Sep 17 00:00:00 2001 From: Rostyslav Zatserkovnyi Date: Wed, 27 Sep 2023 19:03:40 +0300 Subject: [PATCH 06/55] Extra test for compat endpoint --- integrations/server/test_covidcast_endpoints.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/integrations/server/test_covidcast_endpoints.py b/integrations/server/test_covidcast_endpoints.py index e79b6f32f..2b74004bf 100644 --- a/integrations/server/test_covidcast_endpoints.py +++ b/integrations/server/test_covidcast_endpoints.py @@ -115,6 +115,12 @@ def test_compatibility(self): out = self._fetch("/", signal=first.signal_pair(), geo=first.geo_pair(), time="day:*", is_compatibility=True) self.assertEqual(out["epidata"], [row.as_api_compatibility_row_dict() for row in rows]) + with self.subTest("same contents sans excluded columns"): + compat = self._fetch("/", signal=first.signal_pair(), geo=first.geo_pair(), time="day:*", is_compatibility=True) + regular = self._fetch("/", signal=first.signal_pair(), geo=first.geo_pair(), time="day:*") + regular_sans_excluded = [{k: v for k, v in row.items() if k not in ["source", "geo_type", "time_type"]} for row in regular["epidata"]] + self.assertEqual(compat["epidata"], regular_sans_excluded) + def test_compatibility_restricted_source(self): """Restricted request at the /api.php endpoint.""" rows = [CovidcastTestRow.make_default_row(time_value=2020_04_01 + i, value=i, source="quidel") for i in range(10)] From 832b25ccaec86650fae5954acbea4b7a73945a9f Mon Sep 17 00:00:00 2001 From: Rostyslav Zatserkovnyi Date: Wed, 27 Sep 2023 19:07:08 +0300 Subject: [PATCH 07/55] Use del --- integrations/server/test_covidcast_endpoints.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/integrations/server/test_covidcast_endpoints.py b/integrations/server/test_covidcast_endpoints.py index 2b74004bf..206ed3c54 100644 --- a/integrations/server/test_covidcast_endpoints.py +++ b/integrations/server/test_covidcast_endpoints.py @@ -118,8 +118,11 @@ def test_compatibility(self): with self.subTest("same contents sans excluded columns"): compat = self._fetch("/", signal=first.signal_pair(), geo=first.geo_pair(), time="day:*", is_compatibility=True) regular = self._fetch("/", signal=first.signal_pair(), geo=first.geo_pair(), time="day:*") - regular_sans_excluded = [{k: v for k, v in row.items() if k not in ["source", "geo_type", "time_type"]} for row in regular["epidata"]] - self.assertEqual(compat["epidata"], regular_sans_excluded) + # Remove keys from the regular row which are excluded in the compat rows + for row in regular['epidata']: + for key in ['source', 'geo_type', 'time_type']: + del row[key] + self.assertEqual(compat, regular) def test_compatibility_restricted_source(self): """Restricted request at the /api.php endpoint.""" From d5363e3cc1901d6e1c0fe3f70c5466f046e71168 Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Thu, 20 Jul 2023 18:03:24 +0300 Subject: [PATCH 08/55] Added basic integration tests for all endpoints --- integrations/server/test_cdc.py | 86 ++++++++++++ integrations/server/test_delphi.py | 97 +++++++++++++ integrations/server/test_dengue_nowcast.py | 84 ++++++++++++ integrations/server/test_dengue_sensors.py | 90 +++++++++++++ integrations/server/test_ecdc_ili.py | 75 +++++++++++ integrations/server/test_flusurv.py | 80 +++++++++++ integrations/server/test_fluview_clinical.py | 80 +++++++++++ integrations/server/test_gft.py | 62 +++++++++ integrations/server/test_ght.py | 67 +++++++++ integrations/server/test_kcdc_ili.py | 75 +++++++++++ integrations/server/test_meta.py | 72 ++++++++++ integrations/server/test_meta_norostat.py | 135 +++++++++++++++++++ integrations/server/test_nidss_dengue.py | 62 +++++++++ integrations/server/test_nidss_flu.py | 76 +++++++++++ integrations/server/test_norostat.py | 93 +++++++++++++ integrations/server/test_nowcast.py | 66 +++++++++ integrations/server/test_paho_dengue.py | 80 +++++++++++ integrations/server/test_quidel.py | 67 +++++++++ integrations/server/test_sensors.py | 71 ++++++++++ integrations/server/test_signal_dashboard.py | 102 ++++++++++++++ integrations/server/test_twitter.py | 71 ++++++++++ integrations/server/test_wiki.py | 72 ++++++++++ 22 files changed, 1763 insertions(+) create mode 100644 integrations/server/test_cdc.py create mode 100644 integrations/server/test_delphi.py create mode 100644 integrations/server/test_dengue_nowcast.py create mode 100644 integrations/server/test_dengue_sensors.py create mode 100644 integrations/server/test_ecdc_ili.py create mode 100644 integrations/server/test_flusurv.py create mode 100644 integrations/server/test_fluview_clinical.py create mode 100644 integrations/server/test_gft.py create mode 100644 integrations/server/test_ght.py create mode 100644 integrations/server/test_kcdc_ili.py create mode 100644 integrations/server/test_meta.py create mode 100644 integrations/server/test_meta_norostat.py create mode 100644 integrations/server/test_nidss_dengue.py create mode 100644 integrations/server/test_nidss_flu.py create mode 100644 integrations/server/test_norostat.py create mode 100644 integrations/server/test_nowcast.py create mode 100644 integrations/server/test_paho_dengue.py create mode 100644 integrations/server/test_quidel.py create mode 100644 integrations/server/test_sensors.py create mode 100644 integrations/server/test_signal_dashboard.py create mode 100644 integrations/server/test_twitter.py create mode 100644 integrations/server/test_wiki.py diff --git a/integrations/server/test_cdc.py b/integrations/server/test_cdc.py new file mode 100644 index 000000000..2b81d562c --- /dev/null +++ b/integrations/server/test_cdc.py @@ -0,0 +1,86 @@ +# standard library +import unittest + +# third party +import mysql.connector + +# first party +from delphi.epidata.client.delphi_epidata import Epidata +from delphi.epidata.server._limiter import limiter + + +class CdcTest(unittest.TestCase): + """Basic integration tests for cdc endpint.""" + + @classmethod + def setUpClass(cls) -> None: + """Perform one-time setup.""" + + # use local instance of the Epidata API + Epidata.BASE_URL = "http://delphi_web_epidata/epidata/api.php" + + def setUp(self) -> None: + """Perform per-test setup.""" + + # connect to the `epidata` database + cnx = mysql.connector.connect(user="user", password="pass", host="delphi_database_epidata", database="epidata") + cur = cnx.cursor() + + cur.execute("DELETE FROM api_user") + cur.execute("TRUNCATE TABLE user_role") + cur.execute("TRUNCATE TABLE user_role_link") + cur.execute("TRUNCATE TABLE cdc_extract") + + cur.execute('INSERT INTO api_user(api_key, email) VALUES("cdc_key", "cdc_email")') + cur.execute('INSERT INTO user_role(name) VALUES("cdc") ON DUPLICATE KEY UPDATE name="cdc"') + cur.execute( + 'INSERT INTO user_role_link(user_id, role_id) SELECT api_user.id, 1 FROM api_user WHERE api_key="cdc_key"' + ) + + cnx.commit() + cur.close() + + self.cnx = cnx + self.cur = cnx.cursor() + + @staticmethod + def _clear_limits(): + limiter.storage.reset() + + def tearDown(self) -> None: + """Perform per-test teardown.""" + self.cur.close() + self.cnx.close() + self._clear_limits() + + def test_cdc(self): + """Basic integration test for cdc endpoint""" + self.cur.execute( + "INSERT INTO cdc_extract(epiweek, state, num1, num2, num3, num4, num5, num6, num7, num8, total) VALUES(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)", + ("201102", "AK", "16", "35", "51", "96", "30", "748", "243", "433", "65"), + ) + self.cnx.commit() + response = Epidata.cdc(auth="cdc_key", epiweeks=201102, locations="cen9") + self.assertEqual( + response, + { + "epidata": [ + { + "location": "cen9", + "epiweek": 201102, + "num1": 16, + "num2": 35, + "num3": 51, + "num4": 96, + "num5": 30, + "num6": 748, + "num7": 243, + "num8": 433, + "total": 65, + "value": None, + } + ], + "result": 1, + "message": "success", + }, + ) diff --git a/integrations/server/test_delphi.py b/integrations/server/test_delphi.py new file mode 100644 index 000000000..ee6780cd9 --- /dev/null +++ b/integrations/server/test_delphi.py @@ -0,0 +1,97 @@ +# standard library +import unittest + +# third party +import mysql.connector +import json + +# first party +from delphi.epidata.client.delphi_epidata import Epidata +from delphi.epidata.server._limiter import limiter + + +class DelphiTest(unittest.TestCase): + """Basic integration tests for delphi endpint.""" + + @classmethod + def setUpClass(cls) -> None: + """Perform one-time setup.""" + + # use local instance of the Epidata API + Epidata.BASE_URL = "http://delphi_web_epidata/epidata/api.php" + Epidata.auth = ("epidata", "key") + + def setUp(self) -> None: + """Perform per-test setup.""" + + # connect to the `epidata` database + cnx = mysql.connector.connect(user="user", password="pass", host="delphi_database_epidata", database="epidata") + cur = cnx.cursor() + + cur.execute("DELETE FROM api_user") + cur.execute("TRUNCATE TABLE forecasts") + + cur.execute('INSERT INTO api_user(api_key, email) VALUES("key", "email")') + + cnx.commit() + cur.close() + + self.cnx = cnx + self.cur = cnx.cursor() + + @staticmethod + def _clear_limits(): + limiter.storage.reset() + + def tearDown(self) -> None: + """Perform per-test teardown.""" + self.cur.close() + self.cnx.close() + self._clear_limits() + + def test_delphi(self): + """Basic integration test for delphi endpoint""" + self.cur.execute( + "INSERT INTO forecasts (`system`, `epiweek`, `json`) VALUES(%s, %s, %s)", + ( + "eb", + "201441", + json.dumps( + { + "_version": "version", + "name": "name", + "season": "season", + "epiweek": "epiweek", + "year_weeks": 222, + "season_weeks": 111, + "ili_bins": "ili_bins_123", + "ili_bin_size": "ili_bin_size231", + } + ), + ), + ) + self.cnx.commit() + response = Epidata.delphi(system="eb", epiweek=201441) + self.assertEqual( + response, + { + "epidata": [ + { + "epiweek": 201441, + "forecast": { + "_version": "version", + "epiweek": "epiweek", + "ili_bin_size": "ili_bin_size231", + "ili_bins": "ili_bins_123", + "name": "name", + "season": "season", + "season_weeks": 111, + "year_weeks": 222, + }, + "system": "eb", + } + ], + "message": "success", + "result": 1, + }, + ) diff --git a/integrations/server/test_dengue_nowcast.py b/integrations/server/test_dengue_nowcast.py new file mode 100644 index 000000000..de40ae5c3 --- /dev/null +++ b/integrations/server/test_dengue_nowcast.py @@ -0,0 +1,84 @@ +# standard library +import unittest + +# third party +import mysql.connector + +# first party +from delphi.epidata.client.delphi_epidata import Epidata +from delphi.epidata.server._limiter import limiter + + +class DengueNowcastTest(unittest.TestCase): + """Basic integration tests for dengue_nowcast endpint.""" + + @classmethod + def setUpClass(cls) -> None: + """Perform one-time setup.""" + + # use local instance of the Epidata API + Epidata.BASE_URL = "http://delphi_web_epidata/epidata/api.php" + Epidata.auth = ("epidata", "key") + + def setUp(self) -> None: + """Perform per-test setup.""" + + # connect to the `epidata` database + cnx = mysql.connector.connect(user="user", password="pass", host="delphi_database_epidata", database="epidata") + cur = cnx.cursor() + + cur.execute( + """ + CREATE TABLE IF NOT EXISTS `dengue_nowcasts` ( + `id` int NOT NULL AUTO_INCREMENT, + `target` varchar(32) CHARACTER SET utf8mb3 COLLATE utf8mb3_general_ci NOT NULL, + `epiweek` int NOT NULL, + `location` varchar(12) CHARACTER SET utf8mb3 COLLATE utf8mb3_general_ci DEFAULT NULL, + `value` float NOT NULL, + `std` float NOT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `entry` (`target`,`epiweek`,`location`), + KEY `target` (`target`), + KEY `epiweek` (`epiweek`), + KEY `location` (`location`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3; + """ + ) + + cur.execute("DELETE FROM api_user") + cur.execute("TRUNCATE TABLE dengue_nowcasts") + + cur.execute('INSERT INTO api_user(api_key, email) VALUES("key", "email")') + + cnx.commit() + cur.close() + + self.cnx = cnx + self.cur = cnx.cursor() + + @staticmethod + def _clear_limits(): + limiter.storage.reset() + + def tearDown(self) -> None: + """Perform per-test teardown.""" + self.cur.close() + self.cnx.close() + self._clear_limits() + + def test_dengue_nowcasts(self): + """Basic integration test for dengue_nowcasts endpoint""" + self.cur.execute( + "INSERT INTO dengue_nowcasts(target, epiweek, location, value, std) VALUES(%s, %s, %s, %s, %s)", + ("num_dengue", "201409", "ar", "85263", "351456"), + ) + self.cnx.commit() + response = Epidata.dengue_nowcast(locations="ar", epiweeks=201409) + self.assertEqual( + response, + { + "epidata": [{"location": "ar", "epiweek": 201409, "value": 85263.0, "std": 351456.0}], + "result": 1, + "message": "success", + }, + ) diff --git a/integrations/server/test_dengue_sensors.py b/integrations/server/test_dengue_sensors.py new file mode 100644 index 000000000..5a690969f --- /dev/null +++ b/integrations/server/test_dengue_sensors.py @@ -0,0 +1,90 @@ +# standard library +import unittest + +# third party +import mysql.connector + +# first party +from delphi.epidata.client.delphi_epidata import Epidata +from delphi.epidata.server._limiter import limiter + + +class DengueSensorsTest(unittest.TestCase): + """Basic integration tests for dengue_sensors endpint.""" + + @classmethod + def setUpClass(cls) -> None: + """Perform one-time setup.""" + + # use local instance of the Epidata API + Epidata.BASE_URL = "http://delphi_web_epidata/epidata/api.php" + + def setUp(self) -> None: + """Perform per-test setup.""" + + # connect to the `epidata` database + cnx = mysql.connector.connect(user="user", password="pass", host="delphi_database_epidata", database="epidata") + cur = cnx.cursor() + + cur.execute( + """ + CREATE TABLE IF NOT EXISTS `dengue_sensors` ( + `id` int NOT NULL AUTO_INCREMENT, + `target` varchar(32) CHARACTER SET utf8mb3 COLLATE utf8mb3_general_ci NOT NULL, + `name` varchar(8) CHARACTER SET utf8mb3 COLLATE utf8mb3_general_ci NOT NULL, + `epiweek` int NOT NULL, + `location` varchar(12) CHARACTER SET utf8mb3 COLLATE utf8mb3_general_ci DEFAULT NULL, + `value` float NOT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `entry` (`target`,`name`,`epiweek`,`location`), + KEY `sensor` (`target`,`name`), + KEY `epiweek` (`epiweek`), + KEY `location` (`location`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3; + """ + ) + + cur.execute("TRUNCATE TABLE dengue_sensors") + + cur.execute("DELETE FROM api_user") + cur.execute("TRUNCATE TABLE user_role") + cur.execute("TRUNCATE TABLE user_role_link") + + cur.execute('INSERT INTO api_user(api_key, email) VALUES("sensors_key", "sensors_email")') + cur.execute('INSERT INTO user_role(name) VALUES("sensors") ON DUPLICATE KEY UPDATE name="sensors"') + cur.execute( + 'INSERT INTO user_role_link(user_id, role_id) SELECT api_user.id, 1 FROM api_user WHERE api_key="sensors_key"' + ) + + cnx.commit() + cur.close() + + self.cnx = cnx + self.cur = cnx.cursor() + + @staticmethod + def _clear_limits(): + limiter.storage.reset() + + def tearDown(self) -> None: + """Perform per-test teardown.""" + self.cur.close() + self.cnx.close() + self._clear_limits() + + def test_dengue_sensors(self): + """Basic integration test for dengue_sensors endpoint""" + self.cur.execute( + "INSERT INTO dengue_sensors(target, name, epiweek, location, value) VALUES(%s, %s, %s, %s, %s)", + ("num_dengue", "ght", "201432", "ag", "1234"), + ) + self.cnx.commit() + response = Epidata.dengue_sensors(auth="sensors_key", names="ght", locations="ag", epiweeks="201432") + self.assertEqual( + response, + { + "epidata": [{"name": "ght", "location": "ag", "epiweek": 201432, "value": 1234.0}], + "result": 1, + "message": "success", + }, + ) diff --git a/integrations/server/test_ecdc_ili.py b/integrations/server/test_ecdc_ili.py new file mode 100644 index 000000000..d68bbe7a0 --- /dev/null +++ b/integrations/server/test_ecdc_ili.py @@ -0,0 +1,75 @@ +# standard library +import unittest + +# third party +import mysql.connector + +# first party +from delphi.epidata.client.delphi_epidata import Epidata +from delphi.epidata.server._limiter import limiter + + +class EcdcIliTest(unittest.TestCase): + """Basic integration tests for edcd_ili endpint.""" + + @classmethod + def setUpClass(cls) -> None: + """Perform one-time setup.""" + + # use local instance of the Epidata API + Epidata.BASE_URL = "http://delphi_web_epidata/epidata/api.php" + Epidata.auth = ("epidata", "key") + + def setUp(self) -> None: + """Perform per-test setup.""" + + # connect to the `epidata` database + cnx = mysql.connector.connect(user="user", password="pass", host="delphi_database_epidata", database="epidata") + cur = cnx.cursor() + + cur.execute("DELETE FROM api_user") + cur.execute("TRUNCATE TABLE ecdc_ili") + + cur.execute('INSERT INTO api_user(api_key, email) VALUES("key", "email")') + + cnx.commit() + cur.close() + + self.cnx = cnx + self.cur = cnx.cursor() + + @staticmethod + def _clear_limits(): + limiter.storage.reset() + + def tearDown(self) -> None: + """Perform per-test teardown.""" + self.cur.close() + self.cnx.close() + self._clear_limits() + + def test_ecdc_ili(self): + """Basic integration test for ecdc_ili endpoint""" + self.cur.execute( + "INSERT INTO ecdc_ili(`release_date`, `issue`, `epiweek`, `lag`, `region`, `incidence_rate`) VALUES(%s, %s, %s, %s, %s, %s)", + ("2020-03-26", "202012", "201840", "76", "Armenia", "0"), + ) + self.cnx.commit() + response = Epidata.ecdc_ili(regions="Armenia", epiweeks="201840") + self.assertEqual( + response, + { + "epidata": [ + { + "release_date": "2020-03-26", + "region": "Armenia", + "issue": 202012, + "epiweek": 201840, + "lag": 76, + "incidence_rate": 0.0, + } + ], + "result": 1, + "message": "success", + }, + ) diff --git a/integrations/server/test_flusurv.py b/integrations/server/test_flusurv.py new file mode 100644 index 000000000..f6fd4bc16 --- /dev/null +++ b/integrations/server/test_flusurv.py @@ -0,0 +1,80 @@ +# standard library +import unittest + +# third party +import mysql.connector + +# first party +from delphi.epidata.client.delphi_epidata import Epidata +from delphi.epidata.server._limiter import limiter + + +class FlusurvTest(unittest.TestCase): + """Basic integration tests for flusurv endpint.""" + + @classmethod + def setUpClass(cls) -> None: + """Perform one-time setup.""" + + # use local instance of the Epidata API + Epidata.BASE_URL = "http://delphi_web_epidata/epidata/api.php" + Epidata.auth = ("epidata", "key") + + def setUp(self) -> None: + """Perform per-test setup.""" + + # connect to the `epidata` database + cnx = mysql.connector.connect(user="user", password="pass", host="delphi_database_epidata", database="epidata") + cur = cnx.cursor() + + cur.execute("DELETE FROM api_user") + cur.execute("TRUNCATE TABLE flusurv") + + cur.execute('INSERT INTO api_user(api_key, email) VALUES("key", "email")') + + cnx.commit() + cur.close() + + self.cnx = cnx + self.cur = cnx.cursor() + + @staticmethod + def _clear_limits(): + limiter.storage.reset() + + def tearDown(self) -> None: + """Perform per-test teardown.""" + self.cur.close() + self.cnx.close() + self._clear_limits() + + def test_ecdc_ili(self): + """Basic integration test for flusurv endpoint""" + self.cur.execute( + "INSERT INTO flusurv(`release_date`, `issue`, `epiweek`, `location`, `lag`, `rate_age_0`, `rate_age_1`, `rate_age_2`, `rate_age_3`, `rate_age_4`, `rate_overall`, `rate_age_5`, `rate_age_6`, `rate_age_7`) VALUES(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)", + ("2012-11-02", "201243", "201143", "CA", "52", "0", "0", "0", "0.151", "0", "0.029", "0", "0", "0"), + ) + self.cnx.commit() + response = Epidata.flusurv(epiweeks=201143, locations="CA") + self.assertEqual( + response, + { + "epidata": [ + { + "release_date": "2012-11-02", + "location": "CA", + "issue": 201243, + "epiweek": 201143, + "lag": 52, + "rate_age_0": 0.0, + "rate_age_1": 0.0, + "rate_age_2": 0.0, + "rate_age_3": 0.151, + "rate_age_4": 0.0, + "rate_overall": 0.029, + } + ], + "result": 1, + "message": "success", + }, + ) diff --git a/integrations/server/test_fluview_clinical.py b/integrations/server/test_fluview_clinical.py new file mode 100644 index 000000000..5e87dde10 --- /dev/null +++ b/integrations/server/test_fluview_clinical.py @@ -0,0 +1,80 @@ +# standard library +import unittest + +# third party +import mysql.connector + +# first party +from delphi.epidata.client.delphi_epidata import Epidata +from delphi.epidata.server._limiter import limiter + + +class FluviewClinicalTest(unittest.TestCase): + """Basic integration tests for fluview_clinical endpint.""" + + @classmethod + def setUpClass(cls) -> None: + """Perform one-time setup.""" + + # use local instance of the Epidata API + Epidata.BASE_URL = "http://delphi_web_epidata/epidata/api.php" + Epidata.auth = ("epidata", "key") + + def setUp(self) -> None: + """Perform per-test setup.""" + + # connect to the `epidata` database + cnx = mysql.connector.connect(user="user", password="pass", host="delphi_database_epidata", database="epidata") + cur = cnx.cursor() + + cur.execute("DELETE FROM api_user") + cur.execute("TRUNCATE TABLE fluview_clinical") + + cur.execute('INSERT INTO api_user(api_key, email) VALUES("key", "email")') + + cnx.commit() + cur.close() + + self.cnx = cnx + self.cur = cnx.cursor() + + @staticmethod + def _clear_limits(): + limiter.storage.reset() + + def tearDown(self) -> None: + """Perform per-test teardown.""" + self.cur.close() + self.cnx.close() + self._clear_limits() + + def test_ecdc_ili(self): + """Basic integration test for fluview_clinical endpoint""" + self.cur.execute( + "INSERT INTO fluview_clinical(`release_date`, `issue`, `epiweek`, `region`, `lag`, `total_specimens`, `total_a`, `total_b`, `percent_positive`, `percent_a`, `percent_b`) VALUES(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)", + ("2018-10-10", "201839", "201640", "al", "103", "406", "4", "1", "1.32", "0.99", "0.25"), + ) + self.cnx.commit() + response = Epidata.fluview_clinical(epiweeks=201640, regions="al") + self.assertEqual( + response, + { + "epidata": [ + { + "release_date": "2018-10-10", + "region": "al", + "issue": 201839, + "epiweek": 201640, + "lag": 103, + "total_specimens": 406, + "total_a": 4, + "total_b": 1, + "percent_positive": 1.32, + "percent_a": 0.99, + "percent_b": 0.25, + } + ], + "result": 1, + "message": "success", + }, + ) diff --git a/integrations/server/test_gft.py b/integrations/server/test_gft.py new file mode 100644 index 000000000..ee5d8365b --- /dev/null +++ b/integrations/server/test_gft.py @@ -0,0 +1,62 @@ +# standard library +import unittest + +# third party +import mysql.connector + +# first party +from delphi.epidata.client.delphi_epidata import Epidata +from delphi.epidata.server._limiter import limiter + + +class GftTest(unittest.TestCase): + """Basic integration tests for gft endpint.""" + + @classmethod + def setUpClass(cls) -> None: + """Perform one-time setup.""" + + # use local instance of the Epidata API + Epidata.BASE_URL = "http://delphi_web_epidata/epidata/api.php" + Epidata.auth = ("epidata", "key") + + def setUp(self) -> None: + """Perform per-test setup.""" + + # connect to the `epidata` database + cnx = mysql.connector.connect(user="user", password="pass", host="delphi_database_epidata", database="epidata") + cur = cnx.cursor() + + cur.execute("DELETE FROM api_user") + cur.execute("TRUNCATE TABLE gft") + + cur.execute('INSERT INTO api_user(api_key, email) VALUES("key", "email")') + + cnx.commit() + cur.close() + + self.cnx = cnx + self.cur = cnx.cursor() + + @staticmethod + def _clear_limits(): + limiter.storage.reset() + + def tearDown(self) -> None: + """Perform per-test teardown.""" + self.cur.close() + self.cnx.close() + self._clear_limits() + + def test_gft(self): + """Basic integration test for gft endpoint""" + self.cur.execute( + "INSERT INTO gft(epiweek, location, num) VALUES(%s, %s, %s)", + ("200340", "nat", "902"), + ) + self.cnx.commit() + response = Epidata.gft(locations="nat", epiweeks="200340") + self.assertEqual( + response, + {"epidata": [{"location": "nat", "epiweek": 200340, "num": 902}], "result": 1, "message": "success"}, + ) diff --git a/integrations/server/test_ght.py b/integrations/server/test_ght.py new file mode 100644 index 000000000..595ddc700 --- /dev/null +++ b/integrations/server/test_ght.py @@ -0,0 +1,67 @@ +# standard library +import unittest + +# third party +import mysql.connector + +# first party +from delphi.epidata.client.delphi_epidata import Epidata +from delphi.epidata.server._limiter import limiter + + +class GhtTest(unittest.TestCase): + """Basic integration tests for gft endpint.""" + + @classmethod + def setUpClass(cls) -> None: + """Perform one-time setup.""" + + # use local instance of the Epidata API + Epidata.BASE_URL = "http://delphi_web_epidata/epidata/api.php" + + def setUp(self) -> None: + """Perform per-test setup.""" + + # connect to the `epidata` database + cnx = mysql.connector.connect(user="user", password="pass", host="delphi_database_epidata", database="epidata") + cur = cnx.cursor() + + cur.execute("DELETE FROM api_user") + cur.execute("TRUNCATE TABLE user_role") + cur.execute("TRUNCATE TABLE user_role_link") + cur.execute("TRUNCATE TABLE ght") + + cur.execute('INSERT INTO api_user(api_key, email) VALUES("ght_key", "ght_email")') + cur.execute('INSERT INTO user_role(name) VALUES("ght") ON DUPLICATE KEY UPDATE name="ght"') + cur.execute( + 'INSERT INTO user_role_link(user_id, role_id) SELECT api_user.id, 1 FROM api_user WHERE api_key="ght_key"' + ) + + cnx.commit() + cur.close() + + self.cnx = cnx + self.cur = cnx.cursor() + + @staticmethod + def _clear_limits(): + limiter.storage.reset() + + def tearDown(self) -> None: + """Perform per-test teardown.""" + self.cur.close() + self.cnx.close() + self._clear_limits() + + def test_ght(self): + """Basic integration test for ght endpoint""" + self.cur.execute( + "INSERT INTO ght(`query`, `location`, `epiweek`, `value`) VALUES(%s, %s, %s, %s)", + ("/n/query", "US", "200101", "12345"), + ) + self.cnx.commit() + response = Epidata.ght(locations="US", epiweeks="200101", query="/n/query", auth="ght_key") + self.assertEqual( + response, + {"epidata": [{"location": "US", "epiweek": 200101, "value": 12345.0}], "result": 1, "message": "success"}, + ) diff --git a/integrations/server/test_kcdc_ili.py b/integrations/server/test_kcdc_ili.py new file mode 100644 index 000000000..c337db29f --- /dev/null +++ b/integrations/server/test_kcdc_ili.py @@ -0,0 +1,75 @@ +# standard library +import unittest + +# third party +import mysql.connector + +# first party +from delphi.epidata.client.delphi_epidata import Epidata +from delphi.epidata.server._limiter import limiter + + +class KcdcIliTest(unittest.TestCase): + """Basic integration tests for kcdc_ili endpint.""" + + @classmethod + def setUpClass(cls) -> None: + """Perform one-time setup.""" + + # use local instance of the Epidata API + Epidata.BASE_URL = "http://delphi_web_epidata/epidata/api.php" + Epidata.auth = ("epidata", "key") + + def setUp(self) -> None: + """Perform per-test setup.""" + + # connect to the `epidata` database + cnx = mysql.connector.connect(user="user", password="pass", host="delphi_database_epidata", database="epidata") + cur = cnx.cursor() + + cur.execute("DELETE FROM api_user") + cur.execute("TRUNCATE TABLE kcdc_ili") + + cur.execute('INSERT INTO api_user(api_key, email) VALUES("key", "email")') + + cnx.commit() + cur.close() + + self.cnx = cnx + self.cur = cnx.cursor() + + @staticmethod + def _clear_limits(): + limiter.storage.reset() + + def tearDown(self) -> None: + """Perform per-test teardown.""" + self.cur.close() + self.cnx.close() + self._clear_limits() + + def test_kcdc_ili(self): + """Basic integration test for kcdc_ili endpoint""" + self.cur.execute( + "INSERT INTO kcdc_ili(`release_date`, `issue`, `epiweek`, `lag`, `region`, `ili`) VALUES(%s, %s, %s, %s, %s, %s)", + ("2020-03-27", "202013", "200432", "222", "REG", "0.25"), + ) + self.cnx.commit() + response = Epidata.kcdc_ili(regions="REG", epiweeks="200432") + self.assertEqual( + response, + { + "epidata": [ + { + "release_date": "2020-03-27", + "region": "REG", + "issue": 202013, + "epiweek": 200432, + "lag": 222, + "ili": 0.25, + } + ], + "result": 1, + "message": "success", + }, + ) diff --git a/integrations/server/test_meta.py b/integrations/server/test_meta.py new file mode 100644 index 000000000..45cbff125 --- /dev/null +++ b/integrations/server/test_meta.py @@ -0,0 +1,72 @@ +# standard library +import unittest + +# third party +import mysql.connector + +# first party +from delphi.epidata.client.delphi_epidata import Epidata +from delphi.epidata.server._limiter import limiter + + +class MetaTest(unittest.TestCase): + """Basic integration tests for meta endpint.""" + + @classmethod + def setUpClass(cls) -> None: + """Perform one-time setup.""" + + # use local instance of the Epidata API + Epidata.BASE_URL = "http://delphi_web_epidata/epidata/api.php" + Epidata.auth = ("epidata", "key") + + def setUp(self) -> None: + """Perform per-test setup.""" + + # connect to the `epidata` database + cnx = mysql.connector.connect(user="user", password="pass", host="delphi_database_epidata", database="epidata") + cur = cnx.cursor() + + cur.execute("DELETE FROM api_user") + cur.execute("TRUNCATE TABLE forecasts") + cur.execute("TRUNCATE TABLE fluview") + cur.execute("TRUNCATE TABLE wiki") + cur.execute("TRUNCATE TABLE wiki_meta") + cur.execute("TRUNCATE TABLE twitter") + + cur.execute('INSERT INTO api_user(api_key, email) VALUES("key", "email")') + + cnx.commit() + cur.close() + + self.cnx = cnx + self.cur = cnx.cursor() + + @staticmethod + def _clear_limits(): + limiter.storage.reset() + + def tearDown(self) -> None: + """Perform per-test teardown.""" + self.cur.close() + self.cnx.close() + self._clear_limits() + + def test_meta(self): + """Basic integration test for meta endpoint""" + response = Epidata.meta() + self.assertEqual( + response, + { + "epidata": [ + { + "delphi": [], + "fluview": [{"latest_issue": None, "latest_update": None, "table_rows": 0}], + "twitter": [], + "wiki": [{"latest_update": None, "table_rows": 0}], + } + ], + "message": "success", + "result": 1, + }, + ) diff --git a/integrations/server/test_meta_norostat.py b/integrations/server/test_meta_norostat.py new file mode 100644 index 000000000..5462af750 --- /dev/null +++ b/integrations/server/test_meta_norostat.py @@ -0,0 +1,135 @@ +# standard library +import unittest + +# third party +import mysql.connector + +# first party +from delphi.epidata.client.delphi_epidata import Epidata +from delphi.epidata.server._limiter import limiter + + +class MetaNorostatTest(unittest.TestCase): + """Basic integration tests for meta_norostat endpint.""" + + @classmethod + def setUpClass(cls) -> None: + """Perform one-time setup.""" + + # use local instance of the Epidata API + Epidata.BASE_URL = "http://delphi_web_epidata/epidata/api.php" + + def setUp(self) -> None: + """Perform per-test setup.""" + + # connect to the `epidata` database + cnx = mysql.connector.connect(user="user", password="pass", host="delphi_database_epidata", database="epidata") + cur = cnx.cursor() + + cur.execute( + """ + CREATE TABLE IF NOT EXISTS `norostat_raw_datatable_version_list` ( + `release_date` date NOT NULL, + `parse_time` datetime NOT NULL, + PRIMARY KEY (`release_date`,`parse_time`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3; + """ + ) + + # cnx.commit() + + cur.execute( + """ + CREATE TABLE IF NOT EXISTS `norostat_point_version_list` ( + `release_date` date NOT NULL, + `parse_time` datetime NOT NULL, + PRIMARY KEY (`release_date`,`parse_time`), + CONSTRAINT `norostat_point_version_list_ibfk_1` FOREIGN KEY (`release_date`, `parse_time`) REFERENCES `norostat_raw_datatable_version_list` (`release_date`, `parse_time`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3; + """ + ) + + # cnx.commit() + + cur.execute( + """ + CREATE TABLE IF NOT EXISTS `norostat_raw_datatable_location_pool` ( + `location_id` int NOT NULL AUTO_INCREMENT, + `location` varchar(255) CHARACTER SET utf8mb3 COLLATE utf8mb3_general_ci NOT NULL, + PRIMARY KEY (`location_id`), + UNIQUE KEY `location` (`location`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3; + """ + ) + + # cnx.commit() + + cur.execute( + """ + CREATE TABLE IF NOT EXISTS `norostat_point_diffs` ( + `release_date` date NOT NULL, + `parse_time` datetime NOT NULL, + `location_id` int NOT NULL, + `epiweek` int NOT NULL, + `new_value` varchar(255) CHARACTER SET utf8mb3 COLLATE utf8mb3_general_ci DEFAULT NULL, + PRIMARY KEY (`release_date`,`parse_time`,`location_id`,`epiweek`), + UNIQUE KEY `location_id` (`location_id`,`epiweek`,`release_date`,`parse_time`,`new_value`), + CONSTRAINT `norostat_point_diffs_ibfk_1` FOREIGN KEY (`release_date`, `parse_time`) REFERENCES `norostat_point_version_list` (`release_date`, `parse_time`), + CONSTRAINT `norostat_point_diffs_ibfk_2` FOREIGN KEY (`location_id`) REFERENCES `norostat_raw_datatable_location_pool` (`location_id`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3; + """ + ) + + cur.execute("DELETE FROM api_user") + cur.execute("TRUNCATE TABLE user_role") + cur.execute("TRUNCATE TABLE user_role_link") + cur.execute("DELETE FROM norostat_point_diffs") + cur.execute("DELETE FROM norostat_point_version_list") + cur.execute("DELETE FROM norostat_raw_datatable_location_pool") + cur.execute("DELETE FROM norostat_raw_datatable_version_list") + + cur.execute('INSERT INTO api_user(api_key, email) VALUES("norostat_key", "norostat_email")') + cur.execute('INSERT INTO user_role(name) VALUES("norostat") ON DUPLICATE KEY UPDATE name="norostat"') + cur.execute( + 'INSERT INTO user_role_link(user_id, role_id) SELECT api_user.id, 1 FROM api_user WHERE api_key="norostat_key"' + ) + + cnx.commit() + cur.close() + + self.cnx = cnx + self.cur = cnx.cursor() + + @staticmethod + def _clear_limits(): + limiter.storage.reset() + + def tearDown(self) -> None: + """Perform per-test teardown.""" + self.cur.close() + self.cnx.close() + self._clear_limits() + + def test_meta_norostat(self): + """Basic integration test for meta_norostat endpoint""" + + self.cur.execute( + "INSERT INTO norostat_raw_datatable_version_list(release_date, parse_time) VALUES (%s, %s)", + ("2014-10-22", "2048-12-08 15:22:51"), + ) + self.cur.execute( + 'INSERT INTO norostat_raw_datatable_location_pool(`location`) VALUES ("Minnesota, Ohio, Oregon, Tennessee, and Wisconsin")' + ) + self.cnx.commit() + response = Epidata.meta_norostat(auth="norostat_key") + self.assertEqual( + response, + { + "epidata": { + "locations": [{"location": "Minnesota, Ohio, Oregon, Tennessee, and Wisconsin"}], + "releases": [{"release_date": "2014-10-22"}], + }, + "message": "success", + "result": 1, + }, + ) diff --git a/integrations/server/test_nidss_dengue.py b/integrations/server/test_nidss_dengue.py new file mode 100644 index 000000000..d57d2a18a --- /dev/null +++ b/integrations/server/test_nidss_dengue.py @@ -0,0 +1,62 @@ +# standard library +import unittest + +# third party +import mysql.connector + +# first party +from delphi.epidata.client.delphi_epidata import Epidata +from delphi.epidata.server._limiter import limiter + + +class NiddsDengueTest(unittest.TestCase): + """Basic integration tests for nids_dengue endpint.""" + + @classmethod + def setUpClass(cls) -> None: + """Perform one-time setup.""" + + # use local instance of the Epidata API + Epidata.BASE_URL = "http://delphi_web_epidata/epidata/api.php" + Epidata.auth = ("epidata", "key") + + def setUp(self) -> None: + """Perform per-test setup.""" + + # connect to the `epidata` database + cnx = mysql.connector.connect(user="user", password="pass", host="delphi_database_epidata", database="epidata") + cur = cnx.cursor() + + cur.execute("DELETE FROM api_user") + cur.execute("TRUNCATE TABLE nidss_dengue") + + cur.execute('INSERT INTO api_user(api_key, email) VALUES("key", "email")') + + cnx.commit() + cur.close() + + self.cnx = cnx + self.cur = cnx.cursor() + + @staticmethod + def _clear_limits(): + limiter.storage.reset() + + def tearDown(self) -> None: + """Perform per-test teardown.""" + self.cur.close() + self.cnx.close() + self._clear_limits() + + def test_nidss_dengue(self): + """Basic integration test for nidds_dengue endpoint""" + self.cur.execute( + "INSERT INTO nidss_dengue(epiweek, location, region, count) VALUES(%s, %s, %s, %s)", + ("200340", "SomeCity", "Central", "0"), + ) + self.cnx.commit() + response = Epidata.nidss_dengue(locations="SomeCity", epiweeks="200340") + self.assertEqual( + response, + {"epidata": [{"location": "SomeCity", "epiweek": 200340, "count": 0}], "result": 1, "message": "success"}, + ) diff --git a/integrations/server/test_nidss_flu.py b/integrations/server/test_nidss_flu.py new file mode 100644 index 000000000..6f38559ae --- /dev/null +++ b/integrations/server/test_nidss_flu.py @@ -0,0 +1,76 @@ +# standard library +import unittest + +# third party +import mysql.connector + +# first party +from delphi.epidata.client.delphi_epidata import Epidata +from delphi.epidata.server._limiter import limiter + + +class NiddsFluTest(unittest.TestCase): + """Basic integration tests for nids_flu endpint.""" + + @classmethod + def setUpClass(cls) -> None: + """Perform one-time setup.""" + + # use local instance of the Epidata API + Epidata.BASE_URL = "http://delphi_web_epidata/epidata/api.php" + Epidata.auth = ("epidata", "key") + + def setUp(self) -> None: + """Perform per-test setup.""" + + # connect to the `epidata` database + cnx = mysql.connector.connect(user="user", password="pass", host="delphi_database_epidata", database="epidata") + cur = cnx.cursor() + + cur.execute("DELETE FROM api_user") + cur.execute("TRUNCATE TABLE nidss_flu") + + cur.execute('INSERT INTO api_user(api_key, email) VALUES("key", "email")') + + cnx.commit() + cur.close() + + self.cnx = cnx + self.cur = cnx.cursor() + + @staticmethod + def _clear_limits(): + limiter.storage.reset() + + def tearDown(self) -> None: + """Perform per-test teardown.""" + self.cur.close() + self.cnx.close() + self._clear_limits() + + def test_nidss_flu(self): + """Basic integration test for nidds_flu endpoint""" + self.cur.execute( + "INSERT INTO nidss_flu(`release_date`, `issue`, `epiweek`, `region`, `lag`, `visits`, `ili`) VALUES(%s, %s, %s, %s, %s, %s, %s)", + ("2015-09-05", "201530", "200111", "SomeRegion", "222", "333", "444"), + ) + self.cnx.commit() + response = Epidata.nidss_flu(regions="SomeRegion", epiweeks="200111") + self.assertEqual( + response, + { + "epidata": [ + { + "release_date": "2015-09-05", + "region": "SomeRegion", + "issue": 201530, + "epiweek": 200111, + "lag": 222, + "visits": 333, + "ili": 444.0, + } + ], + "result": 1, + "message": "success", + }, + ) diff --git a/integrations/server/test_norostat.py b/integrations/server/test_norostat.py new file mode 100644 index 000000000..461dfc779 --- /dev/null +++ b/integrations/server/test_norostat.py @@ -0,0 +1,93 @@ +# standard library +import unittest + +# third party +import mysql.connector + +# first party +from delphi.epidata.client.delphi_epidata import Epidata +from delphi.epidata.server._limiter import limiter + + +class NorostatTest(unittest.TestCase): + """Basic integration tests for norostat endpint.""" + + @classmethod + def setUpClass(cls) -> None: + """Perform one-time setup.""" + + # use local instance of the Epidata API + Epidata.BASE_URL = "http://delphi_web_epidata/epidata/api.php" + + def setUp(self) -> None: + """Perform per-test setup.""" + + # connect to the `epidata` database + cnx = mysql.connector.connect(user="user", password="pass", host="delphi_database_epidata", database="epidata") + cur = cnx.cursor() + + cur.execute( + """ + CREATE TABLE IF NOT EXISTS `norostat_point_diffs` ( + `release_date` date NOT NULL, + `parse_time` datetime NOT NULL, + `location_id` int NOT NULL, + `epiweek` int NOT NULL, + `new_value` varchar(255) CHARACTER SET utf8mb3 COLLATE utf8mb3_general_ci DEFAULT NULL, + PRIMARY KEY (`release_date`,`parse_time`,`location_id`,`epiweek`), + UNIQUE KEY `location_id` (`location_id`,`epiweek`,`release_date`,`parse_time`,`new_value`), + CONSTRAINT `norostat_point_diffs_ibfk_1` FOREIGN KEY (`release_date`, `parse_time`) REFERENCES `norostat_point_version_list` (`release_date`, `parse_time`), + CONSTRAINT `norostat_point_diffs_ibfk_2` FOREIGN KEY (`location_id`) REFERENCES `norostat_raw_datatable_location_pool` (`location_id`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3; + """ + ) + + cur.execute("DELETE FROM api_user") + cur.execute("TRUNCATE TABLE user_role") + cur.execute("TRUNCATE TABLE user_role_link") + + cur.execute("DELETE FROM norostat_point_diffs") + cur.execute("DELETE FROM norostat_point_version_list") + cur.execute("DELETE FROM norostat_raw_datatable_location_pool") + cur.execute("DELETE FROM norostat_raw_datatable_version_list") + + + cur.execute('INSERT INTO api_user(api_key, email) VALUES("norostat_key", "norostat_email")') + cur.execute('INSERT INTO user_role(name) VALUES("norostat") ON DUPLICATE KEY UPDATE name="norostat"') + cur.execute( + 'INSERT INTO user_role_link(user_id, role_id) SELECT api_user.id, 1 FROM api_user WHERE api_key="norostat_key"' + ) + + cnx.commit() + cur.close() + + self.cnx = cnx + self.cur = cnx.cursor() + + @staticmethod + def _clear_limits(): + limiter.storage.reset() + + def tearDown(self) -> None: + """Perform per-test teardown.""" + self.cur.close() + self.cnx.close() + self._clear_limits() + + def test_norostat(self): + """Basic integration test for norostat endpoint""" + self.cur.execute('INSERT INTO norostat_raw_datatable_version_list(release_date, parse_time) VALUES ("2023-07-19", "2023-07-10 15:24:51")') + self.cur.execute('INSERT INTO norostat_raw_datatable_location_pool(location_id, location) VALUES("1", "SomeTestLocation")') + self.cur.execute('INSERT INTO norostat_point_version_list(`release_date`, `parse_time`) VALUES("2023-07-19", "2023-07-10 15:24:51")') + self.cur.execute('INSERT INTO norostat_point_diffs(release_date, parse_time, location_id, epiweek, new_value) VALUES("2023-07-19", "2023-07-10 15:24:51", "1", "202329", 10)') + self.cnx.commit() + response = Epidata.norostat(auth="norostat_key", location="SomeTestLocation", epiweeks="202329") + self.assertEqual( + response, + { + "epidata": [{"release_date": "2023-07-19", "epiweek": 202329, "value": 10}], + "result": 1, + "message": "success", + }, + ) + return True diff --git a/integrations/server/test_nowcast.py b/integrations/server/test_nowcast.py new file mode 100644 index 000000000..8f3d4293b --- /dev/null +++ b/integrations/server/test_nowcast.py @@ -0,0 +1,66 @@ +# standard library +import unittest + +# third party +import mysql.connector + +# first party +from delphi.epidata.client.delphi_epidata import Epidata +from delphi.epidata.server._limiter import limiter + + +class NowcastTest(unittest.TestCase): + """Basic integration tests for nowcast endpint.""" + + @classmethod + def setUpClass(cls) -> None: + """Perform one-time setup.""" + + # use local instance of the Epidata API + Epidata.BASE_URL = "http://delphi_web_epidata/epidata/api.php" + Epidata.auth = ("epidata", "key") + + def setUp(self) -> None: + """Perform per-test setup.""" + + # connect to the `epidata` database + cnx = mysql.connector.connect(user="user", password="pass", host="delphi_database_epidata", database="epidata") + cur = cnx.cursor() + + cur.execute("DELETE FROM api_user") + cur.execute("TRUNCATE TABLE nowcasts") + + cur.execute('INSERT INTO api_user(api_key, email) VALUES("key", "email")') + + cnx.commit() + cur.close() + + self.cnx = cnx + self.cur = cnx.cursor() + + @staticmethod + def _clear_limits(): + limiter.storage.reset() + + def tearDown(self) -> None: + """Perform per-test teardown.""" + self.cur.close() + self.cnx.close() + self._clear_limits() + + def test_nowcast(self): + """Basic integration test for nowcast endpoint""" + self.cur.execute( + "INSERT INTO nowcasts(`epiweek`, `location`, `value`, `std`) VALUES(%s, %s, %s, %s)", + ("201145", "nat", "12345", "0.01234"), + ) + self.cnx.commit() + response = Epidata.nowcast(locations="nat", epiweeks="201145") + self.assertEqual( + response, + { + "epidata": [{"location": "nat", "epiweek": 201145, "value": 12345.0, "std": 0.01234}], + "result": 1, + "message": "success", + }, + ) diff --git a/integrations/server/test_paho_dengue.py b/integrations/server/test_paho_dengue.py new file mode 100644 index 000000000..01767a582 --- /dev/null +++ b/integrations/server/test_paho_dengue.py @@ -0,0 +1,80 @@ +# standard library +import unittest + +# third party +import mysql.connector + +# first party +from delphi.epidata.client.delphi_epidata import Epidata +from delphi.epidata.server._limiter import limiter + + +class PahoDengueTest(unittest.TestCase): + """Basic integration tests for paho_dengue endpint.""" + + @classmethod + def setUpClass(cls) -> None: + """Perform one-time setup.""" + + # use local instance of the Epidata API + Epidata.BASE_URL = "http://delphi_web_epidata/epidata/api.php" + Epidata.auth = ("epidata", "key") + + def setUp(self) -> None: + """Perform per-test setup.""" + + # connect to the `epidata` database + cnx = mysql.connector.connect(user="user", password="pass", host="delphi_database_epidata", database="epidata") + cur = cnx.cursor() + + cur.execute("DELETE FROM api_user") + cur.execute("TRUNCATE TABLE paho_dengue") + + cur.execute('INSERT INTO api_user(api_key, email) VALUES("key", "email")') + + cnx.commit() + cur.close() + + self.cnx = cnx + self.cur = cnx.cursor() + + @staticmethod + def _clear_limits(): + limiter.storage.reset() + + def tearDown(self) -> None: + """Perform per-test teardown.""" + self.cur.close() + self.cnx.close() + self._clear_limits() + + def test_paho_dengue(self): + """Basic integration test for paho_dengue endpoint""" + self.cur.execute( + "INSERT INTO paho_dengue(`release_date`, `issue`, `epiweek`, `lag`, `region`, `total_pop`, `serotype`, `num_dengue`, `incidence_rate`, `num_severe`, `num_deaths`) VALUES(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)", + ("2018-12-01", "201848", "201454", "204", "AG", "91", "DEN 1,4", "37", "40.66", "0", "0"), + ) + self.cnx.commit() + response = Epidata.paho_dengue(regions="AG", epiweeks="201454") + self.assertEqual( + response, + { + "epidata": [ + { + "release_date": "2018-12-01", + "region": "AG", + "serotype": "DEN 1,4", + "issue": 201848, + "epiweek": 201454, + "lag": 204, + "total_pop": 91, + "num_dengue": 37, + "num_severe": 0, + "num_deaths": 0, + "incidence_rate": 40.66, + } + ], + "result": 1, + "message": "success", + }, + ) diff --git a/integrations/server/test_quidel.py b/integrations/server/test_quidel.py new file mode 100644 index 000000000..c16d4e770 --- /dev/null +++ b/integrations/server/test_quidel.py @@ -0,0 +1,67 @@ +# standard library +import unittest + +# third party +import mysql.connector + +# first party +from delphi.epidata.client.delphi_epidata import Epidata +from delphi.epidata.server._limiter import limiter + + +class QuidelTest(unittest.TestCase): + """Basic integration tests for quidel endpint.""" + + @classmethod + def setUpClass(cls) -> None: + """Perform one-time setup.""" + + # use local instance of the Epidata API + Epidata.BASE_URL = "http://delphi_web_epidata/epidata/api.php" + + def setUp(self) -> None: + """Perform per-test setup.""" + + # connect to the `epidata` database + cnx = mysql.connector.connect(user="user", password="pass", host="delphi_database_epidata", database="epidata") + cur = cnx.cursor() + + cur.execute("DELETE FROM api_user") + cur.execute("TRUNCATE TABLE user_role") + cur.execute("TRUNCATE TABLE user_role_link") + cur.execute("TRUNCATE TABLE quidel") + + cur.execute('INSERT INTO api_user(api_key, email) VALUES("quidel_key", "quidel_email")') + cur.execute('INSERT INTO user_role(name) VALUES("quidel") ON DUPLICATE KEY UPDATE name="quidel"') + cur.execute( + 'INSERT INTO user_role_link(user_id, role_id) SELECT api_user.id, 1 FROM api_user WHERE api_key="quidel_key"' + ) + + cnx.commit() + cur.close() + + self.cnx = cnx + self.cur = cnx.cursor() + + @staticmethod + def _clear_limits(): + limiter.storage.reset() + + def tearDown(self) -> None: + """Perform per-test teardown.""" + self.cur.close() + self.cnx.close() + self._clear_limits() + + def test_quidel(self): + """Basic integration test for quidel endpoint""" + self.cur.execute( + "INSERT INTO quidel(location, epiweek, value, num_rows, num_devices) VALUES(%s, %s, %s, %s, %s)", + ("loc1", "201111", "1", "0", "0"), + ) + self.cnx.commit() + response = Epidata.quidel(locations="loc1", epiweeks="201111", auth="quidel_key") + self.assertEqual( + response, + {"epidata": [{"location": "loc1", "epiweek": 201111, "value": 1.0}], "result": 1, "message": "success"}, + ) diff --git a/integrations/server/test_sensors.py b/integrations/server/test_sensors.py new file mode 100644 index 000000000..6cf851519 --- /dev/null +++ b/integrations/server/test_sensors.py @@ -0,0 +1,71 @@ +# standard library +import unittest + +# third party +import mysql.connector + +# first party +from delphi.epidata.client.delphi_epidata import Epidata +from delphi.epidata.server._limiter import limiter + + +class SensorsTest(unittest.TestCase): + """Basic integration tests for sensors endpint.""" + + @classmethod + def setUpClass(cls) -> None: + """Perform one-time setup.""" + + # use local instance of the Epidata API + Epidata.BASE_URL = "http://delphi_web_epidata/epidata/api.php" + + def setUp(self) -> None: + """Perform per-test setup.""" + + # connect to the `epidata` database + cnx = mysql.connector.connect(user="user", password="pass", host="delphi_database_epidata", database="epidata") + cur = cnx.cursor() + + cur.execute("DELETE FROM api_user") + cur.execute("TRUNCATE TABLE user_role") + cur.execute("TRUNCATE TABLE user_role_link") + cur.execute("TRUNCATE TABLE sensors") + + cur.execute('INSERT INTO api_user(api_key, email) VALUES("sensors_key", "sensors_email")') + cur.execute('INSERT INTO user_role(name) VALUES("sensors") ON DUPLICATE KEY UPDATE name="sensors"') + cur.execute( + 'INSERT INTO user_role_link(user_id, role_id) SELECT api_user.id, 1 FROM api_user WHERE api_key="sensors_key"' + ) + + cnx.commit() + cur.close() + + self.cnx = cnx + self.cur = cnx.cursor() + + @staticmethod + def _clear_limits(): + limiter.storage.reset() + + def tearDown(self) -> None: + """Perform per-test teardown.""" + self.cur.close() + self.cnx.close() + self._clear_limits() + + def test_gft(self): + """Basic integration test for sensors endpoint""" + self.cur.execute( + "INSERT INTO sensors(name, epiweek, location, value) VALUES(%s, %s, %s, %s)", + ("sens1", "201111", "loc1", "222"), + ) + self.cnx.commit() + response = Epidata.sensors(names="sens1", locations="loc1", epiweeks="201111", auth="sensors_key") + self.assertEqual( + response, + { + "epidata": [{"name": "sens1", "location": "loc1", "epiweek": 201111, "value": 222.0}], + "result": 1, + "message": "success", + }, + ) diff --git a/integrations/server/test_signal_dashboard.py b/integrations/server/test_signal_dashboard.py new file mode 100644 index 000000000..a442d49a0 --- /dev/null +++ b/integrations/server/test_signal_dashboard.py @@ -0,0 +1,102 @@ +# standard library +import unittest + +# third party +import mysql.connector + +# first party +from delphi.epidata.client.delphi_epidata import Epidata +from delphi.epidata.server._limiter import limiter + + +class SignalDashboardTest(unittest.TestCase): + """Basic integration tests for signal_dashboard_coverage and signal_dashboard_status endpints.""" + + @classmethod + def setUpClass(cls) -> None: + """Perform one-time setup.""" + + # use local instance of the Epidata API + Epidata.BASE_URL = "http://delphi_web_epidata/epidata/api.php" + Epidata.auth = ("epidata", "key") + + def setUp(self) -> None: + """Perform per-test setup.""" + + # connect to the `epidata` database + cnx = mysql.connector.connect(user="user", password="pass", host="delphi_database_epidata", database="epidata") + cur = cnx.cursor() + + cur.execute("DELETE FROM api_user") + + cur.execute("DELETE FROM dashboard_signal_coverage") + cur.execute("DELETE FROM dashboard_signal") + + + cur.execute('INSERT INTO api_user(api_key, email) VALUES("key", "email")') + + cur.execute( + "INSERT INTO dashboard_signal(id, name, source, covidcast_signal, enabled, latest_coverage_update, latest_status_update) VALUES(%s, %s, %s, %s, %s, %s, %s)", + ("1", "Change", "chng", "smoothed_outpatient_covid", "1", "2021-10-02", "2021-11-27"), + ) + cur.execute( + "INSERT INTO dashboard_signal_coverage(signal_id, date, geo_type, count) VALUES(%s, %s, %s, %s)", + ("1", "2021-10-02", "county", "2222"), + ) + + cnx.commit() + cur.close() + + self.cnx = cnx + self.cur = cnx.cursor() + + @staticmethod + def _clear_limits(): + limiter.storage.reset() + + def tearDown(self) -> None: + """Perform per-test teardown.""" + self.cur.close() + self.cnx.close() + self._clear_limits() + + def test_signal_dashboard_coverage(self): + """Basic integration test for signal_dashboard_coverage endpoint""" + + params = { + "endpoint": "signal_dashboard_coverage", + } + response = Epidata._request(params=params) + self.assertEqual( + response, + { + "epidata": {"Change": {"county": [{"count": 2222, "date": "2021-10-02"}]}}, + "message": "success", + "result": 1, + }, + ) + + def test_signal_dashboard_status(self): + """Basic integration test for signal_dashboard_status endpoint""" + + params = { + "endpoint": "signal_dashboard_status", + } + response = Epidata._request(params=params) + self.assertEqual( + response, + { + "epidata": [ + { + "name": "Change", + "source": "chng", + "covidcast_signal": "smoothed_outpatient_covid", + "latest_issue": None, + "latest_time_value": None, + "coverage": {"county": [{"date": "2021-10-02", "count": 2222}]}, + } + ], + "result": 1, + "message": "success", + }, + ) diff --git a/integrations/server/test_twitter.py b/integrations/server/test_twitter.py new file mode 100644 index 000000000..46cda4acc --- /dev/null +++ b/integrations/server/test_twitter.py @@ -0,0 +1,71 @@ +# standard library +import unittest + +# third party +import mysql.connector + +# first party +from delphi.epidata.client.delphi_epidata import Epidata +from delphi.epidata.server._limiter import limiter + + +class TwitterTest(unittest.TestCase): + """Basic integration tests for twitter endpint.""" + + @classmethod + def setUpClass(cls) -> None: + """Perform one-time setup.""" + + # use local instance of the Epidata API + Epidata.BASE_URL = "http://delphi_web_epidata/epidata/api.php" + + def setUp(self) -> None: + """Perform per-test setup.""" + + # connect to the `epidata` database + cnx = mysql.connector.connect(user="user", password="pass", host="delphi_database_epidata", database="epidata") + cur = cnx.cursor() + + cur.execute("DELETE FROM api_user") + cur.execute("TRUNCATE TABLE user_role") + cur.execute("TRUNCATE TABLE user_role_link") + cur.execute("TRUNCATE TABLE twitter") + + cur.execute('INSERT INTO api_user(api_key, email) VALUES("twitter_key", "twitter_email")') + cur.execute('INSERT INTO user_role(name) VALUES("twitter") ON DUPLICATE KEY UPDATE name="twitter"') + cur.execute( + 'INSERT INTO user_role_link(user_id, role_id) SELECT api_user.id, 1 FROM api_user WHERE api_key="twitter_key"' + ) + + cnx.commit() + cur.close() + + self.cnx = cnx + self.cur = cnx.cursor() + + @staticmethod + def _clear_limits(): + limiter.storage.reset() + + def tearDown(self) -> None: + """Perform per-test teardown.""" + self.cur.close() + self.cnx.close() + self._clear_limits() + + def test_twitter(self): + """Basic integration test for twitter endpoint""" + + self.cur.execute( + 'INSERT INTO twitter(date, state, num, total) VALUES ("2015-07-29", "AK", "1", "223"), ("2020-07-29", "CT", "12", "778")', + ) + self.cnx.commit() + response = Epidata.twitter(auth="twitter_key", locations="cen9", dates="20150701-20160101") + self.assertEqual( + response, + { + "epidata": [{"location": "cen9", "date": "2015-07-29", "num": 1, "total": 223, "percent": 0.4484}], + "result": 1, + "message": "success", + }, + ) diff --git a/integrations/server/test_wiki.py b/integrations/server/test_wiki.py new file mode 100644 index 000000000..6763d7d62 --- /dev/null +++ b/integrations/server/test_wiki.py @@ -0,0 +1,72 @@ +# standard library +import unittest + +# third party +import mysql.connector + +# first party +from delphi.epidata.client.delphi_epidata import Epidata +from delphi.epidata.server._limiter import limiter + + +class WikiTest(unittest.TestCase): + """Basic integration tests for wiki endpint.""" + + @classmethod + def setUpClass(cls) -> None: + """Perform one-time setup.""" + + # use local instance of the Epidata API + Epidata.BASE_URL = "http://delphi_web_epidata/epidata/api.php" + Epidata.auth = ("epidata", "key") + + def setUp(self) -> None: + """Perform per-test setup.""" + + # connect to the `epidata` database + cnx = mysql.connector.connect(user="user", password="pass", host="delphi_database_epidata", database="epidata") + cur = cnx.cursor() + + cur.execute("DELETE FROM api_user") + cur.execute("TRUNCATE TABLE wiki") + cur.execute("TRUNCATE TABLE wiki_meta") + + cur.execute('INSERT INTO api_user(api_key, email) VALUES("key", "email")') + + cnx.commit() + cur.close() + + self.cnx = cnx + self.cur = cnx.cursor() + + @staticmethod + def _clear_limits(): + limiter.storage.reset() + + def tearDown(self) -> None: + """Perform per-test teardown.""" + self.cur.close() + self.cnx.close() + self._clear_limits() + + def test_wiki(self): + """Basic integration test for wiki endpoint""" + + self.cur.execute( + 'INSERT INTO wiki(datetime, article, count, language) VALUES ("2007-12-09 18:00:00", "amantadine", "3", "en"), ("2008-12-09 18:00:00", "test", "5", "en")', + ) + self.cur.execute( + 'INSERT INTO wiki_meta(datetime, date, epiweek, total, language) VALUES ("2007-12-09 18:00:00", "2007-12-09", "200750", "969214", "en"), ("2008-12-09 18:00:00", "2008-12-09", "200750", "123321", "en")' + ) + self.cnx.commit() + response = Epidata.wiki(articles="test", epiweeks="200701-200801") + self.assertEqual( + response, + { + "epidata": [ + {"article": "test", "count": 5, "total": 123321, "hour": -1, "epiweek": 200750, "value": 40.544595} + ], + "result": 1, + "message": "success", + }, + ) From 9d99885f5ffe91e7b1ad68eb96e458f1cbd3ad7b Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Fri, 21 Jul 2023 14:40:52 +0300 Subject: [PATCH 09/55] Added base class for tests. Added API keys tests --- dev/local/Makefile | 1 + integrations/server/test_api_keys.py | 243 +++++++++++++++++++ integrations/server/test_cdc.py | 57 +---- integrations/server/test_delphi.py | 46 +--- integrations/server/test_dengue_nowcast.py | 53 +--- integrations/server/test_dengue_sensors.py | 64 +---- integrations/server/test_ecdc_ili.py | 49 +--- integrations/server/test_flusurv.py | 51 +--- integrations/server/test_fluview_clinical.py | 51 +--- integrations/server/test_gft.py | 49 +--- integrations/server/test_ght.py | 57 +---- integrations/server/test_kcdc_ili.py | 49 +--- integrations/server/test_meta.py | 54 +---- integrations/server/test_meta_norostat.py | 90 ++----- integrations/server/test_nidss_dengue.py | 49 +--- integrations/server/test_nidss_flu.py | 49 +--- integrations/server/test_norostat.py | 84 ++----- integrations/server/test_nowcast.py | 49 +--- integrations/server/test_paho_dengue.py | 49 +--- integrations/server/test_quidel.py | 55 +---- integrations/server/test_sensors.py | 57 +---- integrations/server/test_signal_dashboard.py | 60 +---- integrations/server/test_twitter.py | 55 +---- integrations/server/test_wiki.py | 52 +--- src/common/integration_test_base_class.py | 79 ++++++ src/server/_config.py | 8 + 26 files changed, 519 insertions(+), 1041 deletions(-) create mode 100644 integrations/server/test_api_keys.py create mode 100644 src/common/integration_test_base_class.py diff --git a/dev/local/Makefile b/dev/local/Makefile index e7e896aa6..018413f2a 100644 --- a/dev/local/Makefile +++ b/dev/local/Makefile @@ -102,6 +102,7 @@ web: --env "FLASK_SECRET=abc" --env "FLASK_PREFIX=/epidata" --env "LOG_DEBUG" \ --env "REDIS_HOST=delphi_redis" \ --env "REDIS_PASSWORD=1234" \ + --env "TESTING_MODE=True" \ --env "API_KEY_ADMIN_PASSWORD=test_admin_password" \ --env "API_KEY_REGISTER_WEBHOOK_TOKEN=abc" \ --network delphi-net --name delphi_web_epidata \ diff --git a/integrations/server/test_api_keys.py b/integrations/server/test_api_keys.py new file mode 100644 index 000000000..19bc4d3ff --- /dev/null +++ b/integrations/server/test_api_keys.py @@ -0,0 +1,243 @@ +"""Integration tests for the API Keys""" +import requests + +# first party +from delphi.epidata.common.integration_test_base_class import BasicIntegrationTest + + +class APIKeysTets(BasicIntegrationTest): + """Tests the API Keys behaviour""" + + def setUp(self): + """Perform per-test setup.""" + + self.role_name = "cdc" + super().setUp() + + def _make_request(self, url: str = None, params: dict = {}, auth: tuple = None): + if not url: + url = self.epidata.BASE_URL + response = requests.get(url, params=params, auth=auth) + return response + + def test_public_route(self): + """Test public route""" + public_route = "http://delphi_web_epidata/epidata/version" + status_codes = set() + for _ in range(10): + status_codes.add(self._make_request(public_route).status_code) + self.assertEqual(len(status_codes), 1) + self.assertEqual(next(iter(status_codes)), 200) + + def test_no_multiples_data_source(self): + """Test requests with no multiples and with provided `data_source` and `signal` as a separate query params.""" + params = { + "source": "covidcast", + "data_source": "fb-survey", + "signal": "smoothed_wcli", + "time_type": "day", + "geo_type": "state", + "geo_value": "pa", + "time_values": "20200406", + } + status_codes = set() + for _ in range(10): + status_codes.add(self._make_request(params=params).status_code) + self.assertEqual(len(status_codes), 1) + self.assertEqual(next(iter(status_codes)), 200) + + def test_no_multiples_source_signal(self): + """Test requests with colon-delimited source-signal param presentation.""" + params = { + "source": "covidcast", + "signal": "fb-survey:smoothed_wcli", + "time_type": "day", + "geo_type": "state", + "geo_value": "pa", + "time_values": "20200406", + } + status_codes = set() + for _ in range(10): + status_codes.add(self._make_request(params=params).status_code) + self.assertEqual(len(status_codes), 1) + self.assertEqual(next(iter(status_codes)), 200) + + def test_multiples_allowed_signal(self): + """Test requests with 2 multiples and allowed dashboard signal""" + params = { + "source": "covidcast", + "signal": "fb-survey:smoothed_wcli", + "time_type": "day", + "geo_type": "state", + "geo_value": "pa,ny", + "time_values": "20200406,20200407", + } + status_codes = set() + for _ in range(10): + status_codes.add(self._make_request(params=params).status_code) + self.assertEqual(len(status_codes), 1) + self.assertEqual(next(iter(status_codes)), 200) + + def test_multiples_non_allowed_signal(self): + """Test requests with 2 multiples and non-allowed dashboard signal""" + params = { + "source": "covidcast", + "signal": "hospital-admissions:smoothed_adj_covid19_from_claims", + "time_type": "day", + "geo_type": "state", + "geo_value": "pa,ny", + "time_values": "20200406,20200407", + } + status_codes = set() + for _ in range(10): + status_codes.add(self._make_request(params=params).status_code) + self.assertEqual(len(status_codes), 2) + self.assertEqual(status_codes, {200, 429}) + + def test_multiples_mixed_allowed_signal(self): + """Test requests with 2 multiples and mixed-allowed dashboard signal""" + params = { + "source": "covidcast", + "signal": "fb-survey:smoothed_wcli,hospital-admissions:smoothed_adj_covid19_from_claims", + "time_type": "day", + "geo_type": "state", + "geo_value": "pa", + "time_values": "20200406,20200407", + } + status_codes = set() + for _ in range(10): + status_codes.add(self._make_request(params=params).status_code) + self.assertEqual(len(status_codes), 2) + self.assertEqual(status_codes, {200, 429}) + + def test_multiples_allowed_signal(self): + """Test requests with 3 multiples and allowed dashboard signal""" + params = { + "source": "covidcast", + "signal": "fb-survey:smoothed_wcli,fb-survey:smoothed_wcli", + "time_type": "day", + "geo_type": "state", + "geo_value": "pa,ny", + "time_values": "20200406,20200407", + } + status_codes = set() + for _ in range(10): + status_codes.add(self._make_request(params=params).status_code) + self.assertEqual(len(status_codes), 1) + self.assertEqual(next(iter(status_codes)), 401) + + def test_multiples_mixed_allowed_signal(self): + """Test requests with 3 multiples and mixed-allowed dashboard signal""" + params = { + "source": "covidcast", + "signal": "fb-survey:smoothed_wcli,fb-survey:smoothed_wcli1", + "time_type": "day", + "geo_type": "state", + "geo_value": "pa,ny", + "time_values": "20200406,20200407", + } + status_codes = set() + for _ in range(10): + status_codes.add(self._make_request(params=params).status_code) + self.assertEqual(len(status_codes), 1) + self.assertEqual(next(iter(status_codes)), 401) + + def test_multiples_mixed_allowed_signal_api_key(self): + """Test requests with 3 multiples and mixed-allowed dashboard signal + valid API Key""" + params = { + "source": "covidcast", + "signal": "fb-survey:smoothed_wcli,fb-survey:smoothed_wcli1", + "time_type": "day", + "geo_type": "state", + "geo_value": "pa,ny", + "time_values": "20200406,20200407", + } + status_codes = set() + for _ in range(10): + status_codes.add(self._make_request(params=params, auth=self.epidata.auth).status_code) + self.assertEqual(len(status_codes), 1) + self.assertEqual(next(iter(status_codes)), 200) + + def test_multiples_allowed_signal_api_key(self): + """Test requests with 3 multiples and allowed dashboard signal + valid API Key""" + params = { + "source": "covidcast", + "signal": "fb-survey:smoothed_wcli,fb-survey:smoothed_wcli", + "time_type": "day", + "geo_type": "state", + "geo_value": "pa,ny", + "time_values": "20200406,20200407", + } + status_codes = set() + for _ in range(10): + status_codes.add(self._make_request(params=params, auth=self.epidata.auth).status_code) + self.assertEqual(len(status_codes), 1) + self.assertEqual(next(iter(status_codes)), 200) + + def test_no_multiples_allowed_signal_api_key(self): + """Test requests with no multiples and allowed dashboard signal + valid API Key""" + params = { + "source": "covidcast", + "signal": "fb-survey:smoothed_wcli", + "time_type": "day", + "geo_type": "state", + "geo_value": "pa", + "time_values": "20200406", + } + status_codes = set() + for _ in range(10): + status_codes.add(self._make_request(params=params, auth=self.epidata.auth).status_code) + self.assertEqual(len(status_codes), 1) + self.assertEqual(next(iter(status_codes)), 200) + + def test_no_multiples_allowed_signal_bad_api_key(self): + """Test requests with no multiples and allowed dashboard signal + bad API Key""" + params = { + "source": "covidcast", + "signal": "fb-survey:smoothed_wcli", + "time_type": "day", + "geo_type": "state", + "geo_value": "pa", + "time_values": "20200406", + } + status_codes = set() + for _ in range(10): + status_codes.add(self._make_request(params=params, auth=("bad_key", "bad_email")).status_code) + self.assertEqual(len(status_codes), 1) + self.assertEqual(next(iter(status_codes)), 200) + + def test_restricted_endpoint_no_key(self): + """Test restricted endpoint with no auth key""" + params = {"source": "cdc", "regions": "1as", "epiweeks": "202020"} + status_codes = set() + for _ in range(10): + status_codes.add(self._make_request(params=params).status_code) + self.assertEqual(len(status_codes), 1) + self.assertEqual(next(iter(status_codes)), 401) + + def test_restricted_endpoint_invalid_key(self): + """Test restricted endpoint with invalid auth key""" + params = {"source": "cdc", "regions": "1as", "epiweeks": "202020", "auth": "invalid_key"} + status_codes = set() + for _ in range(10): + status_codes.add(self._make_request(params=params).status_code) + self.assertEqual(len(status_codes), 1) + self.assertEqual(next(iter(status_codes)), 401) + + def test_restricted_endpoint_no_roles_key(self): + """Test restricted endpoint with no roles key""" + params = {"source": "cdc", "regions": "1as", "epiweeks": "202020", "auth": "key"} + status_codes = set() + for _ in range(10): + status_codes.add(self._make_request(params=params).status_code) + self.assertEqual(len(status_codes), 1) + self.assertEqual(next(iter(status_codes)), 401) + + def test_restricted_endpoint_valid_roles_key(self): + """Test restricted endpoint with valid auth key with required role""" + params = {"source": "cdc", "regions": "1as", "epiweeks": "202020", "auth": "cdc_key"} + status_codes = set() + for _ in range(10): + status_codes.add(self._make_request(params=params).status_code) + self.assertEqual(len(status_codes), 1) + self.assertEqual(next(iter(status_codes)), 200) diff --git a/integrations/server/test_cdc.py b/integrations/server/test_cdc.py index 2b81d562c..7234f9eb0 100644 --- a/integrations/server/test_cdc.py +++ b/integrations/server/test_cdc.py @@ -1,66 +1,23 @@ -# standard library -import unittest - -# third party -import mysql.connector - # first party -from delphi.epidata.client.delphi_epidata import Epidata -from delphi.epidata.server._limiter import limiter +from delphi.epidata.common.integration_test_base_class import BasicIntegrationTest -class CdcTest(unittest.TestCase): +class CdcTest(BasicIntegrationTest): """Basic integration tests for cdc endpint.""" - @classmethod - def setUpClass(cls) -> None: - """Perform one-time setup.""" - - # use local instance of the Epidata API - Epidata.BASE_URL = "http://delphi_web_epidata/epidata/api.php" - def setUp(self) -> None: - """Perform per-test setup.""" - - # connect to the `epidata` database - cnx = mysql.connector.connect(user="user", password="pass", host="delphi_database_epidata", database="epidata") - cur = cnx.cursor() - - cur.execute("DELETE FROM api_user") - cur.execute("TRUNCATE TABLE user_role") - cur.execute("TRUNCATE TABLE user_role_link") - cur.execute("TRUNCATE TABLE cdc_extract") - - cur.execute('INSERT INTO api_user(api_key, email) VALUES("cdc_key", "cdc_email")') - cur.execute('INSERT INTO user_role(name) VALUES("cdc") ON DUPLICATE KEY UPDATE name="cdc"') - cur.execute( - 'INSERT INTO user_role_link(user_id, role_id) SELECT api_user.id, 1 FROM api_user WHERE api_key="cdc_key"' - ) - - cnx.commit() - cur.close() - - self.cnx = cnx - self.cur = cnx.cursor() - - @staticmethod - def _clear_limits(): - limiter.storage.reset() - - def tearDown(self) -> None: - """Perform per-test teardown.""" - self.cur.close() - self.cnx.close() - self._clear_limits() + self.truncate_tables_list = ["cdc_extract"] + self.role_name = "cdc" + super().setUp() def test_cdc(self): """Basic integration test for cdc endpoint""" self.cur.execute( - "INSERT INTO cdc_extract(epiweek, state, num1, num2, num3, num4, num5, num6, num7, num8, total) VALUES(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)", + "INSERT INTO `cdc_extract`(`epiweek`, `state`, `num1`, `num2`, `num3`, `num4`, `num5`, `num6`, `num7`, `num8`, `total`) VALUES(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)", ("201102", "AK", "16", "35", "51", "96", "30", "748", "243", "433", "65"), ) self.cnx.commit() - response = Epidata.cdc(auth="cdc_key", epiweeks=201102, locations="cen9") + response = self.epidata.cdc(auth="cdc_key", epiweeks=201102, locations="cen9") self.assertEqual( response, { diff --git a/integrations/server/test_delphi.py b/integrations/server/test_delphi.py index ee6780cd9..481b9d3f5 100644 --- a/integrations/server/test_delphi.py +++ b/integrations/server/test_delphi.py @@ -2,57 +2,23 @@ import unittest # third party -import mysql.connector import json # first party -from delphi.epidata.client.delphi_epidata import Epidata -from delphi.epidata.server._limiter import limiter +from delphi.epidata.common.integration_test_base_class import BasicIntegrationTest -class DelphiTest(unittest.TestCase): +class DelphiTest(BasicIntegrationTest): """Basic integration tests for delphi endpint.""" - @classmethod - def setUpClass(cls) -> None: - """Perform one-time setup.""" - - # use local instance of the Epidata API - Epidata.BASE_URL = "http://delphi_web_epidata/epidata/api.php" - Epidata.auth = ("epidata", "key") - def setUp(self) -> None: - """Perform per-test setup.""" - - # connect to the `epidata` database - cnx = mysql.connector.connect(user="user", password="pass", host="delphi_database_epidata", database="epidata") - cur = cnx.cursor() - - cur.execute("DELETE FROM api_user") - cur.execute("TRUNCATE TABLE forecasts") - - cur.execute('INSERT INTO api_user(api_key, email) VALUES("key", "email")') - - cnx.commit() - cur.close() - - self.cnx = cnx - self.cur = cnx.cursor() - - @staticmethod - def _clear_limits(): - limiter.storage.reset() - - def tearDown(self) -> None: - """Perform per-test teardown.""" - self.cur.close() - self.cnx.close() - self._clear_limits() + self.truncate_tables_list = ["forecasts"] + super().setUp() def test_delphi(self): """Basic integration test for delphi endpoint""" self.cur.execute( - "INSERT INTO forecasts (`system`, `epiweek`, `json`) VALUES(%s, %s, %s)", + "INSERT INTO `forecasts` (`system`, `epiweek`, `json`) VALUES(%s, %s, %s)", ( "eb", "201441", @@ -71,7 +37,7 @@ def test_delphi(self): ), ) self.cnx.commit() - response = Epidata.delphi(system="eb", epiweek=201441) + response = self.epidata.delphi(system="eb", epiweek=201441) self.assertEqual( response, { diff --git a/integrations/server/test_dengue_nowcast.py b/integrations/server/test_dengue_nowcast.py index de40ae5c3..dea91979d 100644 --- a/integrations/server/test_dengue_nowcast.py +++ b/integrations/server/test_dengue_nowcast.py @@ -1,34 +1,14 @@ -# standard library -import unittest - -# third party -import mysql.connector - # first party -from delphi.epidata.client.delphi_epidata import Epidata -from delphi.epidata.server._limiter import limiter +from delphi.epidata.common.integration_test_base_class import BasicIntegrationTest -class DengueNowcastTest(unittest.TestCase): +class DengueNowcastTest(BasicIntegrationTest): """Basic integration tests for dengue_nowcast endpint.""" - @classmethod - def setUpClass(cls) -> None: - """Perform one-time setup.""" - - # use local instance of the Epidata API - Epidata.BASE_URL = "http://delphi_web_epidata/epidata/api.php" - Epidata.auth = ("epidata", "key") - def setUp(self) -> None: """Perform per-test setup.""" - # connect to the `epidata` database - cnx = mysql.connector.connect(user="user", password="pass", host="delphi_database_epidata", database="epidata") - cur = cnx.cursor() - - cur.execute( - """ + create_dengue_nowcasts = """ CREATE TABLE IF NOT EXISTS `dengue_nowcasts` ( `id` int NOT NULL AUTO_INCREMENT, `target` varchar(32) CHARACTER SET utf8mb3 COLLATE utf8mb3_general_ci NOT NULL, @@ -43,28 +23,9 @@ def setUp(self) -> None: KEY `location` (`location`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3; """ - ) - - cur.execute("DELETE FROM api_user") - cur.execute("TRUNCATE TABLE dengue_nowcasts") - - cur.execute('INSERT INTO api_user(api_key, email) VALUES("key", "email")') - - cnx.commit() - cur.close() - - self.cnx = cnx - self.cur = cnx.cursor() - - @staticmethod - def _clear_limits(): - limiter.storage.reset() - - def tearDown(self) -> None: - """Perform per-test teardown.""" - self.cur.close() - self.cnx.close() - self._clear_limits() + self.create_tables_list = [create_dengue_nowcasts] + self.truncate_tables_list = ["dengue_nowcasts"] + super().setUp() def test_dengue_nowcasts(self): """Basic integration test for dengue_nowcasts endpoint""" @@ -73,7 +34,7 @@ def test_dengue_nowcasts(self): ("num_dengue", "201409", "ar", "85263", "351456"), ) self.cnx.commit() - response = Epidata.dengue_nowcast(locations="ar", epiweeks=201409) + response = self.epidata.dengue_nowcast(locations="ar", epiweeks=201409) self.assertEqual( response, { diff --git a/integrations/server/test_dengue_sensors.py b/integrations/server/test_dengue_sensors.py index 5a690969f..75ff9028a 100644 --- a/integrations/server/test_dengue_sensors.py +++ b/integrations/server/test_dengue_sensors.py @@ -1,33 +1,14 @@ -# standard library -import unittest - -# third party -import mysql.connector - # first party -from delphi.epidata.client.delphi_epidata import Epidata -from delphi.epidata.server._limiter import limiter +from delphi.epidata.common.integration_test_base_class import BasicIntegrationTest -class DengueSensorsTest(unittest.TestCase): +class DengueSensorsTest(BasicIntegrationTest): """Basic integration tests for dengue_sensors endpint.""" - @classmethod - def setUpClass(cls) -> None: - """Perform one-time setup.""" - - # use local instance of the Epidata API - Epidata.BASE_URL = "http://delphi_web_epidata/epidata/api.php" - def setUp(self) -> None: - """Perform per-test setup.""" + """Perfor per-test setup.""" - # connect to the `epidata` database - cnx = mysql.connector.connect(user="user", password="pass", host="delphi_database_epidata", database="epidata") - cur = cnx.cursor() - - cur.execute( - """ + create_dengue_sensors = """ CREATE TABLE IF NOT EXISTS `dengue_sensors` ( `id` int NOT NULL AUTO_INCREMENT, `target` varchar(32) CHARACTER SET utf8mb3 COLLATE utf8mb3_general_ci NOT NULL, @@ -42,44 +23,19 @@ def setUp(self) -> None: KEY `location` (`location`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3; """ - ) - - cur.execute("TRUNCATE TABLE dengue_sensors") - - cur.execute("DELETE FROM api_user") - cur.execute("TRUNCATE TABLE user_role") - cur.execute("TRUNCATE TABLE user_role_link") - - cur.execute('INSERT INTO api_user(api_key, email) VALUES("sensors_key", "sensors_email")') - cur.execute('INSERT INTO user_role(name) VALUES("sensors") ON DUPLICATE KEY UPDATE name="sensors"') - cur.execute( - 'INSERT INTO user_role_link(user_id, role_id) SELECT api_user.id, 1 FROM api_user WHERE api_key="sensors_key"' - ) - - cnx.commit() - cur.close() - - self.cnx = cnx - self.cur = cnx.cursor() - - @staticmethod - def _clear_limits(): - limiter.storage.reset() - - def tearDown(self) -> None: - """Perform per-test teardown.""" - self.cur.close() - self.cnx.close() - self._clear_limits() + self.create_tables_list = [create_dengue_sensors] + self.truncate_tables_list = ["dengue_sensors"] + self.role_name = "sensors" + super().setUp() def test_dengue_sensors(self): """Basic integration test for dengue_sensors endpoint""" self.cur.execute( - "INSERT INTO dengue_sensors(target, name, epiweek, location, value) VALUES(%s, %s, %s, %s, %s)", + "INSERT INTO `dengue_sensors`(`target`, `name`, `epiweek`, `location`, `value`) VALUES(%s, %s, %s, %s, %s)", ("num_dengue", "ght", "201432", "ag", "1234"), ) self.cnx.commit() - response = Epidata.dengue_sensors(auth="sensors_key", names="ght", locations="ag", epiweeks="201432") + response = self.epidata.dengue_sensors(auth="sensors_key", names="ght", locations="ag", epiweeks="201432") self.assertEqual( response, { diff --git a/integrations/server/test_ecdc_ili.py b/integrations/server/test_ecdc_ili.py index d68bbe7a0..3c65cb2c2 100644 --- a/integrations/server/test_ecdc_ili.py +++ b/integrations/server/test_ecdc_ili.py @@ -1,61 +1,24 @@ -# standard library -import unittest - -# third party -import mysql.connector - # first party -from delphi.epidata.client.delphi_epidata import Epidata -from delphi.epidata.server._limiter import limiter +from delphi.epidata.common.integration_test_base_class import BasicIntegrationTest -class EcdcIliTest(unittest.TestCase): +class EcdcIliTest(BasicIntegrationTest): """Basic integration tests for edcd_ili endpint.""" - @classmethod - def setUpClass(cls) -> None: - """Perform one-time setup.""" - - # use local instance of the Epidata API - Epidata.BASE_URL = "http://delphi_web_epidata/epidata/api.php" - Epidata.auth = ("epidata", "key") - def setUp(self) -> None: """Perform per-test setup.""" - # connect to the `epidata` database - cnx = mysql.connector.connect(user="user", password="pass", host="delphi_database_epidata", database="epidata") - cur = cnx.cursor() - - cur.execute("DELETE FROM api_user") - cur.execute("TRUNCATE TABLE ecdc_ili") - - cur.execute('INSERT INTO api_user(api_key, email) VALUES("key", "email")') - - cnx.commit() - cur.close() - - self.cnx = cnx - self.cur = cnx.cursor() - - @staticmethod - def _clear_limits(): - limiter.storage.reset() - - def tearDown(self) -> None: - """Perform per-test teardown.""" - self.cur.close() - self.cnx.close() - self._clear_limits() + self.truncate_tables_list = ["ecdc_ili"] + super().setUp() def test_ecdc_ili(self): """Basic integration test for ecdc_ili endpoint""" self.cur.execute( - "INSERT INTO ecdc_ili(`release_date`, `issue`, `epiweek`, `lag`, `region`, `incidence_rate`) VALUES(%s, %s, %s, %s, %s, %s)", + "INSERT INTO `ecdc_ili`(`release_date`, `issue`, `epiweek`, `lag`, `region`, `incidence_rate`) VALUES(%s, %s, %s, %s, %s, %s)", ("2020-03-26", "202012", "201840", "76", "Armenia", "0"), ) self.cnx.commit() - response = Epidata.ecdc_ili(regions="Armenia", epiweeks="201840") + response = self.epidata.ecdc_ili(regions="Armenia", epiweeks="201840") self.assertEqual( response, { diff --git a/integrations/server/test_flusurv.py b/integrations/server/test_flusurv.py index f6fd4bc16..e818a2698 100644 --- a/integrations/server/test_flusurv.py +++ b/integrations/server/test_flusurv.py @@ -1,61 +1,24 @@ -# standard library -import unittest - -# third party -import mysql.connector - # first party -from delphi.epidata.client.delphi_epidata import Epidata -from delphi.epidata.server._limiter import limiter +from delphi.epidata.common.integration_test_base_class import BasicIntegrationTest -class FlusurvTest(unittest.TestCase): +class FlusurvTest(BasicIntegrationTest): """Basic integration tests for flusurv endpint.""" - @classmethod - def setUpClass(cls) -> None: - """Perform one-time setup.""" - - # use local instance of the Epidata API - Epidata.BASE_URL = "http://delphi_web_epidata/epidata/api.php" - Epidata.auth = ("epidata", "key") - def setUp(self) -> None: """Perform per-test setup.""" - # connect to the `epidata` database - cnx = mysql.connector.connect(user="user", password="pass", host="delphi_database_epidata", database="epidata") - cur = cnx.cursor() - - cur.execute("DELETE FROM api_user") - cur.execute("TRUNCATE TABLE flusurv") - - cur.execute('INSERT INTO api_user(api_key, email) VALUES("key", "email")') - - cnx.commit() - cur.close() - - self.cnx = cnx - self.cur = cnx.cursor() - - @staticmethod - def _clear_limits(): - limiter.storage.reset() - - def tearDown(self) -> None: - """Perform per-test teardown.""" - self.cur.close() - self.cnx.close() - self._clear_limits() + self.truncate_tables_list = ["flusurv"] + super().setUp() - def test_ecdc_ili(self): + def test_flusurv(self): """Basic integration test for flusurv endpoint""" self.cur.execute( - "INSERT INTO flusurv(`release_date`, `issue`, `epiweek`, `location`, `lag`, `rate_age_0`, `rate_age_1`, `rate_age_2`, `rate_age_3`, `rate_age_4`, `rate_overall`, `rate_age_5`, `rate_age_6`, `rate_age_7`) VALUES(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)", + "INSERT INTO `flusurv`(`release_date`, `issue`, `epiweek`, `location`, `lag`, `rate_age_0`, `rate_age_1`, `rate_age_2`, `rate_age_3`, `rate_age_4`, `rate_overall`, `rate_age_5`, `rate_age_6`, `rate_age_7`) VALUES(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)", ("2012-11-02", "201243", "201143", "CA", "52", "0", "0", "0", "0.151", "0", "0.029", "0", "0", "0"), ) self.cnx.commit() - response = Epidata.flusurv(epiweeks=201143, locations="CA") + response = self.epidata.flusurv(epiweeks=201143, locations="CA") self.assertEqual( response, { diff --git a/integrations/server/test_fluview_clinical.py b/integrations/server/test_fluview_clinical.py index 5e87dde10..77f0d76ea 100644 --- a/integrations/server/test_fluview_clinical.py +++ b/integrations/server/test_fluview_clinical.py @@ -1,61 +1,24 @@ -# standard library -import unittest - -# third party -import mysql.connector - # first party -from delphi.epidata.client.delphi_epidata import Epidata -from delphi.epidata.server._limiter import limiter +from delphi.epidata.common.integration_test_base_class import BasicIntegrationTest -class FluviewClinicalTest(unittest.TestCase): +class FluviewClinicalTest(BasicIntegrationTest): """Basic integration tests for fluview_clinical endpint.""" - @classmethod - def setUpClass(cls) -> None: - """Perform one-time setup.""" - - # use local instance of the Epidata API - Epidata.BASE_URL = "http://delphi_web_epidata/epidata/api.php" - Epidata.auth = ("epidata", "key") - def setUp(self) -> None: """Perform per-test setup.""" - # connect to the `epidata` database - cnx = mysql.connector.connect(user="user", password="pass", host="delphi_database_epidata", database="epidata") - cur = cnx.cursor() - - cur.execute("DELETE FROM api_user") - cur.execute("TRUNCATE TABLE fluview_clinical") - - cur.execute('INSERT INTO api_user(api_key, email) VALUES("key", "email")') - - cnx.commit() - cur.close() - - self.cnx = cnx - self.cur = cnx.cursor() - - @staticmethod - def _clear_limits(): - limiter.storage.reset() - - def tearDown(self) -> None: - """Perform per-test teardown.""" - self.cur.close() - self.cnx.close() - self._clear_limits() + self.truncate_tables_list = ["fluview_clinical"] + super().setUp() - def test_ecdc_ili(self): + def test_fluview_clinical(self): """Basic integration test for fluview_clinical endpoint""" self.cur.execute( - "INSERT INTO fluview_clinical(`release_date`, `issue`, `epiweek`, `region`, `lag`, `total_specimens`, `total_a`, `total_b`, `percent_positive`, `percent_a`, `percent_b`) VALUES(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)", + "INSERT INTO `fluview_clinical`(`release_date`, `issue`, `epiweek`, `region`, `lag`, `total_specimens`, `total_a`, `total_b`, `percent_positive`, `percent_a`, `percent_b`) VALUES(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)", ("2018-10-10", "201839", "201640", "al", "103", "406", "4", "1", "1.32", "0.99", "0.25"), ) self.cnx.commit() - response = Epidata.fluview_clinical(epiweeks=201640, regions="al") + response = self.epidata.fluview_clinical(epiweeks=201640, regions="al") self.assertEqual( response, { diff --git a/integrations/server/test_gft.py b/integrations/server/test_gft.py index ee5d8365b..f241769f4 100644 --- a/integrations/server/test_gft.py +++ b/integrations/server/test_gft.py @@ -1,61 +1,24 @@ -# standard library -import unittest - -# third party -import mysql.connector - # first party -from delphi.epidata.client.delphi_epidata import Epidata -from delphi.epidata.server._limiter import limiter +from delphi.epidata.common.integration_test_base_class import BasicIntegrationTest -class GftTest(unittest.TestCase): +class GftTest(BasicIntegrationTest): """Basic integration tests for gft endpint.""" - @classmethod - def setUpClass(cls) -> None: - """Perform one-time setup.""" - - # use local instance of the Epidata API - Epidata.BASE_URL = "http://delphi_web_epidata/epidata/api.php" - Epidata.auth = ("epidata", "key") - def setUp(self) -> None: """Perform per-test setup.""" - # connect to the `epidata` database - cnx = mysql.connector.connect(user="user", password="pass", host="delphi_database_epidata", database="epidata") - cur = cnx.cursor() - - cur.execute("DELETE FROM api_user") - cur.execute("TRUNCATE TABLE gft") - - cur.execute('INSERT INTO api_user(api_key, email) VALUES("key", "email")') - - cnx.commit() - cur.close() - - self.cnx = cnx - self.cur = cnx.cursor() - - @staticmethod - def _clear_limits(): - limiter.storage.reset() - - def tearDown(self) -> None: - """Perform per-test teardown.""" - self.cur.close() - self.cnx.close() - self._clear_limits() + self.truncate_tables_list = ["gft"] + super().setUp() def test_gft(self): """Basic integration test for gft endpoint""" self.cur.execute( - "INSERT INTO gft(epiweek, location, num) VALUES(%s, %s, %s)", + "INSERT INTO `gft`(`epiweek`, `location`, `num`) VALUES(%s, %s, %s)", ("200340", "nat", "902"), ) self.cnx.commit() - response = Epidata.gft(locations="nat", epiweeks="200340") + response = self.epidata.gft(locations="nat", epiweeks="200340") self.assertEqual( response, {"epidata": [{"location": "nat", "epiweek": 200340, "num": 902}], "result": 1, "message": "success"}, diff --git a/integrations/server/test_ght.py b/integrations/server/test_ght.py index 595ddc700..a8ea6f33d 100644 --- a/integrations/server/test_ght.py +++ b/integrations/server/test_ght.py @@ -1,66 +1,25 @@ -# standard library -import unittest - -# third party -import mysql.connector - # first party -from delphi.epidata.client.delphi_epidata import Epidata -from delphi.epidata.server._limiter import limiter - +from delphi.epidata.common.integration_test_base_class import BasicIntegrationTest -class GhtTest(unittest.TestCase): - """Basic integration tests for gft endpint.""" - @classmethod - def setUpClass(cls) -> None: - """Perform one-time setup.""" - - # use local instance of the Epidata API - Epidata.BASE_URL = "http://delphi_web_epidata/epidata/api.php" +class GhtTest(BasicIntegrationTest): + """Basic integration tests for ght endpint.""" def setUp(self) -> None: """Perform per-test setup.""" - # connect to the `epidata` database - cnx = mysql.connector.connect(user="user", password="pass", host="delphi_database_epidata", database="epidata") - cur = cnx.cursor() - - cur.execute("DELETE FROM api_user") - cur.execute("TRUNCATE TABLE user_role") - cur.execute("TRUNCATE TABLE user_role_link") - cur.execute("TRUNCATE TABLE ght") - - cur.execute('INSERT INTO api_user(api_key, email) VALUES("ght_key", "ght_email")') - cur.execute('INSERT INTO user_role(name) VALUES("ght") ON DUPLICATE KEY UPDATE name="ght"') - cur.execute( - 'INSERT INTO user_role_link(user_id, role_id) SELECT api_user.id, 1 FROM api_user WHERE api_key="ght_key"' - ) - - cnx.commit() - cur.close() - - self.cnx = cnx - self.cur = cnx.cursor() - - @staticmethod - def _clear_limits(): - limiter.storage.reset() - - def tearDown(self) -> None: - """Perform per-test teardown.""" - self.cur.close() - self.cnx.close() - self._clear_limits() + self.truncate_tables_list = ["ght"] + self.role_name = "ght" + super().setUp() def test_ght(self): """Basic integration test for ght endpoint""" self.cur.execute( - "INSERT INTO ght(`query`, `location`, `epiweek`, `value`) VALUES(%s, %s, %s, %s)", + "INSERT INTO `ght`(`query`, `location`, `epiweek`, `value`) VALUES(%s, %s, %s, %s)", ("/n/query", "US", "200101", "12345"), ) self.cnx.commit() - response = Epidata.ght(locations="US", epiweeks="200101", query="/n/query", auth="ght_key") + response = self.epidata.ght(locations="US", epiweeks="200101", query="/n/query", auth="ght_key") self.assertEqual( response, {"epidata": [{"location": "US", "epiweek": 200101, "value": 12345.0}], "result": 1, "message": "success"}, diff --git a/integrations/server/test_kcdc_ili.py b/integrations/server/test_kcdc_ili.py index c337db29f..5474c28b9 100644 --- a/integrations/server/test_kcdc_ili.py +++ b/integrations/server/test_kcdc_ili.py @@ -1,61 +1,24 @@ -# standard library -import unittest - -# third party -import mysql.connector - # first party -from delphi.epidata.client.delphi_epidata import Epidata -from delphi.epidata.server._limiter import limiter +from delphi.epidata.common.integration_test_base_class import BasicIntegrationTest -class KcdcIliTest(unittest.TestCase): +class KcdcIliTest(BasicIntegrationTest): """Basic integration tests for kcdc_ili endpint.""" - @classmethod - def setUpClass(cls) -> None: - """Perform one-time setup.""" - - # use local instance of the Epidata API - Epidata.BASE_URL = "http://delphi_web_epidata/epidata/api.php" - Epidata.auth = ("epidata", "key") - def setUp(self) -> None: """Perform per-test setup.""" - # connect to the `epidata` database - cnx = mysql.connector.connect(user="user", password="pass", host="delphi_database_epidata", database="epidata") - cur = cnx.cursor() - - cur.execute("DELETE FROM api_user") - cur.execute("TRUNCATE TABLE kcdc_ili") - - cur.execute('INSERT INTO api_user(api_key, email) VALUES("key", "email")') - - cnx.commit() - cur.close() - - self.cnx = cnx - self.cur = cnx.cursor() - - @staticmethod - def _clear_limits(): - limiter.storage.reset() - - def tearDown(self) -> None: - """Perform per-test teardown.""" - self.cur.close() - self.cnx.close() - self._clear_limits() + self.truncate_tables_list = ["kcdc_ili"] + super().setUp() def test_kcdc_ili(self): """Basic integration test for kcdc_ili endpoint""" self.cur.execute( - "INSERT INTO kcdc_ili(`release_date`, `issue`, `epiweek`, `lag`, `region`, `ili`) VALUES(%s, %s, %s, %s, %s, %s)", + "INSERT INTO `kcdc_ili`(`release_date`, `issue`, `epiweek`, `lag`, `region`, `ili`) VALUES(%s, %s, %s, %s, %s, %s)", ("2020-03-27", "202013", "200432", "222", "REG", "0.25"), ) self.cnx.commit() - response = Epidata.kcdc_ili(regions="REG", epiweeks="200432") + response = self.epidata.kcdc_ili(regions="REG", epiweeks="200432") self.assertEqual( response, { diff --git a/integrations/server/test_meta.py b/integrations/server/test_meta.py index 45cbff125..c32214222 100644 --- a/integrations/server/test_meta.py +++ b/integrations/server/test_meta.py @@ -1,60 +1,18 @@ -# standard library -import unittest - -# third party -import mysql.connector - # first party -from delphi.epidata.client.delphi_epidata import Epidata -from delphi.epidata.server._limiter import limiter +from delphi.epidata.common.integration_test_base_class import BasicIntegrationTest -class MetaTest(unittest.TestCase): +class MetaTest(BasicIntegrationTest): """Basic integration tests for meta endpint.""" - @classmethod - def setUpClass(cls) -> None: - """Perform one-time setup.""" - - # use local instance of the Epidata API - Epidata.BASE_URL = "http://delphi_web_epidata/epidata/api.php" - Epidata.auth = ("epidata", "key") - def setUp(self) -> None: - """Perform per-test setup.""" - - # connect to the `epidata` database - cnx = mysql.connector.connect(user="user", password="pass", host="delphi_database_epidata", database="epidata") - cur = cnx.cursor() - - cur.execute("DELETE FROM api_user") - cur.execute("TRUNCATE TABLE forecasts") - cur.execute("TRUNCATE TABLE fluview") - cur.execute("TRUNCATE TABLE wiki") - cur.execute("TRUNCATE TABLE wiki_meta") - cur.execute("TRUNCATE TABLE twitter") - - cur.execute('INSERT INTO api_user(api_key, email) VALUES("key", "email")') - - cnx.commit() - cur.close() - - self.cnx = cnx - self.cur = cnx.cursor() - - @staticmethod - def _clear_limits(): - limiter.storage.reset() - - def tearDown(self) -> None: - """Perform per-test teardown.""" - self.cur.close() - self.cnx.close() - self._clear_limits() + self.truncate_tables_list = ["forecasts", "fluview", "wiki", "wiki_meta", "twitter"] + super().setUp() def test_meta(self): + """Basic integration test for meta endpoint""" - response = Epidata.meta() + response = self.epidata.meta() self.assertEqual( response, { diff --git a/integrations/server/test_meta_norostat.py b/integrations/server/test_meta_norostat.py index 5462af750..b75b86a1e 100644 --- a/integrations/server/test_meta_norostat.py +++ b/integrations/server/test_meta_norostat.py @@ -1,45 +1,22 @@ -# standard library -import unittest - -# third party -import mysql.connector - # first party -from delphi.epidata.client.delphi_epidata import Epidata -from delphi.epidata.server._limiter import limiter +from delphi.epidata.common.integration_test_base_class import BasicIntegrationTest -class MetaNorostatTest(unittest.TestCase): +class MetaNorostatTest(BasicIntegrationTest): """Basic integration tests for meta_norostat endpint.""" - @classmethod - def setUpClass(cls) -> None: - """Perform one-time setup.""" - - # use local instance of the Epidata API - Epidata.BASE_URL = "http://delphi_web_epidata/epidata/api.php" - def setUp(self) -> None: """Perform per-test setup.""" - # connect to the `epidata` database - cnx = mysql.connector.connect(user="user", password="pass", host="delphi_database_epidata", database="epidata") - cur = cnx.cursor() - - cur.execute( - """ + create_raw_datatable_version_list = """ CREATE TABLE IF NOT EXISTS `norostat_raw_datatable_version_list` ( `release_date` date NOT NULL, `parse_time` datetime NOT NULL, PRIMARY KEY (`release_date`,`parse_time`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3; """ - ) - - # cnx.commit() - cur.execute( - """ + create_norostat_version_list = """ CREATE TABLE IF NOT EXISTS `norostat_point_version_list` ( `release_date` date NOT NULL, `parse_time` datetime NOT NULL, @@ -47,12 +24,8 @@ def setUp(self) -> None: CONSTRAINT `norostat_point_version_list_ibfk_1` FOREIGN KEY (`release_date`, `parse_time`) REFERENCES `norostat_raw_datatable_version_list` (`release_date`, `parse_time`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3; """ - ) - # cnx.commit() - - cur.execute( - """ + create_norostat_datatable_location_pool = """ CREATE TABLE IF NOT EXISTS `norostat_raw_datatable_location_pool` ( `location_id` int NOT NULL AUTO_INCREMENT, `location` varchar(255) CHARACTER SET utf8mb3 COLLATE utf8mb3_general_ci NOT NULL, @@ -60,12 +33,8 @@ def setUp(self) -> None: UNIQUE KEY `location` (`location`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3; """ - ) - # cnx.commit() - - cur.execute( - """ + create_norostat_point_diffs = """ CREATE TABLE IF NOT EXISTS `norostat_point_diffs` ( `release_date` date NOT NULL, `parse_time` datetime NOT NULL, @@ -78,50 +47,37 @@ def setUp(self) -> None: CONSTRAINT `norostat_point_diffs_ibfk_2` FOREIGN KEY (`location_id`) REFERENCES `norostat_raw_datatable_location_pool` (`location_id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3; """ - ) - - cur.execute("DELETE FROM api_user") - cur.execute("TRUNCATE TABLE user_role") - cur.execute("TRUNCATE TABLE user_role_link") - cur.execute("DELETE FROM norostat_point_diffs") - cur.execute("DELETE FROM norostat_point_version_list") - cur.execute("DELETE FROM norostat_raw_datatable_location_pool") - cur.execute("DELETE FROM norostat_raw_datatable_version_list") - - cur.execute('INSERT INTO api_user(api_key, email) VALUES("norostat_key", "norostat_email")') - cur.execute('INSERT INTO user_role(name) VALUES("norostat") ON DUPLICATE KEY UPDATE name="norostat"') - cur.execute( - 'INSERT INTO user_role_link(user_id, role_id) SELECT api_user.id, 1 FROM api_user WHERE api_key="norostat_key"' - ) - cnx.commit() - cur.close() + self.create_tables_list = [ + create_raw_datatable_version_list, + create_norostat_version_list, + create_norostat_datatable_location_pool, + create_norostat_point_diffs, + ] - self.cnx = cnx - self.cur = cnx.cursor() + self.delete_from_tables_list = [ + "norostat_point_diffs", + "norostat_point_version_list", + "norostat_raw_datatable_location_pool", + "norostat_raw_datatable_version_list", + ] - @staticmethod - def _clear_limits(): - limiter.storage.reset() + self.role_name = "norostat" - def tearDown(self) -> None: - """Perform per-test teardown.""" - self.cur.close() - self.cnx.close() - self._clear_limits() + super().setUp() def test_meta_norostat(self): """Basic integration test for meta_norostat endpoint""" self.cur.execute( - "INSERT INTO norostat_raw_datatable_version_list(release_date, parse_time) VALUES (%s, %s)", + "INSERT INTO `norostat_raw_datatable_version_list`(`release_date`, `parse_time`) VALUES (%s, %s)", ("2014-10-22", "2048-12-08 15:22:51"), ) self.cur.execute( - 'INSERT INTO norostat_raw_datatable_location_pool(`location`) VALUES ("Minnesota, Ohio, Oregon, Tennessee, and Wisconsin")' + 'INSERT INTO `norostat_raw_datatable_location_pool`(`location`) VALUES ("Minnesota, Ohio, Oregon, Tennessee, and Wisconsin")' ) self.cnx.commit() - response = Epidata.meta_norostat(auth="norostat_key") + response = self.epidata.meta_norostat(auth="norostat_key") self.assertEqual( response, { diff --git a/integrations/server/test_nidss_dengue.py b/integrations/server/test_nidss_dengue.py index d57d2a18a..fc537ffcc 100644 --- a/integrations/server/test_nidss_dengue.py +++ b/integrations/server/test_nidss_dengue.py @@ -1,61 +1,24 @@ -# standard library -import unittest - -# third party -import mysql.connector - # first party -from delphi.epidata.client.delphi_epidata import Epidata -from delphi.epidata.server._limiter import limiter +from delphi.epidata.common.integration_test_base_class import BasicIntegrationTest -class NiddsDengueTest(unittest.TestCase): +class NiddsDengueTest(BasicIntegrationTest): """Basic integration tests for nids_dengue endpint.""" - @classmethod - def setUpClass(cls) -> None: - """Perform one-time setup.""" - - # use local instance of the Epidata API - Epidata.BASE_URL = "http://delphi_web_epidata/epidata/api.php" - Epidata.auth = ("epidata", "key") - def setUp(self) -> None: """Perform per-test setup.""" - # connect to the `epidata` database - cnx = mysql.connector.connect(user="user", password="pass", host="delphi_database_epidata", database="epidata") - cur = cnx.cursor() - - cur.execute("DELETE FROM api_user") - cur.execute("TRUNCATE TABLE nidss_dengue") - - cur.execute('INSERT INTO api_user(api_key, email) VALUES("key", "email")') - - cnx.commit() - cur.close() - - self.cnx = cnx - self.cur = cnx.cursor() - - @staticmethod - def _clear_limits(): - limiter.storage.reset() - - def tearDown(self) -> None: - """Perform per-test teardown.""" - self.cur.close() - self.cnx.close() - self._clear_limits() + self.truncate_tables_list = ["nidss_dengue"] + super().setUp() def test_nidss_dengue(self): """Basic integration test for nidds_dengue endpoint""" self.cur.execute( - "INSERT INTO nidss_dengue(epiweek, location, region, count) VALUES(%s, %s, %s, %s)", + "INSERT INTO `nidss_dengue`(`epiweek`, `location`, `region`, `count`) VALUES(%s, %s, %s, %s)", ("200340", "SomeCity", "Central", "0"), ) self.cnx.commit() - response = Epidata.nidss_dengue(locations="SomeCity", epiweeks="200340") + response = self.epidata.nidss_dengue(locations="SomeCity", epiweeks="200340") self.assertEqual( response, {"epidata": [{"location": "SomeCity", "epiweek": 200340, "count": 0}], "result": 1, "message": "success"}, diff --git a/integrations/server/test_nidss_flu.py b/integrations/server/test_nidss_flu.py index 6f38559ae..9cb06e1fd 100644 --- a/integrations/server/test_nidss_flu.py +++ b/integrations/server/test_nidss_flu.py @@ -1,61 +1,24 @@ -# standard library -import unittest - -# third party -import mysql.connector - # first party -from delphi.epidata.client.delphi_epidata import Epidata -from delphi.epidata.server._limiter import limiter +from delphi.epidata.common.integration_test_base_class import BasicIntegrationTest -class NiddsFluTest(unittest.TestCase): +class NiddsFluTest(BasicIntegrationTest): """Basic integration tests for nids_flu endpint.""" - @classmethod - def setUpClass(cls) -> None: - """Perform one-time setup.""" - - # use local instance of the Epidata API - Epidata.BASE_URL = "http://delphi_web_epidata/epidata/api.php" - Epidata.auth = ("epidata", "key") - def setUp(self) -> None: """Perform per-test setup.""" - # connect to the `epidata` database - cnx = mysql.connector.connect(user="user", password="pass", host="delphi_database_epidata", database="epidata") - cur = cnx.cursor() - - cur.execute("DELETE FROM api_user") - cur.execute("TRUNCATE TABLE nidss_flu") - - cur.execute('INSERT INTO api_user(api_key, email) VALUES("key", "email")') - - cnx.commit() - cur.close() - - self.cnx = cnx - self.cur = cnx.cursor() - - @staticmethod - def _clear_limits(): - limiter.storage.reset() - - def tearDown(self) -> None: - """Perform per-test teardown.""" - self.cur.close() - self.cnx.close() - self._clear_limits() + self.truncate_tables_list = ["nidss_flu"] + super().setUp() def test_nidss_flu(self): """Basic integration test for nidds_flu endpoint""" self.cur.execute( - "INSERT INTO nidss_flu(`release_date`, `issue`, `epiweek`, `region`, `lag`, `visits`, `ili`) VALUES(%s, %s, %s, %s, %s, %s, %s)", + "INSERT INTO `nidss_flu`(`release_date`, `issue`, `epiweek`, `region`, `lag`, `visits`, `ili`) VALUES(%s, %s, %s, %s, %s, %s, %s)", ("2015-09-05", "201530", "200111", "SomeRegion", "222", "333", "444"), ) self.cnx.commit() - response = Epidata.nidss_flu(regions="SomeRegion", epiweeks="200111") + response = self.epidata.nidss_flu(regions="SomeRegion", epiweeks="200111") self.assertEqual( response, { diff --git a/integrations/server/test_norostat.py b/integrations/server/test_norostat.py index 461dfc779..7adb901e7 100644 --- a/integrations/server/test_norostat.py +++ b/integrations/server/test_norostat.py @@ -1,33 +1,14 @@ -# standard library -import unittest - -# third party -import mysql.connector - # first party -from delphi.epidata.client.delphi_epidata import Epidata -from delphi.epidata.server._limiter import limiter +from delphi.epidata.common.integration_test_base_class import BasicIntegrationTest -class NorostatTest(unittest.TestCase): +class NorostatTest(BasicIntegrationTest): """Basic integration tests for norostat endpint.""" - @classmethod - def setUpClass(cls) -> None: - """Perform one-time setup.""" - - # use local instance of the Epidata API - Epidata.BASE_URL = "http://delphi_web_epidata/epidata/api.php" - def setUp(self) -> None: """Perform per-test setup.""" - # connect to the `epidata` database - cnx = mysql.connector.connect(user="user", password="pass", host="delphi_database_epidata", database="epidata") - cur = cnx.cursor() - - cur.execute( - """ + create_norostat_point_diffs = """ CREATE TABLE IF NOT EXISTS `norostat_point_diffs` ( `release_date` date NOT NULL, `parse_time` datetime NOT NULL, @@ -40,48 +21,33 @@ def setUp(self) -> None: CONSTRAINT `norostat_point_diffs_ibfk_2` FOREIGN KEY (`location_id`) REFERENCES `norostat_raw_datatable_location_pool` (`location_id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3; """ - ) - - cur.execute("DELETE FROM api_user") - cur.execute("TRUNCATE TABLE user_role") - cur.execute("TRUNCATE TABLE user_role_link") - - cur.execute("DELETE FROM norostat_point_diffs") - cur.execute("DELETE FROM norostat_point_version_list") - cur.execute("DELETE FROM norostat_raw_datatable_location_pool") - cur.execute("DELETE FROM norostat_raw_datatable_version_list") - - cur.execute('INSERT INTO api_user(api_key, email) VALUES("norostat_key", "norostat_email")') - cur.execute('INSERT INTO user_role(name) VALUES("norostat") ON DUPLICATE KEY UPDATE name="norostat"') - cur.execute( - 'INSERT INTO user_role_link(user_id, role_id) SELECT api_user.id, 1 FROM api_user WHERE api_key="norostat_key"' - ) - - cnx.commit() - cur.close() - - self.cnx = cnx - self.cur = cnx.cursor() - - @staticmethod - def _clear_limits(): - limiter.storage.reset() - - def tearDown(self) -> None: - """Perform per-test teardown.""" - self.cur.close() - self.cnx.close() - self._clear_limits() + self.create_tables_list = [create_norostat_point_diffs] + self.delete_from_tables_list = [ + "norostat_point_diffs", + "norostat_point_version_list", + "norostat_raw_datatable_location_pool", + "norostat_raw_datatable_version_list", + ] + self.role_name = "norostat" + super().setUp() def test_norostat(self): """Basic integration test for norostat endpoint""" - self.cur.execute('INSERT INTO norostat_raw_datatable_version_list(release_date, parse_time) VALUES ("2023-07-19", "2023-07-10 15:24:51")') - self.cur.execute('INSERT INTO norostat_raw_datatable_location_pool(location_id, location) VALUES("1", "SomeTestLocation")') - self.cur.execute('INSERT INTO norostat_point_version_list(`release_date`, `parse_time`) VALUES("2023-07-19", "2023-07-10 15:24:51")') - self.cur.execute('INSERT INTO norostat_point_diffs(release_date, parse_time, location_id, epiweek, new_value) VALUES("2023-07-19", "2023-07-10 15:24:51", "1", "202329", 10)') + self.cur.execute( + 'INSERT INTO `norostat_raw_datatable_version_list`(`release_date`, `parse_time`) VALUES ("2023-07-19", "2023-07-10 15:24:51")' + ) + self.cur.execute( + 'INSERT INTO `norostat_raw_datatable_location_pool`(`location_id`, `location`) VALUES("1", "SomeTestLocation")' + ) + self.cur.execute( + 'INSERT INTO `norostat_point_version_list`(`release_date`, `parse_time`) VALUES("2023-07-19", "2023-07-10 15:24:51")' + ) + self.cur.execute( + 'INSERT INTO `norostat_point_diffs`(`release_date`, `parse_time`, `location_id`, `epiweek`, `new_value`) VALUES("2023-07-19", "2023-07-10 15:24:51", "1", "202329", 10)' + ) self.cnx.commit() - response = Epidata.norostat(auth="norostat_key", location="SomeTestLocation", epiweeks="202329") + response = self.epidata.norostat(auth="norostat_key", location="SomeTestLocation", epiweeks="202329") self.assertEqual( response, { diff --git a/integrations/server/test_nowcast.py b/integrations/server/test_nowcast.py index 8f3d4293b..51332d342 100644 --- a/integrations/server/test_nowcast.py +++ b/integrations/server/test_nowcast.py @@ -1,61 +1,24 @@ -# standard library -import unittest - -# third party -import mysql.connector - # first party -from delphi.epidata.client.delphi_epidata import Epidata -from delphi.epidata.server._limiter import limiter +from delphi.epidata.common.integration_test_base_class import BasicIntegrationTest -class NowcastTest(unittest.TestCase): +class NowcastTest(BasicIntegrationTest): """Basic integration tests for nowcast endpint.""" - @classmethod - def setUpClass(cls) -> None: - """Perform one-time setup.""" - - # use local instance of the Epidata API - Epidata.BASE_URL = "http://delphi_web_epidata/epidata/api.php" - Epidata.auth = ("epidata", "key") - def setUp(self) -> None: """Perform per-test setup.""" - # connect to the `epidata` database - cnx = mysql.connector.connect(user="user", password="pass", host="delphi_database_epidata", database="epidata") - cur = cnx.cursor() - - cur.execute("DELETE FROM api_user") - cur.execute("TRUNCATE TABLE nowcasts") - - cur.execute('INSERT INTO api_user(api_key, email) VALUES("key", "email")') - - cnx.commit() - cur.close() - - self.cnx = cnx - self.cur = cnx.cursor() - - @staticmethod - def _clear_limits(): - limiter.storage.reset() - - def tearDown(self) -> None: - """Perform per-test teardown.""" - self.cur.close() - self.cnx.close() - self._clear_limits() + self.truncate_tables_list = ["nowcasts"] + super().setUp() def test_nowcast(self): """Basic integration test for nowcast endpoint""" self.cur.execute( - "INSERT INTO nowcasts(`epiweek`, `location`, `value`, `std`) VALUES(%s, %s, %s, %s)", + "INSERT INTO `nowcasts`(`epiweek`, `location`, `value`, `std`) VALUES(%s, %s, %s, %s)", ("201145", "nat", "12345", "0.01234"), ) self.cnx.commit() - response = Epidata.nowcast(locations="nat", epiweeks="201145") + response = self.epidata.nowcast(locations="nat", epiweeks="201145") self.assertEqual( response, { diff --git a/integrations/server/test_paho_dengue.py b/integrations/server/test_paho_dengue.py index 01767a582..0bb5391ed 100644 --- a/integrations/server/test_paho_dengue.py +++ b/integrations/server/test_paho_dengue.py @@ -1,61 +1,24 @@ -# standard library -import unittest - -# third party -import mysql.connector - # first party -from delphi.epidata.client.delphi_epidata import Epidata -from delphi.epidata.server._limiter import limiter +from delphi.epidata.common.integration_test_base_class import BasicIntegrationTest -class PahoDengueTest(unittest.TestCase): +class PahoDengueTest(BasicIntegrationTest): """Basic integration tests for paho_dengue endpint.""" - @classmethod - def setUpClass(cls) -> None: - """Perform one-time setup.""" - - # use local instance of the Epidata API - Epidata.BASE_URL = "http://delphi_web_epidata/epidata/api.php" - Epidata.auth = ("epidata", "key") - def setUp(self) -> None: """Perform per-test setup.""" - # connect to the `epidata` database - cnx = mysql.connector.connect(user="user", password="pass", host="delphi_database_epidata", database="epidata") - cur = cnx.cursor() - - cur.execute("DELETE FROM api_user") - cur.execute("TRUNCATE TABLE paho_dengue") - - cur.execute('INSERT INTO api_user(api_key, email) VALUES("key", "email")') - - cnx.commit() - cur.close() - - self.cnx = cnx - self.cur = cnx.cursor() - - @staticmethod - def _clear_limits(): - limiter.storage.reset() - - def tearDown(self) -> None: - """Perform per-test teardown.""" - self.cur.close() - self.cnx.close() - self._clear_limits() + self.truncate_tables_list = ["paho_dengue"] + super().setUp() def test_paho_dengue(self): """Basic integration test for paho_dengue endpoint""" self.cur.execute( - "INSERT INTO paho_dengue(`release_date`, `issue`, `epiweek`, `lag`, `region`, `total_pop`, `serotype`, `num_dengue`, `incidence_rate`, `num_severe`, `num_deaths`) VALUES(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)", + "INSERT INTO `paho_dengue`(`release_date`, `issue`, `epiweek`, `lag`, `region`, `total_pop`, `serotype`, `num_dengue`, `incidence_rate`, `num_severe`, `num_deaths`) VALUES(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)", ("2018-12-01", "201848", "201454", "204", "AG", "91", "DEN 1,4", "37", "40.66", "0", "0"), ) self.cnx.commit() - response = Epidata.paho_dengue(regions="AG", epiweeks="201454") + response = self.epidata.paho_dengue(regions="AG", epiweeks="201454") self.assertEqual( response, { diff --git a/integrations/server/test_quidel.py b/integrations/server/test_quidel.py index c16d4e770..6ceefd273 100644 --- a/integrations/server/test_quidel.py +++ b/integrations/server/test_quidel.py @@ -1,66 +1,25 @@ -# standard library -import unittest - -# third party -import mysql.connector - # first party -from delphi.epidata.client.delphi_epidata import Epidata -from delphi.epidata.server._limiter import limiter +from delphi.epidata.common.integration_test_base_class import BasicIntegrationTest -class QuidelTest(unittest.TestCase): +class QuidelTest(BasicIntegrationTest): """Basic integration tests for quidel endpint.""" - @classmethod - def setUpClass(cls) -> None: - """Perform one-time setup.""" - - # use local instance of the Epidata API - Epidata.BASE_URL = "http://delphi_web_epidata/epidata/api.php" - def setUp(self) -> None: """Perform per-test setup.""" - # connect to the `epidata` database - cnx = mysql.connector.connect(user="user", password="pass", host="delphi_database_epidata", database="epidata") - cur = cnx.cursor() - - cur.execute("DELETE FROM api_user") - cur.execute("TRUNCATE TABLE user_role") - cur.execute("TRUNCATE TABLE user_role_link") - cur.execute("TRUNCATE TABLE quidel") - - cur.execute('INSERT INTO api_user(api_key, email) VALUES("quidel_key", "quidel_email")') - cur.execute('INSERT INTO user_role(name) VALUES("quidel") ON DUPLICATE KEY UPDATE name="quidel"') - cur.execute( - 'INSERT INTO user_role_link(user_id, role_id) SELECT api_user.id, 1 FROM api_user WHERE api_key="quidel_key"' - ) - - cnx.commit() - cur.close() - - self.cnx = cnx - self.cur = cnx.cursor() - - @staticmethod - def _clear_limits(): - limiter.storage.reset() - - def tearDown(self) -> None: - """Perform per-test teardown.""" - self.cur.close() - self.cnx.close() - self._clear_limits() + self.truncate_tables_list = ["quidel"] + self.role_name = "quidel" + super().setUp() def test_quidel(self): """Basic integration test for quidel endpoint""" self.cur.execute( - "INSERT INTO quidel(location, epiweek, value, num_rows, num_devices) VALUES(%s, %s, %s, %s, %s)", + "INSERT INTO `quidel`(`location`, `epiweek`, `value`, `num_rows`, `num_devices`) VALUES(%s, %s, %s, %s, %s)", ("loc1", "201111", "1", "0", "0"), ) self.cnx.commit() - response = Epidata.quidel(locations="loc1", epiweeks="201111", auth="quidel_key") + response = self.epidata.quidel(locations="loc1", epiweeks="201111", auth="quidel_key") self.assertEqual( response, {"epidata": [{"location": "loc1", "epiweek": 201111, "value": 1.0}], "result": 1, "message": "success"}, diff --git a/integrations/server/test_sensors.py b/integrations/server/test_sensors.py index 6cf851519..d236bf7eb 100644 --- a/integrations/server/test_sensors.py +++ b/integrations/server/test_sensors.py @@ -1,66 +1,25 @@ -# standard library -import unittest - -# third party -import mysql.connector - # first party -from delphi.epidata.client.delphi_epidata import Epidata -from delphi.epidata.server._limiter import limiter +from delphi.epidata.common.integration_test_base_class import BasicIntegrationTest -class SensorsTest(unittest.TestCase): +class SensorsTest(BasicIntegrationTest): """Basic integration tests for sensors endpint.""" - @classmethod - def setUpClass(cls) -> None: - """Perform one-time setup.""" - - # use local instance of the Epidata API - Epidata.BASE_URL = "http://delphi_web_epidata/epidata/api.php" - def setUp(self) -> None: """Perform per-test setup.""" - # connect to the `epidata` database - cnx = mysql.connector.connect(user="user", password="pass", host="delphi_database_epidata", database="epidata") - cur = cnx.cursor() - - cur.execute("DELETE FROM api_user") - cur.execute("TRUNCATE TABLE user_role") - cur.execute("TRUNCATE TABLE user_role_link") - cur.execute("TRUNCATE TABLE sensors") - - cur.execute('INSERT INTO api_user(api_key, email) VALUES("sensors_key", "sensors_email")') - cur.execute('INSERT INTO user_role(name) VALUES("sensors") ON DUPLICATE KEY UPDATE name="sensors"') - cur.execute( - 'INSERT INTO user_role_link(user_id, role_id) SELECT api_user.id, 1 FROM api_user WHERE api_key="sensors_key"' - ) - - cnx.commit() - cur.close() - - self.cnx = cnx - self.cur = cnx.cursor() - - @staticmethod - def _clear_limits(): - limiter.storage.reset() - - def tearDown(self) -> None: - """Perform per-test teardown.""" - self.cur.close() - self.cnx.close() - self._clear_limits() + self.truncate_tables_list = ["sensors"] + self.role_name = "sensors" + super().setUp() - def test_gft(self): + def test_sensors(self): """Basic integration test for sensors endpoint""" self.cur.execute( - "INSERT INTO sensors(name, epiweek, location, value) VALUES(%s, %s, %s, %s)", + "INSERT INTO `sensors`(`name`, `epiweek`, `location`, `value`) VALUES(%s, %s, %s, %s)", ("sens1", "201111", "loc1", "222"), ) self.cnx.commit() - response = Epidata.sensors(names="sens1", locations="loc1", epiweeks="201111", auth="sensors_key") + response = self.epidata.sensors(names="sens1", locations="loc1", epiweeks="201111", auth="sensors_key") self.assertEqual( response, { diff --git a/integrations/server/test_signal_dashboard.py b/integrations/server/test_signal_dashboard.py index a442d49a0..bebf6357f 100644 --- a/integrations/server/test_signal_dashboard.py +++ b/integrations/server/test_signal_dashboard.py @@ -1,64 +1,26 @@ -# standard library -import unittest - -# third party -import mysql.connector - # first party -from delphi.epidata.client.delphi_epidata import Epidata -from delphi.epidata.server._limiter import limiter +from delphi.epidata.common.integration_test_base_class import BasicIntegrationTest -class SignalDashboardTest(unittest.TestCase): +class SignalDashboardTest(BasicIntegrationTest): """Basic integration tests for signal_dashboard_coverage and signal_dashboard_status endpints.""" - @classmethod - def setUpClass(cls) -> None: - """Perform one-time setup.""" - - # use local instance of the Epidata API - Epidata.BASE_URL = "http://delphi_web_epidata/epidata/api.php" - Epidata.auth = ("epidata", "key") - def setUp(self) -> None: """Perform per-test setup.""" - # connect to the `epidata` database - cnx = mysql.connector.connect(user="user", password="pass", host="delphi_database_epidata", database="epidata") - cur = cnx.cursor() - - cur.execute("DELETE FROM api_user") + self.delete_from_tables_list = ["dashboard_signal_coverage", "dashboard_signal"] + super().setUp() - cur.execute("DELETE FROM dashboard_signal_coverage") - cur.execute("DELETE FROM dashboard_signal") - - - cur.execute('INSERT INTO api_user(api_key, email) VALUES("key", "email")') - - cur.execute( - "INSERT INTO dashboard_signal(id, name, source, covidcast_signal, enabled, latest_coverage_update, latest_status_update) VALUES(%s, %s, %s, %s, %s, %s, %s)", + self.cur.execute( + "INSERT INTO `dashboard_signal`(`id`, `name`, `source`, `covidcast_signal`, `enabled`, `latest_coverage_update`, `latest_status_update`) VALUES(%s, %s, %s, %s, %s, %s, %s)", ("1", "Change", "chng", "smoothed_outpatient_covid", "1", "2021-10-02", "2021-11-27"), ) - cur.execute( - "INSERT INTO dashboard_signal_coverage(signal_id, date, geo_type, count) VALUES(%s, %s, %s, %s)", + self.cur.execute( + "INSERT INTO `dashboard_signal_coverage`(`signal_id`, `date`, `geo_type`, `count`) VALUES(%s, %s, %s, %s)", ("1", "2021-10-02", "county", "2222"), ) - cnx.commit() - cur.close() - - self.cnx = cnx - self.cur = cnx.cursor() - - @staticmethod - def _clear_limits(): - limiter.storage.reset() - - def tearDown(self) -> None: - """Perform per-test teardown.""" - self.cur.close() - self.cnx.close() - self._clear_limits() + self.cnx.commit() def test_signal_dashboard_coverage(self): """Basic integration test for signal_dashboard_coverage endpoint""" @@ -66,7 +28,7 @@ def test_signal_dashboard_coverage(self): params = { "endpoint": "signal_dashboard_coverage", } - response = Epidata._request(params=params) + response = self.epidata._request(params=params) self.assertEqual( response, { @@ -82,7 +44,7 @@ def test_signal_dashboard_status(self): params = { "endpoint": "signal_dashboard_status", } - response = Epidata._request(params=params) + response = self.epidata._request(params=params) self.assertEqual( response, { diff --git a/integrations/server/test_twitter.py b/integrations/server/test_twitter.py index 46cda4acc..b32ad8290 100644 --- a/integrations/server/test_twitter.py +++ b/integrations/server/test_twitter.py @@ -1,66 +1,25 @@ -# standard library -import unittest - -# third party -import mysql.connector - # first party -from delphi.epidata.client.delphi_epidata import Epidata -from delphi.epidata.server._limiter import limiter +from delphi.epidata.common.integration_test_base_class import BasicIntegrationTest -class TwitterTest(unittest.TestCase): +class TwitterTest(BasicIntegrationTest): """Basic integration tests for twitter endpint.""" - @classmethod - def setUpClass(cls) -> None: - """Perform one-time setup.""" - - # use local instance of the Epidata API - Epidata.BASE_URL = "http://delphi_web_epidata/epidata/api.php" - def setUp(self) -> None: """Perform per-test setup.""" - # connect to the `epidata` database - cnx = mysql.connector.connect(user="user", password="pass", host="delphi_database_epidata", database="epidata") - cur = cnx.cursor() - - cur.execute("DELETE FROM api_user") - cur.execute("TRUNCATE TABLE user_role") - cur.execute("TRUNCATE TABLE user_role_link") - cur.execute("TRUNCATE TABLE twitter") - - cur.execute('INSERT INTO api_user(api_key, email) VALUES("twitter_key", "twitter_email")') - cur.execute('INSERT INTO user_role(name) VALUES("twitter") ON DUPLICATE KEY UPDATE name="twitter"') - cur.execute( - 'INSERT INTO user_role_link(user_id, role_id) SELECT api_user.id, 1 FROM api_user WHERE api_key="twitter_key"' - ) - - cnx.commit() - cur.close() - - self.cnx = cnx - self.cur = cnx.cursor() - - @staticmethod - def _clear_limits(): - limiter.storage.reset() - - def tearDown(self) -> None: - """Perform per-test teardown.""" - self.cur.close() - self.cnx.close() - self._clear_limits() + self.truncate_tables_list = ["twitter"] + self.role_name = "twitter" + super().setUp() def test_twitter(self): """Basic integration test for twitter endpoint""" self.cur.execute( - 'INSERT INTO twitter(date, state, num, total) VALUES ("2015-07-29", "AK", "1", "223"), ("2020-07-29", "CT", "12", "778")', + 'INSERT INTO `twitter`(`date`, `state`, `num`, `total`) VALUES ("2015-07-29", "AK", "1", "223"), ("2020-07-29", "CT", "12", "778")', ) self.cnx.commit() - response = Epidata.twitter(auth="twitter_key", locations="cen9", dates="20150701-20160101") + response = self.epidata.twitter(auth="twitter_key", locations="cen9", dates="20150701-20160101") self.assertEqual( response, { diff --git a/integrations/server/test_wiki.py b/integrations/server/test_wiki.py index 6763d7d62..982c1448b 100644 --- a/integrations/server/test_wiki.py +++ b/integrations/server/test_wiki.py @@ -1,65 +1,27 @@ -# standard library -import unittest - -# third party -import mysql.connector - # first party -from delphi.epidata.client.delphi_epidata import Epidata -from delphi.epidata.server._limiter import limiter +from delphi.epidata.common.integration_test_base_class import BasicIntegrationTest -class WikiTest(unittest.TestCase): +class WikiTest(BasicIntegrationTest): """Basic integration tests for wiki endpint.""" - @classmethod - def setUpClass(cls) -> None: - """Perform one-time setup.""" - - # use local instance of the Epidata API - Epidata.BASE_URL = "http://delphi_web_epidata/epidata/api.php" - Epidata.auth = ("epidata", "key") - def setUp(self) -> None: """Perform per-test setup.""" - # connect to the `epidata` database - cnx = mysql.connector.connect(user="user", password="pass", host="delphi_database_epidata", database="epidata") - cur = cnx.cursor() - - cur.execute("DELETE FROM api_user") - cur.execute("TRUNCATE TABLE wiki") - cur.execute("TRUNCATE TABLE wiki_meta") - - cur.execute('INSERT INTO api_user(api_key, email) VALUES("key", "email")') - - cnx.commit() - cur.close() - - self.cnx = cnx - self.cur = cnx.cursor() - - @staticmethod - def _clear_limits(): - limiter.storage.reset() - - def tearDown(self) -> None: - """Perform per-test teardown.""" - self.cur.close() - self.cnx.close() - self._clear_limits() + self.truncate_tables_list = ["wiki", "wiki_meta"] + super().setUp() def test_wiki(self): """Basic integration test for wiki endpoint""" self.cur.execute( - 'INSERT INTO wiki(datetime, article, count, language) VALUES ("2007-12-09 18:00:00", "amantadine", "3", "en"), ("2008-12-09 18:00:00", "test", "5", "en")', + 'INSERT INTO `wiki`(`datetime`, `article`, `count`, `language`) VALUES ("2007-12-09 18:00:00", "amantadine", "3", "en"), ("2008-12-09 18:00:00", "test", "5", "en")', ) self.cur.execute( - 'INSERT INTO wiki_meta(datetime, date, epiweek, total, language) VALUES ("2007-12-09 18:00:00", "2007-12-09", "200750", "969214", "en"), ("2008-12-09 18:00:00", "2008-12-09", "200750", "123321", "en")' + 'INSERT INTO `wiki_meta`(`datetime`, `date`, `epiweek`, `total`, `language`) VALUES ("2007-12-09 18:00:00", "2007-12-09", "200750", "969214", "en"), ("2008-12-09 18:00:00", "2008-12-09", "200750", "123321", "en")' ) self.cnx.commit() - response = Epidata.wiki(articles="test", epiweeks="200701-200801") + response = self.epidata.wiki(articles="test", epiweeks="200701-200801") self.assertEqual( response, { diff --git a/src/common/integration_test_base_class.py b/src/common/integration_test_base_class.py new file mode 100644 index 000000000..580331d4a --- /dev/null +++ b/src/common/integration_test_base_class.py @@ -0,0 +1,79 @@ +# standard library +import unittest + +# third party +import mysql.connector + +# first party +from delphi.epidata.client.delphi_epidata import Epidata +from delphi.epidata.server._limiter import limiter + + +class BasicIntegrationTest(unittest.TestCase): + """Basic integration test class""" + + def __init__(self, methodName: str = "runTest") -> None: + super().__init__(methodName) + self.delete_from_tables_list = [] + self.truncate_tables_list = [] + self.create_tables_list = [] + self.role_name = None + self.epidata = Epidata + self.epidata.BASE_URL = "http://delphi_web_epidata/epidata/api.php" + self.epidata.auth = ("epidata", "key") + + def delete_from_table(self, cur, table_name: str) -> None: + cur.execute(f"DELETE FROM `{table_name}`") + + def truncate_table(self, cur, table_name: str) -> None: + cur.execute(f"TRUNCATE TABLE `{table_name}`") + + def create_table(self, cur, create_table_stmt: str): + cur.execute(create_table_stmt) + + def create_key_with_role(self, cur, role_name: str): + cur.execute("TRUNCATE TABLE `user_role`") + cur.execute("TRUNCATE TABLE `user_role_link`") + cur.execute(f'INSERT INTO `api_user`(`api_key`, `email`) VALUES("{role_name}_key", "{role_name}_email")') + cur.execute(f'INSERT INTO `user_role`(`name`) VALUES("{role_name}")') + cur.execute( + f'INSERT INTO `user_role_link`(`user_id`, `role_id`) SELECT `api_user`.`id`, 1 FROM `api_user` WHERE `api_key`="{role_name}_key"' + ) + + def setUp(self) -> None: + """Perform per-test setup.""" + + # connect to the `epidata` database + cnx = mysql.connector.connect(user="user", password="pass", host="delphi_database_epidata", database="epidata") + cur = cnx.cursor() + + cur.execute("DELETE FROM `api_user`") + cur.execute('INSERT INTO `api_user`(`api_key`, `email`) VALUES ("key", "email")') + + for stmt in self.create_tables_list: + self.create_table(cur, stmt) + + for table_name in self.delete_from_tables_list: + self.delete_from_table(cur, table_name) + + for table_name in self.truncate_tables_list: + self.truncate_table(cur, table_name) + + if self.role_name: + self.create_key_with_role(cur, self.role_name) + + cnx.commit() + cur.close() + + self.cnx = cnx + self.cur = cnx.cursor() + + @staticmethod + def _clear_limits() -> None: + limiter.storage.reset() + + def tearDown(self) -> None: + """Perform per-test teardown.""" + self.cur.close() + self.cnx.close() + self._clear_limits() diff --git a/src/server/_config.py b/src/server/_config.py index 4a96250c6..cb5b01e47 100644 --- a/src/server/_config.py +++ b/src/server/_config.py @@ -93,8 +93,16 @@ REDIS_HOST = os.environ.get("REDIS_HOST", "delphi_redis") REDIS_PASSWORD = os.environ.get("REDIS_PASSWORD", "1234") +# testing mode to lower rate-limit in order to reduce number of required requests to hit the rate limit while runnig tests +# by default is set to False when is not provided +TESTING_MODE = os.environ.get("TESTING_MODE", False) + # https://flask-limiter.readthedocs.io/en/stable/#rate-limit-string-notation RATE_LIMIT = os.environ.get("RATE_LIMIT", "60/hour") + +if TESTING_MODE is not False: + RATE_LIMIT = "5/hour" + # fixed-window, fixed-window-elastic-expiry, or moving-window # see also https://flask-limiter.readthedocs.io/en/stable/#rate-limiting-strategies RATELIMIT_STRATEGY = os.environ.get("RATELIMIT_STRATEGY", "fixed-window") From 544ba94211531ddb77a344bac507db907c04aa50 Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Fri, 21 Jul 2023 15:52:24 +0300 Subject: [PATCH 10/55] Added TESTING_MODE to the ci.yaml, removed from Makefile --- .github/workflows/ci.yaml | 2 +- dev/local/Makefile | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2dca44222..fd2d5fb4b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -75,7 +75,7 @@ jobs: - name: Start delphi_web_epidata run: | - docker run --rm -d -p 10080:80 --env "MODULE_NAME=delphi.epidata.server.main" --env "SQLALCHEMY_DATABASE_URI=mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata" --env "FLASK_SECRET=abc" --env "FLASK_PREFIX=/epidata" --env "REDIS_HOST=delphi_redis" --env "REDIS_PASSWORD=1234" --env "API_KEY_REGISTER_WEBHOOK_TOKEN=abc" --env "API_KEY_ADMIN_PASSWORD=test_admin_password" --network delphi-net --name delphi_web_epidata delphi_web_epidata + docker run --rm -d -p 10080:80 --env "MODULE_NAME=delphi.epidata.server.main" --env "SQLALCHEMY_DATABASE_URI=mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata" --env "FLASK_SECRET=abc" --env "FLASK_PREFIX=/epidata" --env "REDIS_HOST=delphi_redis" --env "REDIS_PASSWORD=1234" --env "API_KEY_REGISTER_WEBHOOK_TOKEN=abc" --env "API_KEY_ADMIN_PASSWORD=test_admin_password" --env "TESTING_MODE=True" --network delphi-net --name delphi_web_epidata delphi_web_epidata docker ps - name: Run Unit Tests diff --git a/dev/local/Makefile b/dev/local/Makefile index 018413f2a..e7e896aa6 100644 --- a/dev/local/Makefile +++ b/dev/local/Makefile @@ -102,7 +102,6 @@ web: --env "FLASK_SECRET=abc" --env "FLASK_PREFIX=/epidata" --env "LOG_DEBUG" \ --env "REDIS_HOST=delphi_redis" \ --env "REDIS_PASSWORD=1234" \ - --env "TESTING_MODE=True" \ --env "API_KEY_ADMIN_PASSWORD=test_admin_password" \ --env "API_KEY_REGISTER_WEBHOOK_TOKEN=abc" \ --network delphi-net --name delphi_web_epidata \ From 9eed497066fd679f1d0f66837404f16b7ca34b58 Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Wed, 26 Jul 2023 18:13:30 +0300 Subject: [PATCH 11/55] Updated acquisition tests with missing api_key creation --- .../covidcast/test_covidcast_meta_caching.py | 13 +++++++++++++ .../acquisition/covidcast/test_csv_uploading.py | 13 +++++++++++++ 2 files changed, 26 insertions(+) diff --git a/integrations/acquisition/covidcast/test_covidcast_meta_caching.py b/integrations/acquisition/covidcast/test_covidcast_meta_caching.py index 6e4c6378f..16e2506fc 100644 --- a/integrations/acquisition/covidcast/test_covidcast_meta_caching.py +++ b/integrations/acquisition/covidcast/test_covidcast_meta_caching.py @@ -58,6 +58,19 @@ def setUp(self): secrets.db.host = 'delphi_database_epidata' secrets.db.epi = ('user', 'pass') + epidata_cnx = mysql.connector.connect( + user='user', + password='pass', + host='delphi_database_epidata', + database='epidata') + epidata_cur = epidata_cnx.cursor() + + epidata_cur.execute("DELETE FROM `api_user`") + epidata_cur.execute('INSERT INTO `api_user`(`api_key`, `email`) VALUES("key", "email")') + epidata_cnx.commit() + epidata_cur.close() + epidata_cnx.close() + # use the local instance of the Epidata API Epidata.BASE_URL = BASE_URL Epidata.auth = ('epidata', 'key') diff --git a/integrations/acquisition/covidcast/test_csv_uploading.py b/integrations/acquisition/covidcast/test_csv_uploading.py index e4c9d881e..e9c066f6a 100644 --- a/integrations/acquisition/covidcast/test_csv_uploading.py +++ b/integrations/acquisition/covidcast/test_csv_uploading.py @@ -55,6 +55,19 @@ def setUp(self): secrets.db.host = 'delphi_database_epidata' secrets.db.epi = ('user', 'pass') + epidata_cnx = mysql.connector.connect( + user='user', + password='pass', + host='delphi_database_epidata', + database='epidata') + epidata_cur = epidata_cnx.cursor() + + epidata_cur.execute("DELETE FROM `api_user`") + epidata_cur.execute('INSERT INTO `api_user`(`api_key`, `email`) VALUES("key", "email")') + epidata_cnx.commit() + epidata_cur.close() + epidata_cnx.close() + # use the local instance of the Epidata API Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php' Epidata.auth = ('epidata', 'key') From da2a6a681ff03fb08bfe6d1b7292dc84a23b088c Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Tue, 19 Sep 2023 15:09:46 +0300 Subject: [PATCH 12/55] Update src/server/_config.py Comments fix Co-authored-by: melange396 --- src/server/_config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/server/_config.py b/src/server/_config.py index cb5b01e47..cf743300c 100644 --- a/src/server/_config.py +++ b/src/server/_config.py @@ -93,8 +93,8 @@ REDIS_HOST = os.environ.get("REDIS_HOST", "delphi_redis") REDIS_PASSWORD = os.environ.get("REDIS_PASSWORD", "1234") -# testing mode to lower rate-limit in order to reduce number of required requests to hit the rate limit while runnig tests -# by default is set to False when is not provided +# mode to reduce number of required requests to hit rate limit while running tests, +# by default is set to False TESTING_MODE = os.environ.get("TESTING_MODE", False) # https://flask-limiter.readthedocs.io/en/stable/#rate-limit-string-notation From 3b6b652df03aa809c79cc4953584d62833d76777 Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Tue, 19 Sep 2023 15:10:29 +0300 Subject: [PATCH 13/55] Update integrations/server/test_delphi.py Remove unused lib Co-authored-by: melange396 --- integrations/server/test_delphi.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/integrations/server/test_delphi.py b/integrations/server/test_delphi.py index 481b9d3f5..d8610a5ce 100644 --- a/integrations/server/test_delphi.py +++ b/integrations/server/test_delphi.py @@ -1,7 +1,3 @@ -# standard library -import unittest - -# third party import json # first party From b134408bfdbde99c316ae244079a2c2cf24834cc Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Wed, 20 Sep 2023 19:31:51 +0300 Subject: [PATCH 14/55] Update integrations/server/test_meta.py Remove extra newline Co-authored-by: melange396 --- integrations/server/test_meta.py | 1 - 1 file changed, 1 deletion(-) diff --git a/integrations/server/test_meta.py b/integrations/server/test_meta.py index c32214222..4bcc6e7cb 100644 --- a/integrations/server/test_meta.py +++ b/integrations/server/test_meta.py @@ -10,7 +10,6 @@ def setUp(self) -> None: super().setUp() def test_meta(self): - """Basic integration test for meta endpoint""" response = self.epidata.meta() self.assertEqual( From 15177ac59c366b48dc0590f6a6d194a3a566d5b9 Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Thu, 21 Sep 2023 22:23:01 +0300 Subject: [PATCH 15/55] Requested changes from PR's comments --- dev/local/Makefile | 1 + integrations/server/test_api_keys.py | 103 ++++++++++--------- integrations/server/test_cdc.py | 9 +- integrations/server/test_delphi.py | 9 +- integrations/server/test_dengue_nowcast.py | 11 +- integrations/server/test_dengue_sensors.py | 11 +- integrations/server/test_ecdc_ili.py | 11 +- integrations/server/test_flusurv.py | 11 +- integrations/server/test_fluview_clinical.py | 11 +- integrations/server/test_gft.py | 11 +- integrations/server/test_ght.py | 11 +- integrations/server/test_kcdc_ili.py | 11 +- integrations/server/test_meta.py | 9 +- integrations/server/test_meta_norostat.py | 12 +-- integrations/server/test_nidss_dengue.py | 11 +- integrations/server/test_nidss_flu.py | 11 +- integrations/server/test_norostat.py | 11 +- integrations/server/test_nowcast.py | 11 +- integrations/server/test_paho_dengue.py | 11 +- integrations/server/test_quidel.py | 11 +- integrations/server/test_sensors.py | 11 +- integrations/server/test_signal_dashboard.py | 8 +- integrations/server/test_twitter.py | 11 +- integrations/server/test_wiki.py | 11 +- src/common/integration_test_base_class.py | 36 +++---- 25 files changed, 161 insertions(+), 213 deletions(-) diff --git a/dev/local/Makefile b/dev/local/Makefile index e7e896aa6..fb02668ee 100644 --- a/dev/local/Makefile +++ b/dev/local/Makefile @@ -104,6 +104,7 @@ web: --env "REDIS_PASSWORD=1234" \ --env "API_KEY_ADMIN_PASSWORD=test_admin_password" \ --env "API_KEY_REGISTER_WEBHOOK_TOKEN=abc" \ + --env "TESTING_MODE=True" \ --network delphi-net --name delphi_web_epidata \ delphi_web_epidata >$(LOG_WEB) 2>&1 & diff --git a/integrations/server/test_api_keys.py b/integrations/server/test_api_keys.py index 19bc4d3ff..e522fe9d7 100644 --- a/integrations/server/test_api_keys.py +++ b/integrations/server/test_api_keys.py @@ -2,32 +2,28 @@ import requests # first party -from delphi.epidata.common.integration_test_base_class import BasicIntegrationTest +from delphi.epidata.common.integration_test_base_class import DelphiTestBase -class APIKeysTets(BasicIntegrationTest): +class APIKeysTets(DelphiTestBase): """Tests the API Keys behaviour""" - def setUp(self): - """Perform per-test setup.""" - + def localSetUp(self): self.role_name = "cdc" - super().setUp() def _make_request(self, url: str = None, params: dict = {}, auth: tuple = None): if not url: - url = self.epidata.BASE_URL + url = self.epidata_client.BASE_URL response = requests.get(url, params=params, auth=auth) return response - + def test_public_route(self): """Test public route""" public_route = "http://delphi_web_epidata/epidata/version" status_codes = set() for _ in range(10): status_codes.add(self._make_request(public_route).status_code) - self.assertEqual(len(status_codes), 1) - self.assertEqual(next(iter(status_codes)), 200) + self.assertEqual(status_codes, {200}) def test_no_multiples_data_source(self): """Test requests with no multiples and with provided `data_source` and `signal` as a separate query params.""" @@ -43,8 +39,7 @@ def test_no_multiples_data_source(self): status_codes = set() for _ in range(10): status_codes.add(self._make_request(params=params).status_code) - self.assertEqual(len(status_codes), 1) - self.assertEqual(next(iter(status_codes)), 200) + self.assertEqual(status_codes, {200}) def test_no_multiples_source_signal(self): """Test requests with colon-delimited source-signal param presentation.""" @@ -59,10 +54,9 @@ def test_no_multiples_source_signal(self): status_codes = set() for _ in range(10): status_codes.add(self._make_request(params=params).status_code) - self.assertEqual(len(status_codes), 1) - self.assertEqual(next(iter(status_codes)), 200) + self.assertEqual(status_codes, {200}) - def test_multiples_allowed_signal(self): + def test_multiples_allowed_signal_two_multiples(self): """Test requests with 2 multiples and allowed dashboard signal""" params = { "source": "covidcast", @@ -75,8 +69,7 @@ def test_multiples_allowed_signal(self): status_codes = set() for _ in range(10): status_codes.add(self._make_request(params=params).status_code) - self.assertEqual(len(status_codes), 1) - self.assertEqual(next(iter(status_codes)), 200) + self.assertEqual(status_codes, {200}) def test_multiples_non_allowed_signal(self): """Test requests with 2 multiples and non-allowed dashboard signal""" @@ -91,10 +84,9 @@ def test_multiples_non_allowed_signal(self): status_codes = set() for _ in range(10): status_codes.add(self._make_request(params=params).status_code) - self.assertEqual(len(status_codes), 2) self.assertEqual(status_codes, {200, 429}) - def test_multiples_mixed_allowed_signal(self): + def test_multiples_mixed_allowed_signal_two_multiples(self): """Test requests with 2 multiples and mixed-allowed dashboard signal""" params = { "source": "covidcast", @@ -107,10 +99,9 @@ def test_multiples_mixed_allowed_signal(self): status_codes = set() for _ in range(10): status_codes.add(self._make_request(params=params).status_code) - self.assertEqual(len(status_codes), 2) self.assertEqual(status_codes, {200, 429}) - def test_multiples_allowed_signal(self): + def test_multiples_allowed_signal_three_multiples(self): """Test requests with 3 multiples and allowed dashboard signal""" params = { "source": "covidcast", @@ -123,10 +114,9 @@ def test_multiples_allowed_signal(self): status_codes = set() for _ in range(10): status_codes.add(self._make_request(params=params).status_code) - self.assertEqual(len(status_codes), 1) - self.assertEqual(next(iter(status_codes)), 401) + self.assertEqual(status_codes, {401}) - def test_multiples_mixed_allowed_signal(self): + def test_multiples_mixed_allowed_signal_three_multiples(self): """Test requests with 3 multiples and mixed-allowed dashboard signal""" params = { "source": "covidcast", @@ -139,8 +129,7 @@ def test_multiples_mixed_allowed_signal(self): status_codes = set() for _ in range(10): status_codes.add(self._make_request(params=params).status_code) - self.assertEqual(len(status_codes), 1) - self.assertEqual(next(iter(status_codes)), 401) + self.assertEqual(status_codes, {401}) def test_multiples_mixed_allowed_signal_api_key(self): """Test requests with 3 multiples and mixed-allowed dashboard signal + valid API Key""" @@ -154,9 +143,11 @@ def test_multiples_mixed_allowed_signal_api_key(self): } status_codes = set() for _ in range(10): - status_codes.add(self._make_request(params=params, auth=self.epidata.auth).status_code) + status_codes.add( + self._make_request(params=params, auth=self.epidata_client.auth).status_code + ) self.assertEqual(len(status_codes), 1) - self.assertEqual(next(iter(status_codes)), 200) + self.assertEqual(status_codes, {200}) def test_multiples_allowed_signal_api_key(self): """Test requests with 3 multiples and allowed dashboard signal + valid API Key""" @@ -170,9 +161,10 @@ def test_multiples_allowed_signal_api_key(self): } status_codes = set() for _ in range(10): - status_codes.add(self._make_request(params=params, auth=self.epidata.auth).status_code) - self.assertEqual(len(status_codes), 1) - self.assertEqual(next(iter(status_codes)), 200) + status_codes.add( + self._make_request(params=params, auth=self.epidata_client.auth).status_code + ) + self.assertEqual(status_codes, {200}) def test_no_multiples_allowed_signal_api_key(self): """Test requests with no multiples and allowed dashboard signal + valid API Key""" @@ -186,9 +178,10 @@ def test_no_multiples_allowed_signal_api_key(self): } status_codes = set() for _ in range(10): - status_codes.add(self._make_request(params=params, auth=self.epidata.auth).status_code) - self.assertEqual(len(status_codes), 1) - self.assertEqual(next(iter(status_codes)), 200) + status_codes.add( + self._make_request(params=params, auth=self.epidata_client.auth).status_code + ) + self.assertEqual(status_codes, {200}) def test_no_multiples_allowed_signal_bad_api_key(self): """Test requests with no multiples and allowed dashboard signal + bad API Key""" @@ -202,9 +195,12 @@ def test_no_multiples_allowed_signal_bad_api_key(self): } status_codes = set() for _ in range(10): - status_codes.add(self._make_request(params=params, auth=("bad_key", "bad_email")).status_code) - self.assertEqual(len(status_codes), 1) - self.assertEqual(next(iter(status_codes)), 200) + status_codes.add( + self._make_request( + params=params, auth=("bad_key", "bad_email") + ).status_code + ) + self.assertEqual(status_codes, {200}) def test_restricted_endpoint_no_key(self): """Test restricted endpoint with no auth key""" @@ -212,32 +208,43 @@ def test_restricted_endpoint_no_key(self): status_codes = set() for _ in range(10): status_codes.add(self._make_request(params=params).status_code) - self.assertEqual(len(status_codes), 1) - self.assertEqual(next(iter(status_codes)), 401) + self.assertEqual(status_codes, {401}) def test_restricted_endpoint_invalid_key(self): """Test restricted endpoint with invalid auth key""" - params = {"source": "cdc", "regions": "1as", "epiweeks": "202020", "auth": "invalid_key"} + params = { + "source": "cdc", + "regions": "1as", + "epiweeks": "202020", + "auth": "invalid_key", + } status_codes = set() for _ in range(10): status_codes.add(self._make_request(params=params).status_code) - self.assertEqual(len(status_codes), 1) - self.assertEqual(next(iter(status_codes)), 401) + self.assertEqual(status_codes, {401}) def test_restricted_endpoint_no_roles_key(self): """Test restricted endpoint with no roles key""" - params = {"source": "cdc", "regions": "1as", "epiweeks": "202020", "auth": "key"} + params = { + "source": "cdc", + "regions": "1as", + "epiweeks": "202020", + "auth": "key", + } status_codes = set() for _ in range(10): status_codes.add(self._make_request(params=params).status_code) - self.assertEqual(len(status_codes), 1) - self.assertEqual(next(iter(status_codes)), 401) + self.assertEqual(status_codes, {401}) def test_restricted_endpoint_valid_roles_key(self): """Test restricted endpoint with valid auth key with required role""" - params = {"source": "cdc", "regions": "1as", "epiweeks": "202020", "auth": "cdc_key"} + params = { + "source": "cdc", + "regions": "1as", + "epiweeks": "202020", + "auth": "cdc_key", + } status_codes = set() for _ in range(10): status_codes.add(self._make_request(params=params).status_code) - self.assertEqual(len(status_codes), 1) - self.assertEqual(next(iter(status_codes)), 200) + self.assertEqual(status_codes, {200}) diff --git a/integrations/server/test_cdc.py b/integrations/server/test_cdc.py index 7234f9eb0..d468bd162 100644 --- a/integrations/server/test_cdc.py +++ b/integrations/server/test_cdc.py @@ -1,14 +1,13 @@ # first party -from delphi.epidata.common.integration_test_base_class import BasicIntegrationTest +from delphi.epidata.common.integration_test_base_class import DelphiTestBase -class CdcTest(BasicIntegrationTest): +class CdcTest(DelphiTestBase): """Basic integration tests for cdc endpint.""" - def setUp(self) -> None: + def localSetUp(self) -> None: self.truncate_tables_list = ["cdc_extract"] self.role_name = "cdc" - super().setUp() def test_cdc(self): """Basic integration test for cdc endpoint""" @@ -17,7 +16,7 @@ def test_cdc(self): ("201102", "AK", "16", "35", "51", "96", "30", "748", "243", "433", "65"), ) self.cnx.commit() - response = self.epidata.cdc(auth="cdc_key", epiweeks=201102, locations="cen9") + response = self.epidata_client.cdc(auth="cdc_key", epiweeks=201102, locations="cen9") self.assertEqual( response, { diff --git a/integrations/server/test_delphi.py b/integrations/server/test_delphi.py index d8610a5ce..fc3e3bc7e 100644 --- a/integrations/server/test_delphi.py +++ b/integrations/server/test_delphi.py @@ -1,15 +1,14 @@ import json # first party -from delphi.epidata.common.integration_test_base_class import BasicIntegrationTest +from delphi.epidata.common.integration_test_base_class import DelphiTestBase -class DelphiTest(BasicIntegrationTest): +class DelphiTest(DelphiTestBase): """Basic integration tests for delphi endpint.""" - def setUp(self) -> None: + def localSetUp(self): self.truncate_tables_list = ["forecasts"] - super().setUp() def test_delphi(self): """Basic integration test for delphi endpoint""" @@ -33,7 +32,7 @@ def test_delphi(self): ), ) self.cnx.commit() - response = self.epidata.delphi(system="eb", epiweek=201441) + response = self.epidata_client.delphi(system="eb", epiweek=201441) self.assertEqual( response, { diff --git a/integrations/server/test_dengue_nowcast.py b/integrations/server/test_dengue_nowcast.py index dea91979d..79f9765f4 100644 --- a/integrations/server/test_dengue_nowcast.py +++ b/integrations/server/test_dengue_nowcast.py @@ -1,13 +1,11 @@ # first party -from delphi.epidata.common.integration_test_base_class import BasicIntegrationTest +from delphi.epidata.common.integration_test_base_class import DelphiTestBase -class DengueNowcastTest(BasicIntegrationTest): +class DengueNowcastTest(DelphiTestBase): """Basic integration tests for dengue_nowcast endpint.""" - def setUp(self) -> None: - """Perform per-test setup.""" - + def localSetUp(self): create_dengue_nowcasts = """ CREATE TABLE IF NOT EXISTS `dengue_nowcasts` ( `id` int NOT NULL AUTO_INCREMENT, @@ -25,7 +23,6 @@ def setUp(self) -> None: """ self.create_tables_list = [create_dengue_nowcasts] self.truncate_tables_list = ["dengue_nowcasts"] - super().setUp() def test_dengue_nowcasts(self): """Basic integration test for dengue_nowcasts endpoint""" @@ -34,7 +31,7 @@ def test_dengue_nowcasts(self): ("num_dengue", "201409", "ar", "85263", "351456"), ) self.cnx.commit() - response = self.epidata.dengue_nowcast(locations="ar", epiweeks=201409) + response = self.epidata_client.dengue_nowcast(locations="ar", epiweeks=201409) self.assertEqual( response, { diff --git a/integrations/server/test_dengue_sensors.py b/integrations/server/test_dengue_sensors.py index 75ff9028a..55d103367 100644 --- a/integrations/server/test_dengue_sensors.py +++ b/integrations/server/test_dengue_sensors.py @@ -1,13 +1,11 @@ # first party -from delphi.epidata.common.integration_test_base_class import BasicIntegrationTest +from delphi.epidata.common.integration_test_base_class import DelphiTestBase -class DengueSensorsTest(BasicIntegrationTest): +class DengueSensorsTest(DelphiTestBase): """Basic integration tests for dengue_sensors endpint.""" - def setUp(self) -> None: - """Perfor per-test setup.""" - + def localSetUp(self): create_dengue_sensors = """ CREATE TABLE IF NOT EXISTS `dengue_sensors` ( `id` int NOT NULL AUTO_INCREMENT, @@ -26,7 +24,6 @@ def setUp(self) -> None: self.create_tables_list = [create_dengue_sensors] self.truncate_tables_list = ["dengue_sensors"] self.role_name = "sensors" - super().setUp() def test_dengue_sensors(self): """Basic integration test for dengue_sensors endpoint""" @@ -35,7 +32,7 @@ def test_dengue_sensors(self): ("num_dengue", "ght", "201432", "ag", "1234"), ) self.cnx.commit() - response = self.epidata.dengue_sensors(auth="sensors_key", names="ght", locations="ag", epiweeks="201432") + response = self.epidata_client.dengue_sensors(auth="sensors_key", names="ght", locations="ag", epiweeks="201432") self.assertEqual( response, { diff --git a/integrations/server/test_ecdc_ili.py b/integrations/server/test_ecdc_ili.py index 3c65cb2c2..5fe24bcd3 100644 --- a/integrations/server/test_ecdc_ili.py +++ b/integrations/server/test_ecdc_ili.py @@ -1,15 +1,12 @@ # first party -from delphi.epidata.common.integration_test_base_class import BasicIntegrationTest +from delphi.epidata.common.integration_test_base_class import DelphiTestBase -class EcdcIliTest(BasicIntegrationTest): +class EcdcIliTest(DelphiTestBase): """Basic integration tests for edcd_ili endpint.""" - def setUp(self) -> None: - """Perform per-test setup.""" - + def localSetUp(self): self.truncate_tables_list = ["ecdc_ili"] - super().setUp() def test_ecdc_ili(self): """Basic integration test for ecdc_ili endpoint""" @@ -18,7 +15,7 @@ def test_ecdc_ili(self): ("2020-03-26", "202012", "201840", "76", "Armenia", "0"), ) self.cnx.commit() - response = self.epidata.ecdc_ili(regions="Armenia", epiweeks="201840") + response = self.epidata_client.ecdc_ili(regions="Armenia", epiweeks="201840") self.assertEqual( response, { diff --git a/integrations/server/test_flusurv.py b/integrations/server/test_flusurv.py index e818a2698..33f0f00b8 100644 --- a/integrations/server/test_flusurv.py +++ b/integrations/server/test_flusurv.py @@ -1,15 +1,12 @@ # first party -from delphi.epidata.common.integration_test_base_class import BasicIntegrationTest +from delphi.epidata.common.integration_test_base_class import DelphiTestBase -class FlusurvTest(BasicIntegrationTest): +class FlusurvTest(DelphiTestBase): """Basic integration tests for flusurv endpint.""" - def setUp(self) -> None: - """Perform per-test setup.""" - + def localSetUp(self): self.truncate_tables_list = ["flusurv"] - super().setUp() def test_flusurv(self): """Basic integration test for flusurv endpoint""" @@ -18,7 +15,7 @@ def test_flusurv(self): ("2012-11-02", "201243", "201143", "CA", "52", "0", "0", "0", "0.151", "0", "0.029", "0", "0", "0"), ) self.cnx.commit() - response = self.epidata.flusurv(epiweeks=201143, locations="CA") + response = self.epidata_client.flusurv(epiweeks=201143, locations="CA") self.assertEqual( response, { diff --git a/integrations/server/test_fluview_clinical.py b/integrations/server/test_fluview_clinical.py index 77f0d76ea..8ebccfc9e 100644 --- a/integrations/server/test_fluview_clinical.py +++ b/integrations/server/test_fluview_clinical.py @@ -1,15 +1,12 @@ # first party -from delphi.epidata.common.integration_test_base_class import BasicIntegrationTest +from delphi.epidata.common.integration_test_base_class import DelphiTestBase -class FluviewClinicalTest(BasicIntegrationTest): +class FluviewClinicalTest(DelphiTestBase): """Basic integration tests for fluview_clinical endpint.""" - def setUp(self) -> None: - """Perform per-test setup.""" - + def localSetUp(self): self.truncate_tables_list = ["fluview_clinical"] - super().setUp() def test_fluview_clinical(self): """Basic integration test for fluview_clinical endpoint""" @@ -18,7 +15,7 @@ def test_fluview_clinical(self): ("2018-10-10", "201839", "201640", "al", "103", "406", "4", "1", "1.32", "0.99", "0.25"), ) self.cnx.commit() - response = self.epidata.fluview_clinical(epiweeks=201640, regions="al") + response = self.epidata_client.fluview_clinical(epiweeks=201640, regions="al") self.assertEqual( response, { diff --git a/integrations/server/test_gft.py b/integrations/server/test_gft.py index f241769f4..3f2a68526 100644 --- a/integrations/server/test_gft.py +++ b/integrations/server/test_gft.py @@ -1,15 +1,12 @@ # first party -from delphi.epidata.common.integration_test_base_class import BasicIntegrationTest +from delphi.epidata.common.integration_test_base_class import DelphiTestBase -class GftTest(BasicIntegrationTest): +class GftTest(DelphiTestBase): """Basic integration tests for gft endpint.""" - def setUp(self) -> None: - """Perform per-test setup.""" - + def localSetUp(self): self.truncate_tables_list = ["gft"] - super().setUp() def test_gft(self): """Basic integration test for gft endpoint""" @@ -18,7 +15,7 @@ def test_gft(self): ("200340", "nat", "902"), ) self.cnx.commit() - response = self.epidata.gft(locations="nat", epiweeks="200340") + response = self.epidata_client.gft(locations="nat", epiweeks="200340") self.assertEqual( response, {"epidata": [{"location": "nat", "epiweek": 200340, "num": 902}], "result": 1, "message": "success"}, diff --git a/integrations/server/test_ght.py b/integrations/server/test_ght.py index a8ea6f33d..67b135aef 100644 --- a/integrations/server/test_ght.py +++ b/integrations/server/test_ght.py @@ -1,16 +1,13 @@ # first party -from delphi.epidata.common.integration_test_base_class import BasicIntegrationTest +from delphi.epidata.common.integration_test_base_class import DelphiTestBase -class GhtTest(BasicIntegrationTest): +class GhtTest(DelphiTestBase): """Basic integration tests for ght endpint.""" - def setUp(self) -> None: - """Perform per-test setup.""" - + def localSetUp(self): self.truncate_tables_list = ["ght"] self.role_name = "ght" - super().setUp() def test_ght(self): """Basic integration test for ght endpoint""" @@ -19,7 +16,7 @@ def test_ght(self): ("/n/query", "US", "200101", "12345"), ) self.cnx.commit() - response = self.epidata.ght(locations="US", epiweeks="200101", query="/n/query", auth="ght_key") + response = self.epidata_client.ght(locations="US", epiweeks="200101", query="/n/query", auth="ght_key") self.assertEqual( response, {"epidata": [{"location": "US", "epiweek": 200101, "value": 12345.0}], "result": 1, "message": "success"}, diff --git a/integrations/server/test_kcdc_ili.py b/integrations/server/test_kcdc_ili.py index 5474c28b9..aab40baa6 100644 --- a/integrations/server/test_kcdc_ili.py +++ b/integrations/server/test_kcdc_ili.py @@ -1,15 +1,12 @@ # first party -from delphi.epidata.common.integration_test_base_class import BasicIntegrationTest +from delphi.epidata.common.integration_test_base_class import DelphiTestBase -class KcdcIliTest(BasicIntegrationTest): +class KcdcIliTest(DelphiTestBase): """Basic integration tests for kcdc_ili endpint.""" - def setUp(self) -> None: - """Perform per-test setup.""" - + def localSetUp(self): self.truncate_tables_list = ["kcdc_ili"] - super().setUp() def test_kcdc_ili(self): """Basic integration test for kcdc_ili endpoint""" @@ -18,7 +15,7 @@ def test_kcdc_ili(self): ("2020-03-27", "202013", "200432", "222", "REG", "0.25"), ) self.cnx.commit() - response = self.epidata.kcdc_ili(regions="REG", epiweeks="200432") + response = self.epidata_client.kcdc_ili(regions="REG", epiweeks="200432") self.assertEqual( response, { diff --git a/integrations/server/test_meta.py b/integrations/server/test_meta.py index 4bcc6e7cb..295a5cd98 100644 --- a/integrations/server/test_meta.py +++ b/integrations/server/test_meta.py @@ -1,17 +1,16 @@ # first party -from delphi.epidata.common.integration_test_base_class import BasicIntegrationTest +from delphi.epidata.common.integration_test_base_class import DelphiTestBase -class MetaTest(BasicIntegrationTest): +class MetaTest(DelphiTestBase): """Basic integration tests for meta endpint.""" - def setUp(self) -> None: + def localSetUp(self): self.truncate_tables_list = ["forecasts", "fluview", "wiki", "wiki_meta", "twitter"] - super().setUp() def test_meta(self): """Basic integration test for meta endpoint""" - response = self.epidata.meta() + response = self.epidata_client.meta() self.assertEqual( response, { diff --git a/integrations/server/test_meta_norostat.py b/integrations/server/test_meta_norostat.py index b75b86a1e..e336edba3 100644 --- a/integrations/server/test_meta_norostat.py +++ b/integrations/server/test_meta_norostat.py @@ -1,13 +1,11 @@ # first party -from delphi.epidata.common.integration_test_base_class import BasicIntegrationTest +from delphi.epidata.common.integration_test_base_class import DelphiTestBase -class MetaNorostatTest(BasicIntegrationTest): +class MetaNorostatTest(DelphiTestBase): """Basic integration tests for meta_norostat endpint.""" - def setUp(self) -> None: - """Perform per-test setup.""" - + def localSetUp(self): create_raw_datatable_version_list = """ CREATE TABLE IF NOT EXISTS `norostat_raw_datatable_version_list` ( `release_date` date NOT NULL, @@ -64,8 +62,6 @@ def setUp(self) -> None: self.role_name = "norostat" - super().setUp() - def test_meta_norostat(self): """Basic integration test for meta_norostat endpoint""" @@ -77,7 +73,7 @@ def test_meta_norostat(self): 'INSERT INTO `norostat_raw_datatable_location_pool`(`location`) VALUES ("Minnesota, Ohio, Oregon, Tennessee, and Wisconsin")' ) self.cnx.commit() - response = self.epidata.meta_norostat(auth="norostat_key") + response = self.epidata_client.meta_norostat(auth="norostat_key") self.assertEqual( response, { diff --git a/integrations/server/test_nidss_dengue.py b/integrations/server/test_nidss_dengue.py index fc537ffcc..679937ea6 100644 --- a/integrations/server/test_nidss_dengue.py +++ b/integrations/server/test_nidss_dengue.py @@ -1,15 +1,12 @@ # first party -from delphi.epidata.common.integration_test_base_class import BasicIntegrationTest +from delphi.epidata.common.integration_test_base_class import DelphiTestBase -class NiddsDengueTest(BasicIntegrationTest): +class NiddsDengueTest(DelphiTestBase): """Basic integration tests for nids_dengue endpint.""" - def setUp(self) -> None: - """Perform per-test setup.""" - + def localSetUp(self): self.truncate_tables_list = ["nidss_dengue"] - super().setUp() def test_nidss_dengue(self): """Basic integration test for nidds_dengue endpoint""" @@ -18,7 +15,7 @@ def test_nidss_dengue(self): ("200340", "SomeCity", "Central", "0"), ) self.cnx.commit() - response = self.epidata.nidss_dengue(locations="SomeCity", epiweeks="200340") + response = self.epidata_client.nidss_dengue(locations="SomeCity", epiweeks="200340") self.assertEqual( response, {"epidata": [{"location": "SomeCity", "epiweek": 200340, "count": 0}], "result": 1, "message": "success"}, diff --git a/integrations/server/test_nidss_flu.py b/integrations/server/test_nidss_flu.py index 9cb06e1fd..0b13ee67f 100644 --- a/integrations/server/test_nidss_flu.py +++ b/integrations/server/test_nidss_flu.py @@ -1,15 +1,12 @@ # first party -from delphi.epidata.common.integration_test_base_class import BasicIntegrationTest +from delphi.epidata.common.integration_test_base_class import DelphiTestBase -class NiddsFluTest(BasicIntegrationTest): +class NiddsFluTest(DelphiTestBase): """Basic integration tests for nids_flu endpint.""" - def setUp(self) -> None: - """Perform per-test setup.""" - + def localSetUp(self): self.truncate_tables_list = ["nidss_flu"] - super().setUp() def test_nidss_flu(self): """Basic integration test for nidds_flu endpoint""" @@ -18,7 +15,7 @@ def test_nidss_flu(self): ("2015-09-05", "201530", "200111", "SomeRegion", "222", "333", "444"), ) self.cnx.commit() - response = self.epidata.nidss_flu(regions="SomeRegion", epiweeks="200111") + response = self.epidata_client.nidss_flu(regions="SomeRegion", epiweeks="200111") self.assertEqual( response, { diff --git a/integrations/server/test_norostat.py b/integrations/server/test_norostat.py index 7adb901e7..956533c1e 100644 --- a/integrations/server/test_norostat.py +++ b/integrations/server/test_norostat.py @@ -1,13 +1,11 @@ # first party -from delphi.epidata.common.integration_test_base_class import BasicIntegrationTest +from delphi.epidata.common.integration_test_base_class import DelphiTestBase -class NorostatTest(BasicIntegrationTest): +class NorostatTest(DelphiTestBase): """Basic integration tests for norostat endpint.""" - def setUp(self) -> None: - """Perform per-test setup.""" - + def localSetUp(self): create_norostat_point_diffs = """ CREATE TABLE IF NOT EXISTS `norostat_point_diffs` ( `release_date` date NOT NULL, @@ -30,7 +28,6 @@ def setUp(self) -> None: "norostat_raw_datatable_version_list", ] self.role_name = "norostat" - super().setUp() def test_norostat(self): """Basic integration test for norostat endpoint""" @@ -47,7 +44,7 @@ def test_norostat(self): 'INSERT INTO `norostat_point_diffs`(`release_date`, `parse_time`, `location_id`, `epiweek`, `new_value`) VALUES("2023-07-19", "2023-07-10 15:24:51", "1", "202329", 10)' ) self.cnx.commit() - response = self.epidata.norostat(auth="norostat_key", location="SomeTestLocation", epiweeks="202329") + response = self.epidata_client.norostat(auth="norostat_key", location="SomeTestLocation", epiweeks="202329") self.assertEqual( response, { diff --git a/integrations/server/test_nowcast.py b/integrations/server/test_nowcast.py index 51332d342..2b48dd0da 100644 --- a/integrations/server/test_nowcast.py +++ b/integrations/server/test_nowcast.py @@ -1,15 +1,12 @@ # first party -from delphi.epidata.common.integration_test_base_class import BasicIntegrationTest +from delphi.epidata.common.integration_test_base_class import DelphiTestBase -class NowcastTest(BasicIntegrationTest): +class NowcastTest(DelphiTestBase): """Basic integration tests for nowcast endpint.""" - def setUp(self) -> None: - """Perform per-test setup.""" - + def localSetUp(self): self.truncate_tables_list = ["nowcasts"] - super().setUp() def test_nowcast(self): """Basic integration test for nowcast endpoint""" @@ -18,7 +15,7 @@ def test_nowcast(self): ("201145", "nat", "12345", "0.01234"), ) self.cnx.commit() - response = self.epidata.nowcast(locations="nat", epiweeks="201145") + response = self.epidata_client.nowcast(locations="nat", epiweeks="201145") self.assertEqual( response, { diff --git a/integrations/server/test_paho_dengue.py b/integrations/server/test_paho_dengue.py index 0bb5391ed..bbe8953f8 100644 --- a/integrations/server/test_paho_dengue.py +++ b/integrations/server/test_paho_dengue.py @@ -1,15 +1,12 @@ # first party -from delphi.epidata.common.integration_test_base_class import BasicIntegrationTest +from delphi.epidata.common.integration_test_base_class import DelphiTestBase -class PahoDengueTest(BasicIntegrationTest): +class PahoDengueTest(DelphiTestBase): """Basic integration tests for paho_dengue endpint.""" - def setUp(self) -> None: - """Perform per-test setup.""" - + def localSetUp(self): self.truncate_tables_list = ["paho_dengue"] - super().setUp() def test_paho_dengue(self): """Basic integration test for paho_dengue endpoint""" @@ -18,7 +15,7 @@ def test_paho_dengue(self): ("2018-12-01", "201848", "201454", "204", "AG", "91", "DEN 1,4", "37", "40.66", "0", "0"), ) self.cnx.commit() - response = self.epidata.paho_dengue(regions="AG", epiweeks="201454") + response = self.epidata_client.paho_dengue(regions="AG", epiweeks="201454") self.assertEqual( response, { diff --git a/integrations/server/test_quidel.py b/integrations/server/test_quidel.py index 6ceefd273..696a7ee41 100644 --- a/integrations/server/test_quidel.py +++ b/integrations/server/test_quidel.py @@ -1,16 +1,13 @@ # first party -from delphi.epidata.common.integration_test_base_class import BasicIntegrationTest +from delphi.epidata.common.integration_test_base_class import DelphiTestBase -class QuidelTest(BasicIntegrationTest): +class QuidelTest(DelphiTestBase): """Basic integration tests for quidel endpint.""" - def setUp(self) -> None: - """Perform per-test setup.""" - + def localSetUp(self): self.truncate_tables_list = ["quidel"] self.role_name = "quidel" - super().setUp() def test_quidel(self): """Basic integration test for quidel endpoint""" @@ -19,7 +16,7 @@ def test_quidel(self): ("loc1", "201111", "1", "0", "0"), ) self.cnx.commit() - response = self.epidata.quidel(locations="loc1", epiweeks="201111", auth="quidel_key") + response = self.epidata_client.quidel(locations="loc1", epiweeks="201111", auth="quidel_key") self.assertEqual( response, {"epidata": [{"location": "loc1", "epiweek": 201111, "value": 1.0}], "result": 1, "message": "success"}, diff --git a/integrations/server/test_sensors.py b/integrations/server/test_sensors.py index d236bf7eb..835b53893 100644 --- a/integrations/server/test_sensors.py +++ b/integrations/server/test_sensors.py @@ -1,16 +1,13 @@ # first party -from delphi.epidata.common.integration_test_base_class import BasicIntegrationTest +from delphi.epidata.common.integration_test_base_class import DelphiTestBase -class SensorsTest(BasicIntegrationTest): +class SensorsTest(DelphiTestBase): """Basic integration tests for sensors endpint.""" - def setUp(self) -> None: - """Perform per-test setup.""" - + def localSetUp(self): self.truncate_tables_list = ["sensors"] self.role_name = "sensors" - super().setUp() def test_sensors(self): """Basic integration test for sensors endpoint""" @@ -19,7 +16,7 @@ def test_sensors(self): ("sens1", "201111", "loc1", "222"), ) self.cnx.commit() - response = self.epidata.sensors(names="sens1", locations="loc1", epiweeks="201111", auth="sensors_key") + response = self.epidata_client.sensors(names="sens1", locations="loc1", epiweeks="201111", auth="sensors_key") self.assertEqual( response, { diff --git a/integrations/server/test_signal_dashboard.py b/integrations/server/test_signal_dashboard.py index bebf6357f..e99a464fd 100644 --- a/integrations/server/test_signal_dashboard.py +++ b/integrations/server/test_signal_dashboard.py @@ -1,8 +1,8 @@ # first party -from delphi.epidata.common.integration_test_base_class import BasicIntegrationTest +from delphi.epidata.common.integration_test_base_class import DelphiTestBase -class SignalDashboardTest(BasicIntegrationTest): +class SignalDashboardTest(DelphiTestBase): """Basic integration tests for signal_dashboard_coverage and signal_dashboard_status endpints.""" def setUp(self) -> None: @@ -28,7 +28,7 @@ def test_signal_dashboard_coverage(self): params = { "endpoint": "signal_dashboard_coverage", } - response = self.epidata._request(params=params) + response = self.epidata_client._request(params=params) self.assertEqual( response, { @@ -44,7 +44,7 @@ def test_signal_dashboard_status(self): params = { "endpoint": "signal_dashboard_status", } - response = self.epidata._request(params=params) + response = self.epidata_client._request(params=params) self.assertEqual( response, { diff --git a/integrations/server/test_twitter.py b/integrations/server/test_twitter.py index b32ad8290..7ba162480 100644 --- a/integrations/server/test_twitter.py +++ b/integrations/server/test_twitter.py @@ -1,16 +1,13 @@ # first party -from delphi.epidata.common.integration_test_base_class import BasicIntegrationTest +from delphi.epidata.common.integration_test_base_class import DelphiTestBase -class TwitterTest(BasicIntegrationTest): +class TwitterTest(DelphiTestBase): """Basic integration tests for twitter endpint.""" - def setUp(self) -> None: - """Perform per-test setup.""" - + def localSetUp(self): self.truncate_tables_list = ["twitter"] self.role_name = "twitter" - super().setUp() def test_twitter(self): """Basic integration test for twitter endpoint""" @@ -19,7 +16,7 @@ def test_twitter(self): 'INSERT INTO `twitter`(`date`, `state`, `num`, `total`) VALUES ("2015-07-29", "AK", "1", "223"), ("2020-07-29", "CT", "12", "778")', ) self.cnx.commit() - response = self.epidata.twitter(auth="twitter_key", locations="cen9", dates="20150701-20160101") + response = self.epidata_client.twitter(auth="twitter_key", locations="cen9", dates="20150701-20160101") self.assertEqual( response, { diff --git a/integrations/server/test_wiki.py b/integrations/server/test_wiki.py index 982c1448b..d53cef0c9 100644 --- a/integrations/server/test_wiki.py +++ b/integrations/server/test_wiki.py @@ -1,15 +1,12 @@ # first party -from delphi.epidata.common.integration_test_base_class import BasicIntegrationTest +from delphi.epidata.common.integration_test_base_class import DelphiTestBase -class WikiTest(BasicIntegrationTest): +class WikiTest(DelphiTestBase): """Basic integration tests for wiki endpint.""" - def setUp(self) -> None: - """Perform per-test setup.""" - + def localSetUp(self): self.truncate_tables_list = ["wiki", "wiki_meta"] - super().setUp() def test_wiki(self): """Basic integration test for wiki endpoint""" @@ -21,7 +18,7 @@ def test_wiki(self): 'INSERT INTO `wiki_meta`(`datetime`, `date`, `epiweek`, `total`, `language`) VALUES ("2007-12-09 18:00:00", "2007-12-09", "200750", "969214", "en"), ("2008-12-09 18:00:00", "2008-12-09", "200750", "123321", "en")' ) self.cnx.commit() - response = self.epidata.wiki(articles="test", epiweeks="200701-200801") + response = self.epidata_client.wiki(articles="test", epiweeks="200701-200801") self.assertEqual( response, { diff --git a/src/common/integration_test_base_class.py b/src/common/integration_test_base_class.py index 580331d4a..8ccbdf3ce 100644 --- a/src/common/integration_test_base_class.py +++ b/src/common/integration_test_base_class.py @@ -9,7 +9,7 @@ from delphi.epidata.server._limiter import limiter -class BasicIntegrationTest(unittest.TestCase): +class DelphiTestBase(unittest.TestCase): """Basic integration test class""" def __init__(self, methodName: str = "runTest") -> None: @@ -18,26 +18,15 @@ def __init__(self, methodName: str = "runTest") -> None: self.truncate_tables_list = [] self.create_tables_list = [] self.role_name = None - self.epidata = Epidata - self.epidata.BASE_URL = "http://delphi_web_epidata/epidata/api.php" - self.epidata.auth = ("epidata", "key") - - def delete_from_table(self, cur, table_name: str) -> None: - cur.execute(f"DELETE FROM `{table_name}`") - - def truncate_table(self, cur, table_name: str) -> None: - cur.execute(f"TRUNCATE TABLE `{table_name}`") - - def create_table(self, cur, create_table_stmt: str): - cur.execute(create_table_stmt) + self.epidata_client = Epidata + self.epidata_client.BASE_URL = "http://delphi_web_epidata/epidata/api.php" + self.epidata_client.auth = ("epidata", "key") def create_key_with_role(self, cur, role_name: str): - cur.execute("TRUNCATE TABLE `user_role`") - cur.execute("TRUNCATE TABLE `user_role_link`") cur.execute(f'INSERT INTO `api_user`(`api_key`, `email`) VALUES("{role_name}_key", "{role_name}_email")') cur.execute(f'INSERT INTO `user_role`(`name`) VALUES("{role_name}")') cur.execute( - f'INSERT INTO `user_role_link`(`user_id`, `role_id`) SELECT `api_user`.`id`, 1 FROM `api_user` WHERE `api_key`="{role_name}_key"' + f'INSERT INTO `user_role_link`(`user_id`, `role_id`) SELECT `api_user`.`id`, `user_role`.`id` FROM `api_user` JOIN `user_role` WHERE `api_user`.`api_key`="{role_name}_key" AND `user_role`.`name`="{role_name}"' ) def setUp(self) -> None: @@ -48,16 +37,20 @@ def setUp(self) -> None: cur = cnx.cursor() cur.execute("DELETE FROM `api_user`") + cur.execute("TRUNCATE TABLE `user_role`") + cur.execute("TRUNCATE TABLE `user_role_link`") cur.execute('INSERT INTO `api_user`(`api_key`, `email`) VALUES ("key", "email")') + self.localSetUp() + for stmt in self.create_tables_list: - self.create_table(cur, stmt) + cur.execute(stmt) for table_name in self.delete_from_tables_list: - self.delete_from_table(cur, table_name) + cur.execute(f"DELETE FROM `{table_name}`") for table_name in self.truncate_tables_list: - self.truncate_table(cur, table_name) + cur.execute(f"TRUNCATE TABLE `{table_name}`") if self.role_name: self.create_key_with_role(cur, self.role_name) @@ -68,6 +61,11 @@ def setUp(self) -> None: self.cnx = cnx self.cur = cnx.cursor() + def localSetUp(self): + # stub; override in subclasses to perform custom setup. + # runs after tables have been truncated but before database changes have been committed + pass + @staticmethod def _clear_limits() -> None: limiter.storage.reset() From 9e4a332dcadd27ced7693c9689629ab50d2b3e45 Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Thu, 21 Sep 2023 22:28:31 +0300 Subject: [PATCH 16/55] Moved test_meta_norostat to test_norostat.py --- integrations/server/test_norostat.py | 57 +++++++++++++++++++++++++++- 1 file changed, 56 insertions(+), 1 deletion(-) diff --git a/integrations/server/test_norostat.py b/integrations/server/test_norostat.py index 956533c1e..c6ccfc119 100644 --- a/integrations/server/test_norostat.py +++ b/integrations/server/test_norostat.py @@ -19,8 +19,39 @@ def localSetUp(self): CONSTRAINT `norostat_point_diffs_ibfk_2` FOREIGN KEY (`location_id`) REFERENCES `norostat_raw_datatable_location_pool` (`location_id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3; """ + create_raw_datatable_version_list = """ + CREATE TABLE IF NOT EXISTS `norostat_raw_datatable_version_list` ( + `release_date` date NOT NULL, + `parse_time` datetime NOT NULL, + PRIMARY KEY (`release_date`,`parse_time`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3; + """ + + create_norostat_version_list = """ + CREATE TABLE IF NOT EXISTS `norostat_point_version_list` ( + `release_date` date NOT NULL, + `parse_time` datetime NOT NULL, + PRIMARY KEY (`release_date`,`parse_time`), + CONSTRAINT `norostat_point_version_list_ibfk_1` FOREIGN KEY (`release_date`, `parse_time`) REFERENCES `norostat_raw_datatable_version_list` (`release_date`, `parse_time`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3; + """ + + create_norostat_datatable_location_pool = """ + CREATE TABLE IF NOT EXISTS `norostat_raw_datatable_location_pool` ( + `location_id` int NOT NULL AUTO_INCREMENT, + `location` varchar(255) CHARACTER SET utf8mb3 COLLATE utf8mb3_general_ci NOT NULL, + PRIMARY KEY (`location_id`), + UNIQUE KEY `location` (`location`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3; + """ + + self.create_tables_list = [ + create_raw_datatable_version_list, + create_norostat_version_list, + create_norostat_datatable_location_pool, + create_norostat_point_diffs, + ] - self.create_tables_list = [create_norostat_point_diffs] self.delete_from_tables_list = [ "norostat_point_diffs", "norostat_point_version_list", @@ -54,3 +85,27 @@ def test_norostat(self): }, ) return True + + def test_meta_norostat(self): + """Basic integration test for meta_norostat endpoint""" + + self.cur.execute( + "INSERT INTO `norostat_raw_datatable_version_list`(`release_date`, `parse_time`) VALUES (%s, %s)", + ("2014-10-22", "2048-12-08 15:22:51"), + ) + self.cur.execute( + 'INSERT INTO `norostat_raw_datatable_location_pool`(`location`) VALUES ("Minnesota, Ohio, Oregon, Tennessee, and Wisconsin")' + ) + self.cnx.commit() + response = self.epidata_client.meta_norostat(auth="norostat_key") + self.assertEqual( + response, + { + "epidata": { + "locations": [{"location": "Minnesota, Ohio, Oregon, Tennessee, and Wisconsin"}], + "releases": [{"release_date": "2014-10-22"}], + }, + "message": "success", + "result": 1, + }, + ) From 80c355c9da39efae9d6edbdd65177ecbbce50928 Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Fri, 22 Sep 2023 13:28:06 +0300 Subject: [PATCH 17/55] Remove test_meta_norostat.py as this test was moved to test_norostat.py --- integrations/server/test_meta_norostat.py | 87 ----------------------- 1 file changed, 87 deletions(-) delete mode 100644 integrations/server/test_meta_norostat.py diff --git a/integrations/server/test_meta_norostat.py b/integrations/server/test_meta_norostat.py deleted file mode 100644 index e336edba3..000000000 --- a/integrations/server/test_meta_norostat.py +++ /dev/null @@ -1,87 +0,0 @@ -# first party -from delphi.epidata.common.integration_test_base_class import DelphiTestBase - - -class MetaNorostatTest(DelphiTestBase): - """Basic integration tests for meta_norostat endpint.""" - - def localSetUp(self): - create_raw_datatable_version_list = """ - CREATE TABLE IF NOT EXISTS `norostat_raw_datatable_version_list` ( - `release_date` date NOT NULL, - `parse_time` datetime NOT NULL, - PRIMARY KEY (`release_date`,`parse_time`) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3; - """ - - create_norostat_version_list = """ - CREATE TABLE IF NOT EXISTS `norostat_point_version_list` ( - `release_date` date NOT NULL, - `parse_time` datetime NOT NULL, - PRIMARY KEY (`release_date`,`parse_time`), - CONSTRAINT `norostat_point_version_list_ibfk_1` FOREIGN KEY (`release_date`, `parse_time`) REFERENCES `norostat_raw_datatable_version_list` (`release_date`, `parse_time`) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3; - """ - - create_norostat_datatable_location_pool = """ - CREATE TABLE IF NOT EXISTS `norostat_raw_datatable_location_pool` ( - `location_id` int NOT NULL AUTO_INCREMENT, - `location` varchar(255) CHARACTER SET utf8mb3 COLLATE utf8mb3_general_ci NOT NULL, - PRIMARY KEY (`location_id`), - UNIQUE KEY `location` (`location`) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3; - """ - - create_norostat_point_diffs = """ - CREATE TABLE IF NOT EXISTS `norostat_point_diffs` ( - `release_date` date NOT NULL, - `parse_time` datetime NOT NULL, - `location_id` int NOT NULL, - `epiweek` int NOT NULL, - `new_value` varchar(255) CHARACTER SET utf8mb3 COLLATE utf8mb3_general_ci DEFAULT NULL, - PRIMARY KEY (`release_date`,`parse_time`,`location_id`,`epiweek`), - UNIQUE KEY `location_id` (`location_id`,`epiweek`,`release_date`,`parse_time`,`new_value`), - CONSTRAINT `norostat_point_diffs_ibfk_1` FOREIGN KEY (`release_date`, `parse_time`) REFERENCES `norostat_point_version_list` (`release_date`, `parse_time`), - CONSTRAINT `norostat_point_diffs_ibfk_2` FOREIGN KEY (`location_id`) REFERENCES `norostat_raw_datatable_location_pool` (`location_id`) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3; - """ - - self.create_tables_list = [ - create_raw_datatable_version_list, - create_norostat_version_list, - create_norostat_datatable_location_pool, - create_norostat_point_diffs, - ] - - self.delete_from_tables_list = [ - "norostat_point_diffs", - "norostat_point_version_list", - "norostat_raw_datatable_location_pool", - "norostat_raw_datatable_version_list", - ] - - self.role_name = "norostat" - - def test_meta_norostat(self): - """Basic integration test for meta_norostat endpoint""" - - self.cur.execute( - "INSERT INTO `norostat_raw_datatable_version_list`(`release_date`, `parse_time`) VALUES (%s, %s)", - ("2014-10-22", "2048-12-08 15:22:51"), - ) - self.cur.execute( - 'INSERT INTO `norostat_raw_datatable_location_pool`(`location`) VALUES ("Minnesota, Ohio, Oregon, Tennessee, and Wisconsin")' - ) - self.cnx.commit() - response = self.epidata_client.meta_norostat(auth="norostat_key") - self.assertEqual( - response, - { - "epidata": { - "locations": [{"location": "Minnesota, Ohio, Oregon, Tennessee, and Wisconsin"}], - "releases": [{"release_date": "2014-10-22"}], - }, - "message": "success", - "result": 1, - }, - ) From 5e3210f6131cf6d0f737777df2fc82b97e01cc74 Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Fri, 22 Sep 2023 13:29:26 +0300 Subject: [PATCH 18/55] Update integrations/server/test_api_keys.py Co-authored-by: melange396 --- integrations/server/test_api_keys.py | 1 - 1 file changed, 1 deletion(-) diff --git a/integrations/server/test_api_keys.py b/integrations/server/test_api_keys.py index e522fe9d7..418e265ac 100644 --- a/integrations/server/test_api_keys.py +++ b/integrations/server/test_api_keys.py @@ -146,7 +146,6 @@ def test_multiples_mixed_allowed_signal_api_key(self): status_codes.add( self._make_request(params=params, auth=self.epidata_client.auth).status_code ) - self.assertEqual(len(status_codes), 1) self.assertEqual(status_codes, {200}) def test_multiples_allowed_signal_api_key(self): From dca4c0b3346b8c3c24e202eab3292affb899495f Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Fri, 22 Sep 2023 13:31:26 +0300 Subject: [PATCH 19/55] Update integrations/server/test_norostat.py Co-authored-by: melange396 --- integrations/server/test_norostat.py | 1 + 1 file changed, 1 insertion(+) diff --git a/integrations/server/test_norostat.py b/integrations/server/test_norostat.py index c6ccfc119..8fc3ca532 100644 --- a/integrations/server/test_norostat.py +++ b/integrations/server/test_norostat.py @@ -19,6 +19,7 @@ def localSetUp(self): CONSTRAINT `norostat_point_diffs_ibfk_2` FOREIGN KEY (`location_id`) REFERENCES `norostat_raw_datatable_location_pool` (`location_id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3; """ + create_raw_datatable_version_list = """ CREATE TABLE IF NOT EXISTS `norostat_raw_datatable_version_list` ( `release_date` date NOT NULL, From 27f66a429879fce6d09dbca2d5c381d16fb4853e Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Fri, 22 Sep 2023 13:31:51 +0300 Subject: [PATCH 20/55] Update src/common/integration_test_base_class.py Co-authored-by: melange396 --- src/common/integration_test_base_class.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/integration_test_base_class.py b/src/common/integration_test_base_class.py index 8ccbdf3ce..387c0f617 100644 --- a/src/common/integration_test_base_class.py +++ b/src/common/integration_test_base_class.py @@ -63,7 +63,7 @@ def setUp(self) -> None: def localSetUp(self): # stub; override in subclasses to perform custom setup. - # runs after tables have been truncated but before database changes have been committed + # runs after user/api_key tables have been truncated, but before test-specific tables are created/deleted/truncated and before database changes have been committed pass @staticmethod From d9d83f3b6afb5225b7c675db8e3770a6f2af3629 Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Fri, 22 Sep 2023 13:32:25 +0300 Subject: [PATCH 21/55] Update integrations/server/test_norostat.py Co-authored-by: melange396 --- integrations/server/test_norostat.py | 1 + 1 file changed, 1 insertion(+) diff --git a/integrations/server/test_norostat.py b/integrations/server/test_norostat.py index 8fc3ca532..a7866a91d 100644 --- a/integrations/server/test_norostat.py +++ b/integrations/server/test_norostat.py @@ -59,6 +59,7 @@ def localSetUp(self): "norostat_raw_datatable_location_pool", "norostat_raw_datatable_version_list", ] + self.role_name = "norostat" def test_norostat(self): From 386c348308e87e26749cc1f11839ac16f657f4f6 Mon Sep 17 00:00:00 2001 From: Rostyslav Zatserkovnyi Date: Tue, 19 Sep 2023 15:32:08 +0300 Subject: [PATCH 22/55] Remove usage of PHP alias in the Python client --- .../covid_hosp/facility/test_scenarios.py | 2 +- .../covid_hosp/state_daily/test_scenarios.py | 2 +- .../state_timeseries/test_scenarios.py | 2 +- .../covidcast/test_covidcast_meta_caching.py | 6 +- .../covidcast/test_csv_uploading.py | 17 ++- .../covidcast_nowcast/test_csv_uploading.py | 2 +- integrations/client/test_delphi_epidata.py | 41 ++++--- integrations/client/test_nowcast.py | 4 +- integrations/server/test_covid_hosp.py | 2 +- integrations/server/test_covidcast.py | 35 +++--- integrations/server/test_covidcast_meta.py | 5 +- integrations/server/test_covidcast_nowcast.py | 5 +- integrations/server/test_fluview.py | 2 +- integrations/server/test_fluview_meta.py | 2 +- src/client/delphi_epidata.py | 109 +++++++----------- 15 files changed, 108 insertions(+), 128 deletions(-) diff --git a/integrations/acquisition/covid_hosp/facility/test_scenarios.py b/integrations/acquisition/covid_hosp/facility/test_scenarios.py index c6c51e2f5..44ee3572d 100644 --- a/integrations/acquisition/covid_hosp/facility/test_scenarios.py +++ b/integrations/acquisition/covid_hosp/facility/test_scenarios.py @@ -28,7 +28,7 @@ def setUp(self): self.test_utils = UnitTestUtils(__file__) # use the local instance of the Epidata API - Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php' + Epidata.BASE_URL = 'http://delphi_web_epidata/epidata' Epidata.auth = ('epidata', 'key') # use the local instance of the epidata database diff --git a/integrations/acquisition/covid_hosp/state_daily/test_scenarios.py b/integrations/acquisition/covid_hosp/state_daily/test_scenarios.py index 2054d19c8..8636295bc 100644 --- a/integrations/acquisition/covid_hosp/state_daily/test_scenarios.py +++ b/integrations/acquisition/covid_hosp/state_daily/test_scenarios.py @@ -32,7 +32,7 @@ def setUp(self): self.test_utils = UnitTestUtils(__file__) # use the local instance of the Epidata API - Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php' + Epidata.BASE_URL = 'http://delphi_web_epidata/epidata' Epidata.auth = ('epidata', 'key') # use the local instance of the epidata database diff --git a/integrations/acquisition/covid_hosp/state_timeseries/test_scenarios.py b/integrations/acquisition/covid_hosp/state_timeseries/test_scenarios.py index 8565b8e7f..46bdeebcd 100644 --- a/integrations/acquisition/covid_hosp/state_timeseries/test_scenarios.py +++ b/integrations/acquisition/covid_hosp/state_timeseries/test_scenarios.py @@ -28,7 +28,7 @@ def setUp(self): self.test_utils = UnitTestUtils(__file__) # use the local instance of the Epidata API - Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php' + Epidata.BASE_URL = 'http://delphi_web_epidata/epidata' Epidata.auth = ('epidata', 'key') # use the local instance of the epidata database diff --git a/integrations/acquisition/covidcast/test_covidcast_meta_caching.py b/integrations/acquisition/covidcast/test_covidcast_meta_caching.py index 16e2506fc..30b6bbcf2 100644 --- a/integrations/acquisition/covidcast/test_covidcast_meta_caching.py +++ b/integrations/acquisition/covidcast/test_covidcast_meta_caching.py @@ -22,7 +22,7 @@ ) # use the local instance of the Epidata API -BASE_URL = 'http://delphi_web_epidata/epidata/api.php' +BASE_URL = 'http://delphi_web_epidata/epidata' class CovidcastMetaCacheTests(unittest.TestCase): @@ -82,8 +82,8 @@ def tearDown(self): @staticmethod def _make_request(): - params = {'endpoint': 'covidcast_meta', 'cached': 'true'} - response = requests.get(Epidata.BASE_URL, params=params, auth=Epidata.auth) + params = {'cached': 'true'} + response = requests.get(f"{Epidata.BASE_URL}/covidcast_meta", params=params, auth=Epidata.auth) response.raise_for_status() return response.json() diff --git a/integrations/acquisition/covidcast/test_csv_uploading.py b/integrations/acquisition/covidcast/test_csv_uploading.py index e9c066f6a..dab35f414 100644 --- a/integrations/acquisition/covidcast/test_csv_uploading.py +++ b/integrations/acquisition/covidcast/test_csv_uploading.py @@ -69,7 +69,7 @@ def setUp(self): epidata_cnx.close() # use the local instance of the Epidata API - Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php' + Epidata.BASE_URL = 'http://delphi_web_epidata/epidata' Epidata.auth = ('epidata', 'key') def tearDown(self): @@ -132,7 +132,7 @@ def test_uploading(self): main(args) response = Epidata.covidcast('src-name', signal_name, 'day', 'state', 20200419, '*') - expected_values = pd.concat([values, pd.DataFrame({ "time_value": [20200419] * 3, "signal": [signal_name] * 3, "direction": [None] * 3})], axis=1).rename(columns=uploader_column_rename).to_dict(orient="records") + expected_values = pd.concat([values, pd.DataFrame({ "geo_type": "state", "source": "src-name", "time_type": "day", "time_value": [20200419] * 3, "signal": [signal_name] * 3, "direction": [None] * 3})], axis=1).rename(columns=uploader_column_rename).to_dict(orient="records") expected_response = {'result': 1, 'epidata': self.apply_lag(expected_values), 'message': 'success'} self.assertEqual(response, expected_response) @@ -161,6 +161,9 @@ def test_uploading(self): response = Epidata.covidcast('src-name', signal_name, 'day', 'state', 20200419, '*') expected_values = pd.concat([values, pd.DataFrame({ + "geo_type": "state", + "source": "src-name", + "time_type": "day", "time_value": [20200419] * 3, "signal": [signal_name] * 3, "direction": [None] * 3, @@ -194,7 +197,7 @@ def test_uploading(self): main(args) response = Epidata.covidcast('src-name', signal_name, 'day', 'state', 20200419, '*') - expected_response = {'result': -2, 'message': 'no results'} + expected_response = {'epidata': [], 'result': -2, 'message': 'no results'} self.assertEqual(response, expected_response) self.verify_timestamps_and_defaults() @@ -220,6 +223,9 @@ def test_uploading(self): response = Epidata.covidcast('src-name', signal_name, 'day', 'state', 20200419, '*') expected_values_df = pd.concat([values, pd.DataFrame({ + "geo_type": "state", + "source": "src-name", + "time_type": "day", "time_value": [20200419], "signal": [signal_name], "direction": [None]})], axis=1).rename(columns=uploader_column_rename) @@ -253,6 +259,9 @@ def test_uploading(self): response = Epidata.covidcast('src-name', signal_name, 'day', 'state', 20200419, '*') expected_values = pd.concat([values, pd.DataFrame({ + "geo_type": "state", + "source": "src-name", + "time_type": "day", "time_value": [20200419], "signal": [signal_name], "direction": [None] @@ -283,7 +292,7 @@ def test_uploading(self): main(args) response = Epidata.covidcast('src-name', signal_name, 'day', 'state', 20200419, '*') - expected_response = {'result': -2, 'message': 'no results'} + expected_response = {'epidata': [], 'result': -2, 'message': 'no results'} self.assertEqual(response, expected_response) self.verify_timestamps_and_defaults() diff --git a/integrations/acquisition/covidcast_nowcast/test_csv_uploading.py b/integrations/acquisition/covidcast_nowcast/test_csv_uploading.py index 1299c6144..bf1a0f9a0 100644 --- a/integrations/acquisition/covidcast_nowcast/test_csv_uploading.py +++ b/integrations/acquisition/covidcast_nowcast/test_csv_uploading.py @@ -55,7 +55,7 @@ def setUp(self): secrets.db.epi = ('user', 'pass') # use the local instance of the Epidata API - Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php' + Epidata.BASE_URL = 'http://delphi_web_epidata/epidata' Epidata.auth = ('epidata', 'key') def tearDown(self): diff --git a/integrations/client/test_delphi_epidata.py b/integrations/client/test_delphi_epidata.py index 4ef1fa6a3..7497890c2 100644 --- a/integrations/client/test_delphi_epidata.py +++ b/integrations/client/test_delphi_epidata.py @@ -24,9 +24,9 @@ def fake_epidata_endpoint(func): """This can be used as a decorator to enable a bogus Epidata endpoint to return 404 responses.""" def wrapper(*args): - Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/fake_api.php' + Epidata.BASE_URL = 'http://delphi_web_epidata/fake_epidata' func(*args) - Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php' + Epidata.BASE_URL = 'http://delphi_web_epidata/epidata' return wrapper class DelphiEpidataPythonClientTests(CovidcastBase): @@ -39,7 +39,7 @@ def localSetUp(self): self._db._cursor.execute('update covidcast_meta_cache set timestamp = 0, epidata = "[]"') # use the local instance of the Epidata API - Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php' + Epidata.BASE_URL = 'http://delphi_web_epidata/epidata' Epidata.auth = ('epidata', 'key') # use the local instance of the epidata database @@ -65,8 +65,8 @@ def test_covidcast(self): ) expected = [ - row_latest_issue.as_api_compatibility_row_dict(), - rows[-1].as_api_compatibility_row_dict() + row_latest_issue.as_api_row_dict(), + rows[-1].as_api_row_dict() ] self.assertEqual(response['epidata'], expected) @@ -85,10 +85,10 @@ def test_covidcast(self): expected = [{ rows[0].signal: [ - row_latest_issue.as_api_compatibility_row_dict(ignore_fields=['signal']), + row_latest_issue.as_api_row_dict(ignore_fields=['signal']), ], rows[-1].signal: [ - rows[-1].as_api_compatibility_row_dict(ignore_fields=['signal']), + rows[-1].as_api_row_dict(ignore_fields=['signal']), ], }] @@ -105,7 +105,7 @@ def test_covidcast(self): **self.params_from_row(rows[0]) ) - expected = [row_latest_issue.as_api_compatibility_row_dict()] + expected = [row_latest_issue.as_api_row_dict()] # check result self.assertEqual(response_1, { @@ -120,7 +120,7 @@ def test_covidcast(self): **self.params_from_row(rows[0], as_of=rows[1].issue) ) - expected = [rows[1].as_api_compatibility_row_dict()] + expected = [rows[1].as_api_row_dict()] # check result self.maxDiff=None @@ -137,8 +137,8 @@ def test_covidcast(self): ) expected = [ - rows[0].as_api_compatibility_row_dict(), - rows[1].as_api_compatibility_row_dict() + rows[0].as_api_row_dict(), + rows[1].as_api_row_dict() ] # check result @@ -154,7 +154,7 @@ def test_covidcast(self): **self.params_from_row(rows[0], lag=2) ) - expected = [row_latest_issue.as_api_compatibility_row_dict()] + expected = [row_latest_issue.as_api_row_dict()] # check result self.assertDictEqual(response_3, { @@ -170,7 +170,7 @@ def test_covidcast(self): ) # check result - self.assertEqual(response_1, {'message': 'no results', 'result': -2}) + self.assertEqual(response_1, {'epidata': [], 'message': 'no results', 'result': -2}) @patch('requests.post') @patch('requests.get') @@ -196,7 +196,7 @@ def test_retry_request(self, get): mock_response = MagicMock() mock_response.status_code = 200 get.side_effect = [JSONDecodeError('Expecting value', "", 0), mock_response] - response = Epidata._request(None) + response = Epidata._request(None, "") self.assertEqual(get.call_count, 2) self.assertEqual(response, mock_response.json()) @@ -207,7 +207,7 @@ def test_retry_request(self, get): get.side_effect = [JSONDecodeError('Expecting value', "", 0), JSONDecodeError('Expecting value', "", 0), mock_response] - response = Epidata._request(None) + response = Epidata._request(None, "") self.assertEqual(get.call_count, 2) # 2 from previous test + 2 from this one self.assertEqual(response, {'result': 0, 'message': 'error: Expecting value: line 1 column 1 (char 0)'} @@ -228,7 +228,7 @@ def test_geo_value(self): self._insert_rows(rows) counties = [ - rows[i].as_api_compatibility_row_dict() for i in range(N) + rows[i].as_api_row_dict() for i in range(N) ] def fetch(geo): @@ -336,9 +336,9 @@ def test_async_epidata(self): self._insert_rows(rows) test_output = Epidata.async_epidata([ - self.params_from_row(rows[0], source='covidcast'), - self.params_from_row(rows[1], source='covidcast') - ]*12, batch_size=10) + self.params_from_row(rows[0]), + self.params_from_row(rows[1]) + ]*12, 'covidcast', batch_size=10) responses = [i[0] for i in test_output] # check response is same as standard covidcast call, using 24 calls to test batch sizing self.assertEqual( @@ -354,7 +354,6 @@ def test_async_epidata_fail(self): with pytest.raises(ClientResponseError, match="404, message='NOT FOUND'"): Epidata.async_epidata([ { - 'source': 'covidcast', 'data_source': 'src', 'signals': 'sig', 'time_type': 'day', @@ -362,4 +361,4 @@ def test_async_epidata_fail(self): 'geo_value': '11111', 'time_values': '20200414' } - ]) + ], 'covidcast') diff --git a/integrations/client/test_nowcast.py b/integrations/client/test_nowcast.py index f5124e021..84fc0e080 100644 --- a/integrations/client/test_nowcast.py +++ b/integrations/client/test_nowcast.py @@ -39,7 +39,7 @@ def setUp(self): self.cur = cnx.cursor() # use the local instance of the Epidata API - Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php' + Epidata.BASE_URL = 'http://delphi_web_epidata/epidata' Epidata.auth = ('epidata', 'key') # use the local instance of the epidata database @@ -133,4 +133,4 @@ def test_covidcast_nowcast(self): response = Epidata.covidcast_nowcast( 'src', 'sig1', 'sensor', 'day', 'county', 22222222, '01001') - self.assertEqual(response, {'result': -2, 'message': 'no results'}) + self.assertEqual(response, {'epidata': [], 'result': -2, 'message': 'no results'}) diff --git a/integrations/server/test_covid_hosp.py b/integrations/server/test_covid_hosp.py index 37aa77363..100d961c4 100644 --- a/integrations/server/test_covid_hosp.py +++ b/integrations/server/test_covid_hosp.py @@ -16,7 +16,7 @@ def setUp(self): """Perform per-test setup.""" # use the local instance of the Epidata API - Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php' + Epidata.BASE_URL = 'http://delphi_web_epidata/epidata' Epidata.auth = ('epidata', 'key') # use the local instance of the epidata database diff --git a/integrations/server/test_covidcast.py b/integrations/server/test_covidcast.py index 73787d664..935d7badb 100644 --- a/integrations/server/test_covidcast.py +++ b/integrations/server/test_covidcast.py @@ -23,7 +23,7 @@ def localSetUp(self): def request_based_on_row(self, row: CovidcastTestRow, **kwargs): params = self.params_from_row(row, endpoint='covidcast', **kwargs) # use the local instance of the Epidata API - Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php' + Epidata.BASE_URL = 'http://delphi_web_epidata/epidata' Epidata.auth = ('epidata', 'key') response = Epidata.covidcast(**params) @@ -90,7 +90,7 @@ def test_round_trip(self): # make the request response = self.request_based_on_row(row) - expected = [row.as_api_compatibility_row_dict()] + expected = [row.as_api_row_dict()] self.assertEqual(response, { 'result': 1, @@ -158,11 +158,12 @@ def test_csv_format(self): # This is a hardcoded mess because of api.php. column_order = [ - "geo_value", "signal", "time_value", "direction", "issue", "lag", "missing_value", + "geo_value", "signal", "source", "geo_type", "time_type", + "time_value", "direction", "issue", "lag", "missing_value", "missing_stderr", "missing_sample_size", "value", "stderr", "sample_size" ] expected = ( - row.as_api_compatibility_row_df() + row.as_api_row_df() .assign(direction = None) .to_csv(columns=column_order, index=False) ) @@ -179,7 +180,7 @@ def test_raw_json_format(self): # make the request response = self.request_based_on_row(row, **{'format':'json'}) - expected = [row.as_api_compatibility_row_dict()] + expected = [row.as_api_row_dict()] # assert that the right data came back self.assertEqual(response, expected) @@ -191,13 +192,13 @@ def test_fields(self): row = self._insert_placeholder_set_one() # limit fields - response = self.request_based_on_row(row, **{"fields":"time_value,geo_value"}) + response = self.request_based_on_row(row, **{"fields":"time_value,geo_value,geo_type,source,time_type"}) - expected = row.as_api_compatibility_row_dict() + expected = row.as_api_row_dict() expected_all = { 'result': 1, 'epidata': [{ - k: expected[k] for k in ['time_value', 'geo_value'] + k: expected[k] for k in ['time_value', 'geo_value', 'geo_type', 'source', 'time_type'] }], 'message': 'success', } @@ -206,7 +207,7 @@ def test_fields(self): self.assertEqual(response, expected_all) # limit using invalid fields - response = self.request_based_on_row(row, fields='time_value,geo_value,doesnt_exist') + response = self.request_based_on_row(row, fields='time_value,geo_value,,geo_type,source,time_type,doesnt_exist') # assert that the right data came back (only valid fields) self.assertEqual(response, expected_all) @@ -226,7 +227,7 @@ def test_location_wildcard(self): # insert placeholder data rows = self._insert_placeholder_set_two() - expected = [row.as_api_compatibility_row_dict() for row in rows[:3]] + expected = [row.as_api_row_dict() for row in rows[:3]] # make the request response = self.request_based_on_row(rows[0], geo_value="*") @@ -243,7 +244,7 @@ def test_time_values_wildcard(self): # insert placeholder data rows = self._insert_placeholder_set_three() - expected = [row.as_api_compatibility_row_dict() for row in rows[:3]] + expected = [row.as_api_row_dict() for row in rows[:3]] # make the request response = self.request_based_on_row(rows[0], time_values="*") @@ -261,7 +262,7 @@ def test_issues_wildcard(self): # insert placeholder data rows = self._insert_placeholder_set_five() - expected = [row.as_api_compatibility_row_dict() for row in rows[:3]] + expected = [row.as_api_row_dict() for row in rows[:3]] # make the request response = self.request_based_on_row(rows[0], issues="*") @@ -279,7 +280,7 @@ def test_signal_wildcard(self): # insert placeholder data rows = self._insert_placeholder_set_four() - expected_signals = [row.as_api_compatibility_row_dict() for row in rows[:3]] + expected_signals = [row.as_api_row_dict() for row in rows[:3]] # make the request response = self.request_based_on_row(rows[0], signals="*") @@ -297,7 +298,7 @@ def test_geo_value(self): # insert placeholder data rows = self._insert_placeholder_set_two() - expected = [row.as_api_compatibility_row_dict() for row in rows[:3]] + expected = [row.as_api_row_dict() for row in rows[:3]] def fetch(geo_value): # make the request @@ -335,7 +336,7 @@ def test_location_timeline(self): # insert placeholder data rows = self._insert_placeholder_set_three() - expected_timeseries = [row.as_api_compatibility_row_dict() for row in rows[:3]] + expected_timeseries = [row.as_api_row_dict() for row in rows[:3]] # make the request response = self.request_based_on_row(rows[0], time_values='20000101-20000105') @@ -372,7 +373,7 @@ def test_nullable_columns(self): # make the request response = self.request_based_on_row(row) - expected = row.as_api_compatibility_row_dict() + expected = row.as_api_row_dict() # assert that the right data came back self.assertEqual(response, { @@ -393,7 +394,7 @@ def test_temporal_partitioning(self): # make the request response = self.request_based_on_row(rows[1], time_values="*") - expected = [rows[1].as_api_compatibility_row_dict()] + expected = [rows[1].as_api_row_dict()] # assert that the right data came back self.assertEqual(response, { diff --git a/integrations/server/test_covidcast_meta.py b/integrations/server/test_covidcast_meta.py index d03317c98..857422a41 100644 --- a/integrations/server/test_covidcast_meta.py +++ b/integrations/server/test_covidcast_meta.py @@ -14,7 +14,7 @@ import delphi.operations.secrets as secrets # use the local instance of the Epidata API -BASE_URL = 'http://delphi_web_epidata/epidata/api.php' +BASE_URL = 'http://delphi_web_epidata/epidata' AUTH = ('epidata', 'key') @@ -152,8 +152,7 @@ def _get_id(self): @staticmethod def _fetch(auth=AUTH, **kwargs): params = kwargs.copy() - params['endpoint'] = 'covidcast_meta' - response = requests.get(BASE_URL, params=params, auth=auth) + response = requests.get(f"{BASE_URL}/covidcast_meta", params=params, auth=auth) response.raise_for_status() return response.json() diff --git a/integrations/server/test_covidcast_nowcast.py b/integrations/server/test_covidcast_nowcast.py index 889d962dd..32445afdf 100644 --- a/integrations/server/test_covidcast_nowcast.py +++ b/integrations/server/test_covidcast_nowcast.py @@ -9,7 +9,7 @@ # use the local instance of the Epidata API -BASE_URL = 'http://delphi_web_epidata/epidata/api.php' +BASE_URL = 'http://delphi_web_epidata/epidata' AUTH = ('epidata', 'key') @@ -43,7 +43,7 @@ def tearDown(self): @staticmethod def _make_request(params: dict): - response = requests.get(BASE_URL, params=params, auth=AUTH) + response = requests.get(f"{BASE_URL}/covidcast_nowcast", params=params, auth=AUTH) response.raise_for_status() return response.json() @@ -59,7 +59,6 @@ def test_query(self): self.cnx.commit() # make the request with specified issue date params={ - 'source': 'covidcast_nowcast', 'data_source': 'src', 'signals': 'sig', 'sensor_names': 'sensor', diff --git a/integrations/server/test_fluview.py b/integrations/server/test_fluview.py index c192da637..48d9585fd 100644 --- a/integrations/server/test_fluview.py +++ b/integrations/server/test_fluview.py @@ -18,7 +18,7 @@ def setUpClass(cls): """Perform one-time setup.""" # use the local instance of the Epidata API - Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php' + Epidata.BASE_URL = 'http://delphi_web_epidata/epidata' Epidata.auth = ('epidata', 'key') def setUp(self): diff --git a/integrations/server/test_fluview_meta.py b/integrations/server/test_fluview_meta.py index 1e2cf73e3..6f81c1859 100644 --- a/integrations/server/test_fluview_meta.py +++ b/integrations/server/test_fluview_meta.py @@ -18,7 +18,7 @@ def setUpClass(cls): """Perform one-time setup.""" # use the local instance of the Epidata API - Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php' + Epidata.BASE_URL = 'http://delphi_web_epidata/epidata' Epidata.auth = ('epidata', 'key') def setUp(self): diff --git a/src/client/delphi_epidata.py b/src/client/delphi_epidata.py index fe8dbe51d..ae618e2b1 100644 --- a/src/client/delphi_epidata.py +++ b/src/client/delphi_epidata.py @@ -44,7 +44,7 @@ class Epidata: """An interface to DELPHI's Epidata API.""" # API base url - BASE_URL = "https://api.delphi.cmu.edu/epidata/api.php" + BASE_URL = "https://api.delphi.cmu.edu/epidata" auth = None client_version = _version @@ -68,17 +68,18 @@ def _list(values): @staticmethod @retry(reraise=True, stop=stop_after_attempt(2)) - def _request_with_retry(params): + def _request_with_retry(params, endpoint): """Make request with a retry if an exception is thrown.""" - req = requests.get(Epidata.BASE_URL, params, auth=Epidata.auth, headers=_HEADERS) + request_url = f"{Epidata.BASE_URL}/{endpoint}" + req = requests.get(request_url, params, auth=Epidata.auth, headers=_HEADERS) if req.status_code == 414: - req = requests.post(Epidata.BASE_URL, params, auth=Epidata.auth, headers=_HEADERS) + req = requests.post(request_url, params, auth=Epidata.auth, headers=_HEADERS) # handle 401 and 429 req.raise_for_status() return req @staticmethod - def _request(params): + def _request(params, endpoint): """Request and parse epidata. We default to GET since it has better caching and logging @@ -86,7 +87,7 @@ def _request(params): long and returns a 414. """ try: - result = Epidata._request_with_retry(params) + result = Epidata._request_with_retry(params, endpoint) except Exception as e: return {"result": 0, "message": "error: " + str(e)} if params is not None and "format" in params and params["format"] == "csv": @@ -125,7 +126,6 @@ def fluview(regions, epiweeks, issues=None, lag=None, auth=None): raise EpidataBadRequestException(ISSUES_LAG_EXCLUSIVE) # Set up request params = { - "endpoint": "fluview", "regions": Epidata._list(regions), "epiweeks": Epidata._list(epiweeks), } @@ -136,18 +136,13 @@ def fluview(regions, epiweeks, issues=None, lag=None, auth=None): if auth is not None: params["auth"] = auth # Make the API call - return Epidata._request(params) + return Epidata._request(params, "fluview") # Fetch FluView metadata @staticmethod def fluview_meta(): """Fetch FluView metadata.""" - # Set up request - params = { - "endpoint": "fluview_meta", - } - # Make the API call - return Epidata._request(params) + return Epidata._request({}, "fluview_meta") # Fetch FluView clinical data @staticmethod @@ -160,7 +155,6 @@ def fluview_clinical(regions, epiweeks, issues=None, lag=None): raise EpidataBadRequestException(REGIONS_EPIWEEKS_REQUIRED) # Set up request params = { - "endpoint": "fluview_clinical", "regions": Epidata._list(regions), "epiweeks": Epidata._list(epiweeks), } @@ -169,7 +163,7 @@ def fluview_clinical(regions, epiweeks, issues=None, lag=None): if lag is not None: params["lag"] = lag # Make the API call - return Epidata._request(params) + return Epidata._request(params, "fluview_clinical") # Fetch FluSurv data @staticmethod @@ -182,7 +176,6 @@ def flusurv(locations, epiweeks, issues=None, lag=None): raise EpidataBadRequestException(REGIONS_EPIWEEKS_REQUIRED) # Set up request params = { - "endpoint": "flusurv", "locations": Epidata._list(locations), "epiweeks": Epidata._list(epiweeks), } @@ -191,7 +184,7 @@ def flusurv(locations, epiweeks, issues=None, lag=None): if lag is not None: params["lag"] = lag # Make the API call - return Epidata._request(params) + return Epidata._request(params, "flusurv") # Fetch PAHO Dengue data @staticmethod @@ -204,7 +197,6 @@ def paho_dengue(regions, epiweeks, issues=None, lag=None): raise EpidataBadRequestException(REGIONS_EPIWEEKS_REQUIRED) # Set up request params = { - "endpoint": "paho_dengue", "regions": Epidata._list(regions), "epiweeks": Epidata._list(epiweeks), } @@ -213,7 +205,7 @@ def paho_dengue(regions, epiweeks, issues=None, lag=None): if lag is not None: params["lag"] = lag # Make the API call - return Epidata._request(params) + return Epidata._request(params, "paho_dengue") # Fetch ECDC ILI data @staticmethod @@ -226,7 +218,6 @@ def ecdc_ili(regions, epiweeks, issues=None, lag=None): raise EpidataBadRequestException(REGIONS_EPIWEEKS_REQUIRED) # Set up request params = { - "endpoint": "ecdc_ili", "regions": Epidata._list(regions), "epiweeks": Epidata._list(epiweeks), } @@ -235,7 +226,7 @@ def ecdc_ili(regions, epiweeks, issues=None, lag=None): if lag is not None: params["lag"] = lag # Make the API call - return Epidata._request(params) + return Epidata._request(params, "ecdc_ili") # Fetch KCDC ILI data @staticmethod @@ -248,7 +239,6 @@ def kcdc_ili(regions, epiweeks, issues=None, lag=None): raise EpidataBadRequestException(REGIONS_EPIWEEKS_REQUIRED) # Set up request params = { - "endpoint": "kcdc_ili", "regions": Epidata._list(regions), "epiweeks": Epidata._list(epiweeks), } @@ -257,7 +247,7 @@ def kcdc_ili(regions, epiweeks, issues=None, lag=None): if lag is not None: params["lag"] = lag # Make the API call - return Epidata._request(params) + return Epidata._request(params, "kcdc_ili") # Fetch Google Flu Trends data @staticmethod @@ -268,12 +258,11 @@ def gft(locations, epiweeks): raise EpidataBadRequestException(LOCATIONS_EPIWEEKS_REQUIRED) # Set up request params = { - "endpoint": "gft", "locations": Epidata._list(locations), "epiweeks": Epidata._list(epiweeks), } # Make the API call - return Epidata._request(params) + return Epidata._request(params, "gft") # Fetch Google Health Trends data @staticmethod @@ -286,14 +275,13 @@ def ght(auth, locations, epiweeks, query): ) # Set up request params = { - "endpoint": "ght", "auth": auth, "locations": Epidata._list(locations), "epiweeks": Epidata._list(epiweeks), "query": query, } # Make the API call - return Epidata._request(params) + return Epidata._request(params, "ght") # Fetch HealthTweets data @staticmethod @@ -306,7 +294,6 @@ def twitter(auth, locations, dates=None, epiweeks=None): raise EpidataBadRequestException("exactly one of `dates` and `epiweeks` is required") # Set up request params = { - "endpoint": "twitter", "auth": auth, "locations": Epidata._list(locations), } @@ -315,7 +302,7 @@ def twitter(auth, locations, dates=None, epiweeks=None): if epiweeks is not None: params["epiweeks"] = Epidata._list(epiweeks) # Make the API call - return Epidata._request(params) + return Epidata._request(params, "twitter") # Fetch Wikipedia access data @staticmethod @@ -328,7 +315,6 @@ def wiki(articles, dates=None, epiweeks=None, hours=None, language="en"): raise EpidataBadRequestException("exactly one of `dates` and `epiweeks` is required") # Set up request params = { - "endpoint": "wiki", "articles": Epidata._list(articles), "language": language, } @@ -339,7 +325,7 @@ def wiki(articles, dates=None, epiweeks=None, hours=None, language="en"): if hours is not None: params["hours"] = Epidata._list(hours) # Make the API call - return Epidata._request(params) + return Epidata._request(params, "wiki") # Fetch CDC page hits @staticmethod @@ -350,13 +336,12 @@ def cdc(auth, epiweeks, locations): raise EpidataBadRequestException("`auth`, `epiweeks`, and `locations` are all required") # Set up request params = { - "endpoint": "cdc", "auth": auth, "epiweeks": Epidata._list(epiweeks), "locations": Epidata._list(locations), } # Make the API call - return Epidata._request(params) + return Epidata._request(params, "cdc") # Fetch Quidel data @staticmethod @@ -367,13 +352,12 @@ def quidel(auth, epiweeks, locations): raise EpidataBadRequestException("`auth`, `epiweeks`, and `locations` are all required") # Set up request params = { - "endpoint": "quidel", "auth": auth, "epiweeks": Epidata._list(epiweeks), "locations": Epidata._list(locations), } # Make the API call - return Epidata._request(params) + return Epidata._request(params, "quidel") # Fetch NoroSTAT data (point data, no min/max) @staticmethod @@ -384,13 +368,12 @@ def norostat(auth, location, epiweeks): raise EpidataBadRequestException("`auth`, `location`, and `epiweeks` are all required") # Set up request params = { - "endpoint": "norostat", "auth": auth, "location": location, "epiweeks": Epidata._list(epiweeks), } # Make the API call - return Epidata._request(params) + return Epidata._request(params, "norostat") # Fetch NoroSTAT metadata @staticmethod @@ -401,11 +384,10 @@ def meta_norostat(auth): raise EpidataBadRequestException("`auth` is required") # Set up request params = { - "endpoint": "meta_norostat", "auth": auth, } # Make the API call - return Epidata._request(params) + return Epidata._request(params, "meta_norostat") # Fetch NIDSS flu data @staticmethod @@ -418,7 +400,6 @@ def nidss_flu(regions, epiweeks, issues=None, lag=None): raise EpidataBadRequestException(REGIONS_EPIWEEKS_REQUIRED) # Set up request params = { - "endpoint": "nidss_flu", "regions": Epidata._list(regions), "epiweeks": Epidata._list(epiweeks), } @@ -427,7 +408,7 @@ def nidss_flu(regions, epiweeks, issues=None, lag=None): if lag is not None: params["lag"] = lag # Make the API call - return Epidata._request(params) + return Epidata._request(params, "nidss_flu") # Fetch NIDSS dengue data @staticmethod @@ -438,12 +419,11 @@ def nidss_dengue(locations, epiweeks): raise EpidataBadRequestException(REGIONS_EPIWEEKS_REQUIRED) # Set up request params = { - "endpoint": "nidss_dengue", "locations": Epidata._list(locations), "epiweeks": Epidata._list(epiweeks), } # Make the API call - return Epidata._request(params) + return Epidata._request(params, "nidss_dengue") # Fetch Delphi's forecast @staticmethod @@ -454,12 +434,11 @@ def delphi(system, epiweek): raise EpidataBadRequestException("`system` and `epiweek` are both required") # Set up request params = { - "endpoint": "delphi", "system": system, "epiweek": epiweek, } # Make the API call - return Epidata._request(params) + return Epidata._request(params, "delphi") # Fetch Delphi's digital surveillance sensors @staticmethod @@ -472,7 +451,6 @@ def sensors(auth, names, locations, epiweeks): ) # Set up request params = { - "endpoint": "sensors", "names": Epidata._list(names), "locations": Epidata._list(locations), "epiweeks": Epidata._list(epiweeks), @@ -480,7 +458,7 @@ def sensors(auth, names, locations, epiweeks): if auth is not None: params["auth"] = auth # Make the API call - return Epidata._request(params) + return Epidata._request(params, "sensors") # Fetch Delphi's dengue digital surveillance sensors @staticmethod @@ -493,14 +471,13 @@ def dengue_sensors(auth, names, locations, epiweeks): ) # Set up request params = { - "endpoint": "dengue_sensors", "auth": auth, "names": Epidata._list(names), "locations": Epidata._list(locations), "epiweeks": Epidata._list(epiweeks), } # Make the API call - return Epidata._request(params) + return Epidata._request(params, "dengue_sensors") # Fetch Delphi's wILI nowcast @staticmethod @@ -511,12 +488,11 @@ def nowcast(locations, epiweeks): raise EpidataBadRequestException(REGIONS_EPIWEEKS_REQUIRED) # Set up request params = { - "endpoint": "nowcast", "locations": Epidata._list(locations), "epiweeks": Epidata._list(epiweeks), } # Make the API call - return Epidata._request(params) + return Epidata._request(params, "nowcast") # Fetch Delphi's dengue nowcast @staticmethod @@ -527,18 +503,17 @@ def dengue_nowcast(locations, epiweeks): raise EpidataBadRequestException(REGIONS_EPIWEEKS_REQUIRED) # Set up request params = { - "endpoint": "dengue_nowcast", "locations": Epidata._list(locations), "epiweeks": Epidata._list(epiweeks), } # Make the API call - return Epidata._request(params) + return Epidata._request(params, "dengue_nowcast") # Fetch API metadata @staticmethod def meta(): """Fetch API metadata.""" - return Epidata._request({"endpoint": "meta"}) + return Epidata._request({}, "meta") # Fetch Delphi's COVID-19 Surveillance Streams @staticmethod @@ -568,7 +543,6 @@ def covidcast( raise EpidataBadRequestException(REGIONS_EPIWEEKS_REQUIRED) # Set up request params = { - "endpoint": "covidcast", "data_source": data_source, "signals": Epidata._list(signals), "time_type": time_type, @@ -594,13 +568,13 @@ def covidcast( params["fields"] = kwargs["fields"] # Make the API call - return Epidata._request(params) + return Epidata._request(params, "covidcast") # Fetch Delphi's COVID-19 Surveillance Streams metadata @staticmethod def covidcast_meta(): """Fetch Delphi's COVID-19 Surveillance Streams metadata""" - return Epidata._request({"endpoint": "covidcast_meta"}) + return Epidata._request({}, "covidcast_meta") # Fetch COVID hospitalization data @staticmethod @@ -611,7 +585,6 @@ def covid_hosp(states, dates, issues=None, as_of=None): raise EpidataBadRequestException("`states` and `dates` are both required") # Set up request params = { - "endpoint": "covid_hosp", "states": Epidata._list(states), "dates": Epidata._list(dates), } @@ -620,7 +593,7 @@ def covid_hosp(states, dates, issues=None, as_of=None): if as_of is not None: params["as_of"] = as_of # Make the API call - return Epidata._request(params) + return Epidata._request(params, "covid_hosp_state_timeseries") # Fetch COVID hospitalization data for specific facilities @staticmethod @@ -633,21 +606,20 @@ def covid_hosp_facility(hospital_pks, collection_weeks, publication_dates=None): ) # Set up request params = { - "source": "covid_hosp_facility", "hospital_pks": Epidata._list(hospital_pks), "collection_weeks": Epidata._list(collection_weeks), } if publication_dates is not None: params["publication_dates"] = Epidata._list(publication_dates) # Make the API call - return Epidata._request(params) + return Epidata._request(params, "covid_hosp_facility") # Lookup COVID hospitalization facility identifiers @staticmethod def covid_hosp_facility_lookup(state=None, ccn=None, city=None, zip=None, fips_code=None): """Lookup COVID hospitalization facility identifiers.""" # Set up request - params = {"source": "covid_hosp_facility_lookup"} + params = {} if state is not None: params["state"] = state elif ccn is not None: @@ -663,7 +635,7 @@ def covid_hosp_facility_lookup(state=None, ccn=None, city=None, zip=None, fips_c "one of `state`, `ccn`, `city`, `zip`, or `fips_code` is required" ) # Make the API call - return Epidata._request(params) + return Epidata._request(params, "covid_hosp_facility_lookup") # Fetch Delphi's COVID-19 Nowcast sensors @staticmethod @@ -693,7 +665,6 @@ def covidcast_nowcast( raise EpidataBadRequestException(REGIONS_EPIWEEKS_REQUIRED) # Set up request params = { - "source": "covidcast_nowcast", "data_source": data_source, "signals": Epidata._list(signals), "sensor_names": Epidata._list(sensor_names), @@ -717,15 +688,17 @@ def covidcast_nowcast( params["format"] = kwargs["format"] # Make the API call - return Epidata._request(params) + return Epidata._request(params, "covidcast_nowcast") @staticmethod - def async_epidata(param_list, batch_size=50): + def async_epidata(param_list, endpoint, batch_size=50): """Make asynchronous Epidata calls for a list of parameters.""" + request_url = f"{Epidata.BASE_URL}/{endpoint}" + async def async_get(params, session): """Helper function to make Epidata GET requests.""" - async with session.get(Epidata.BASE_URL, params=params) as response: + async with session.get(request_url, params=params) as response: response.raise_for_status() return await response.json(), params From 591059c3d015c535e9d8a225273125110d7065b7 Mon Sep 17 00:00:00 2001 From: Rostyslav Zatserkovnyi Date: Fri, 22 Sep 2023 11:54:16 +0300 Subject: [PATCH 23/55] Post review tweaks --- docs/new_endpoint_tutorial.md | 6 +- integrations/client/test_delphi_epidata.py | 12 ++-- integrations/server/test_covidcast.py | 5 +- .../server/test_covidcast_endpoints.py | 20 +++++- src/client/delphi_epidata.py | 66 +++++++++---------- src/maintenance/signal_dash_data_generator.py | 2 +- 6 files changed, 64 insertions(+), 47 deletions(-) diff --git a/docs/new_endpoint_tutorial.md b/docs/new_endpoint_tutorial.md index 6e6094161..abf8a5fa1 100644 --- a/docs/new_endpoint_tutorial.md +++ b/docs/new_endpoint_tutorial.md @@ -123,11 +123,9 @@ Here's what we add to each client: def fluview_meta(): """Fetch FluView metadata.""" # Set up request - params = { - 'endpoint': 'fluview_meta', - } + params = {} # Make the API call - return Epidata._request(params) + return Epidata._request("fluview_meta", params) ``` - [`delphi_epidata.R`](https://github.com/cmu-delphi/delphi-epidata/blob/dev/src/client/delphi_epidata.R) diff --git a/integrations/client/test_delphi_epidata.py b/integrations/client/test_delphi_epidata.py index 7497890c2..99473f0ac 100644 --- a/integrations/client/test_delphi_epidata.py +++ b/integrations/client/test_delphi_epidata.py @@ -196,7 +196,7 @@ def test_retry_request(self, get): mock_response = MagicMock() mock_response.status_code = 200 get.side_effect = [JSONDecodeError('Expecting value', "", 0), mock_response] - response = Epidata._request(None, "") + response = Epidata._request("") self.assertEqual(get.call_count, 2) self.assertEqual(response, mock_response.json()) @@ -207,7 +207,7 @@ def test_retry_request(self, get): get.side_effect = [JSONDecodeError('Expecting value', "", 0), JSONDecodeError('Expecting value', "", 0), mock_response] - response = Epidata._request(None, "") + response = Epidata._request("") self.assertEqual(get.call_count, 2) # 2 from previous test + 2 from this one self.assertEqual(response, {'result': 0, 'message': 'error: Expecting value: line 1 column 1 (char 0)'} @@ -335,10 +335,10 @@ def test_async_epidata(self): ] self._insert_rows(rows) - test_output = Epidata.async_epidata([ + test_output = Epidata.async_epidata('covidcast', [ self.params_from_row(rows[0]), self.params_from_row(rows[1]) - ]*12, 'covidcast', batch_size=10) + ]*12, batch_size=10) responses = [i[0] for i in test_output] # check response is same as standard covidcast call, using 24 calls to test batch sizing self.assertEqual( @@ -352,7 +352,7 @@ def test_async_epidata(self): @fake_epidata_endpoint def test_async_epidata_fail(self): with pytest.raises(ClientResponseError, match="404, message='NOT FOUND'"): - Epidata.async_epidata([ + Epidata.async_epidata('covidcast', [ { 'data_source': 'src', 'signals': 'sig', @@ -361,4 +361,4 @@ def test_async_epidata_fail(self): 'geo_value': '11111', 'time_values': '20200414' } - ], 'covidcast') + ]) diff --git a/integrations/server/test_covidcast.py b/integrations/server/test_covidcast.py index 935d7badb..4963ca9a8 100644 --- a/integrations/server/test_covidcast.py +++ b/integrations/server/test_covidcast.py @@ -156,7 +156,8 @@ def test_csv_format(self): **{'format':'csv'} ) - # This is a hardcoded mess because of api.php. + # This is a hardcoded mess because of the field ordering constructed here: + # https://github.com/cmu-delphi/delphi-epidata/blob/f7da6598a810be8df5374e3a71512c631c3a14f1/src/server/endpoints/covidcast.py#L83-L93 column_order = [ "geo_value", "signal", "source", "geo_type", "time_type", "time_value", "direction", "issue", "lag", "missing_value", @@ -207,7 +208,7 @@ def test_fields(self): self.assertEqual(response, expected_all) # limit using invalid fields - response = self.request_based_on_row(row, fields='time_value,geo_value,,geo_type,source,time_type,doesnt_exist') + response = self.request_based_on_row(row, fields='time_value,geo_value,geo_type,source,time_type,doesnt_exist') # assert that the right data came back (only valid fields) self.assertEqual(response, expected_all) diff --git a/integrations/server/test_covidcast_endpoints.py b/integrations/server/test_covidcast_endpoints.py index 3ba0af039..e79b6f32f 100644 --- a/integrations/server/test_covidcast_endpoints.py +++ b/integrations/server/test_covidcast_endpoints.py @@ -113,7 +113,25 @@ def test_compatibility(self): with self.subTest("simple"): out = self._fetch("/", signal=first.signal_pair(), geo=first.geo_pair(), time="day:*", is_compatibility=True) - self.assertEqual(len(out["epidata"]), len(rows)) + self.assertEqual(out["epidata"], [row.as_api_compatibility_row_dict() for row in rows]) + + def test_compatibility_restricted_source(self): + """Restricted request at the /api.php endpoint.""" + rows = [CovidcastTestRow.make_default_row(time_value=2020_04_01 + i, value=i, source="quidel") for i in range(10)] + first = rows[0] + self._insert_rows(rows) + + with self.subTest("no_roles"): + out = self._fetch("/", signal=first.signal_pair(), geo=first.geo_pair(), time="day:*", is_compatibility=True) + self.assertTrue("epidata" not in out) + + with self.subTest("no_api_key"): + out = self._fetch("/", auth=None, signal=first.signal_pair(), geo=first.geo_pair(), time="day:*", is_compatibility=True) + self.assertTrue("epidata" not in out) + + with self.subTest("quidel_role"): + out = self._fetch("/", auth=("epidata", "quidel_key"), signal=first.signal_pair(), geo=first.geo_pair(), time="day:*", is_compatibility=True) + self.assertEqual(out["epidata"], [row.as_api_compatibility_row_dict() for row in rows]) def test_trend(self): """Request a signal from the /trend endpoint.""" diff --git a/src/client/delphi_epidata.py b/src/client/delphi_epidata.py index ae618e2b1..e773bdc50 100644 --- a/src/client/delphi_epidata.py +++ b/src/client/delphi_epidata.py @@ -68,7 +68,7 @@ def _list(values): @staticmethod @retry(reraise=True, stop=stop_after_attempt(2)) - def _request_with_retry(params, endpoint): + def _request_with_retry(endpoint, params={}): """Make request with a retry if an exception is thrown.""" request_url = f"{Epidata.BASE_URL}/{endpoint}" req = requests.get(request_url, params, auth=Epidata.auth, headers=_HEADERS) @@ -79,7 +79,7 @@ def _request_with_retry(params, endpoint): return req @staticmethod - def _request(params, endpoint): + def _request(endpoint, params={}): """Request and parse epidata. We default to GET since it has better caching and logging @@ -87,7 +87,7 @@ def _request(params, endpoint): long and returns a 414. """ try: - result = Epidata._request_with_retry(params, endpoint) + result = Epidata._request_with_retry(endpoint, params) except Exception as e: return {"result": 0, "message": "error: " + str(e)} if params is not None and "format" in params and params["format"] == "csv": @@ -136,13 +136,13 @@ def fluview(regions, epiweeks, issues=None, lag=None, auth=None): if auth is not None: params["auth"] = auth # Make the API call - return Epidata._request(params, "fluview") + return Epidata._request("fluview", params) # Fetch FluView metadata @staticmethod def fluview_meta(): """Fetch FluView metadata.""" - return Epidata._request({}, "fluview_meta") + return Epidata._request("fluview_meta") # Fetch FluView clinical data @staticmethod @@ -163,7 +163,7 @@ def fluview_clinical(regions, epiweeks, issues=None, lag=None): if lag is not None: params["lag"] = lag # Make the API call - return Epidata._request(params, "fluview_clinical") + return Epidata._request("fluview_clinical", params) # Fetch FluSurv data @staticmethod @@ -184,7 +184,7 @@ def flusurv(locations, epiweeks, issues=None, lag=None): if lag is not None: params["lag"] = lag # Make the API call - return Epidata._request(params, "flusurv") + return Epidata._request("flusurv", params) # Fetch PAHO Dengue data @staticmethod @@ -205,7 +205,7 @@ def paho_dengue(regions, epiweeks, issues=None, lag=None): if lag is not None: params["lag"] = lag # Make the API call - return Epidata._request(params, "paho_dengue") + return Epidata._request("paho_dengue", params) # Fetch ECDC ILI data @staticmethod @@ -226,7 +226,7 @@ def ecdc_ili(regions, epiweeks, issues=None, lag=None): if lag is not None: params["lag"] = lag # Make the API call - return Epidata._request(params, "ecdc_ili") + return Epidata._request("ecdc_ili", params) # Fetch KCDC ILI data @staticmethod @@ -247,7 +247,7 @@ def kcdc_ili(regions, epiweeks, issues=None, lag=None): if lag is not None: params["lag"] = lag # Make the API call - return Epidata._request(params, "kcdc_ili") + return Epidata._request("kcdc_ili", params) # Fetch Google Flu Trends data @staticmethod @@ -262,7 +262,7 @@ def gft(locations, epiweeks): "epiweeks": Epidata._list(epiweeks), } # Make the API call - return Epidata._request(params, "gft") + return Epidata._request("gft", params) # Fetch Google Health Trends data @staticmethod @@ -281,7 +281,7 @@ def ght(auth, locations, epiweeks, query): "query": query, } # Make the API call - return Epidata._request(params, "ght") + return Epidata._request("ght", params) # Fetch HealthTweets data @staticmethod @@ -302,7 +302,7 @@ def twitter(auth, locations, dates=None, epiweeks=None): if epiweeks is not None: params["epiweeks"] = Epidata._list(epiweeks) # Make the API call - return Epidata._request(params, "twitter") + return Epidata._request("twitter", params) # Fetch Wikipedia access data @staticmethod @@ -325,7 +325,7 @@ def wiki(articles, dates=None, epiweeks=None, hours=None, language="en"): if hours is not None: params["hours"] = Epidata._list(hours) # Make the API call - return Epidata._request(params, "wiki") + return Epidata._request("wiki", params) # Fetch CDC page hits @staticmethod @@ -341,7 +341,7 @@ def cdc(auth, epiweeks, locations): "locations": Epidata._list(locations), } # Make the API call - return Epidata._request(params, "cdc") + return Epidata._request("cdc", params) # Fetch Quidel data @staticmethod @@ -357,7 +357,7 @@ def quidel(auth, epiweeks, locations): "locations": Epidata._list(locations), } # Make the API call - return Epidata._request(params, "quidel") + return Epidata._request("quidel", params) # Fetch NoroSTAT data (point data, no min/max) @staticmethod @@ -373,7 +373,7 @@ def norostat(auth, location, epiweeks): "epiweeks": Epidata._list(epiweeks), } # Make the API call - return Epidata._request(params, "norostat") + return Epidata._request("norostat", params) # Fetch NoroSTAT metadata @staticmethod @@ -387,7 +387,7 @@ def meta_norostat(auth): "auth": auth, } # Make the API call - return Epidata._request(params, "meta_norostat") + return Epidata._request("meta_norostat", params) # Fetch NIDSS flu data @staticmethod @@ -408,7 +408,7 @@ def nidss_flu(regions, epiweeks, issues=None, lag=None): if lag is not None: params["lag"] = lag # Make the API call - return Epidata._request(params, "nidss_flu") + return Epidata._request("nidss_flu", params) # Fetch NIDSS dengue data @staticmethod @@ -423,7 +423,7 @@ def nidss_dengue(locations, epiweeks): "epiweeks": Epidata._list(epiweeks), } # Make the API call - return Epidata._request(params, "nidss_dengue") + return Epidata._request("nidss_dengue", params) # Fetch Delphi's forecast @staticmethod @@ -438,7 +438,7 @@ def delphi(system, epiweek): "epiweek": epiweek, } # Make the API call - return Epidata._request(params, "delphi") + return Epidata._request("delphi", params) # Fetch Delphi's digital surveillance sensors @staticmethod @@ -458,7 +458,7 @@ def sensors(auth, names, locations, epiweeks): if auth is not None: params["auth"] = auth # Make the API call - return Epidata._request(params, "sensors") + return Epidata._request("sensors", params) # Fetch Delphi's dengue digital surveillance sensors @staticmethod @@ -477,7 +477,7 @@ def dengue_sensors(auth, names, locations, epiweeks): "epiweeks": Epidata._list(epiweeks), } # Make the API call - return Epidata._request(params, "dengue_sensors") + return Epidata._request("dengue_sensors", params) # Fetch Delphi's wILI nowcast @staticmethod @@ -492,7 +492,7 @@ def nowcast(locations, epiweeks): "epiweeks": Epidata._list(epiweeks), } # Make the API call - return Epidata._request(params, "nowcast") + return Epidata._request("nowcast", params) # Fetch Delphi's dengue nowcast @staticmethod @@ -507,13 +507,13 @@ def dengue_nowcast(locations, epiweeks): "epiweeks": Epidata._list(epiweeks), } # Make the API call - return Epidata._request(params, "dengue_nowcast") + return Epidata._request("dengue_nowcast", params) # Fetch API metadata @staticmethod def meta(): """Fetch API metadata.""" - return Epidata._request({}, "meta") + return Epidata._request("meta") # Fetch Delphi's COVID-19 Surveillance Streams @staticmethod @@ -568,13 +568,13 @@ def covidcast( params["fields"] = kwargs["fields"] # Make the API call - return Epidata._request(params, "covidcast") + return Epidata._request("covidcast", params) # Fetch Delphi's COVID-19 Surveillance Streams metadata @staticmethod def covidcast_meta(): """Fetch Delphi's COVID-19 Surveillance Streams metadata""" - return Epidata._request({}, "covidcast_meta") + return Epidata._request("covidcast_meta") # Fetch COVID hospitalization data @staticmethod @@ -593,7 +593,7 @@ def covid_hosp(states, dates, issues=None, as_of=None): if as_of is not None: params["as_of"] = as_of # Make the API call - return Epidata._request(params, "covid_hosp_state_timeseries") + return Epidata._request("covid_hosp_state_timeseries", params) # Fetch COVID hospitalization data for specific facilities @staticmethod @@ -612,7 +612,7 @@ def covid_hosp_facility(hospital_pks, collection_weeks, publication_dates=None): if publication_dates is not None: params["publication_dates"] = Epidata._list(publication_dates) # Make the API call - return Epidata._request(params, "covid_hosp_facility") + return Epidata._request("covid_hosp_facility", params) # Lookup COVID hospitalization facility identifiers @staticmethod @@ -635,7 +635,7 @@ def covid_hosp_facility_lookup(state=None, ccn=None, city=None, zip=None, fips_c "one of `state`, `ccn`, `city`, `zip`, or `fips_code` is required" ) # Make the API call - return Epidata._request(params, "covid_hosp_facility_lookup") + return Epidata._request("covid_hosp_facility_lookup", params) # Fetch Delphi's COVID-19 Nowcast sensors @staticmethod @@ -688,10 +688,10 @@ def covidcast_nowcast( params["format"] = kwargs["format"] # Make the API call - return Epidata._request(params, "covidcast_nowcast") + return Epidata._request("covidcast_nowcast", params) @staticmethod - def async_epidata(param_list, endpoint, batch_size=50): + def async_epidata(endpoint, param_list, batch_size=50): """Make asynchronous Epidata calls for a list of parameters.""" request_url = f"{Epidata.BASE_URL}/{endpoint}" diff --git a/src/maintenance/signal_dash_data_generator.py b/src/maintenance/signal_dash_data_generator.py index 6eea06579..b7f1048f5 100644 --- a/src/maintenance/signal_dash_data_generator.py +++ b/src/maintenance/signal_dash_data_generator.py @@ -19,7 +19,7 @@ LOOKBACK_DAYS_FOR_COVERAGE = 56 -BASE_COVIDCAST = covidcast.covidcast.Epidata.BASE_URL[:-len("api.php")] + "covidcast" +BASE_COVIDCAST = covidcast.covidcast.Epidata.BASE_URL + "/covidcast" COVERAGE_URL = f"{BASE_COVIDCAST}/coverage?format=csv&signal={{source}}:{{signal}}&days={LOOKBACK_DAYS_FOR_COVERAGE}" @dataclass From 66257b2b57e1c70095f58006aa3f4c74319c6c93 Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Mon, 25 Sep 2023 19:55:08 +0300 Subject: [PATCH 24/55] Adjusted tests to use new API url --- integrations/server/test_api_keys.py | 53 +++++++------------- integrations/server/test_signal_dashboard.py | 11 +--- src/common/integration_test_base_class.py | 2 +- 3 files changed, 22 insertions(+), 44 deletions(-) diff --git a/integrations/server/test_api_keys.py b/integrations/server/test_api_keys.py index 418e265ac..3ba67e324 100644 --- a/integrations/server/test_api_keys.py +++ b/integrations/server/test_api_keys.py @@ -11,24 +11,22 @@ class APIKeysTets(DelphiTestBase): def localSetUp(self): self.role_name = "cdc" - def _make_request(self, url: str = None, params: dict = {}, auth: tuple = None): + def _make_request(self, url: str = None, endpoint: str = None, params: dict = {}, auth: tuple = None): if not url: url = self.epidata_client.BASE_URL - response = requests.get(url, params=params, auth=auth) + response = requests.get(f"{url}/{endpoint}", params=params, auth=auth) return response def test_public_route(self): """Test public route""" - public_route = "http://delphi_web_epidata/epidata/version" status_codes = set() for _ in range(10): - status_codes.add(self._make_request(public_route).status_code) + status_codes.add(self._make_request(endpoint="version").status_code) self.assertEqual(status_codes, {200}) def test_no_multiples_data_source(self): """Test requests with no multiples and with provided `data_source` and `signal` as a separate query params.""" params = { - "source": "covidcast", "data_source": "fb-survey", "signal": "smoothed_wcli", "time_type": "day", @@ -38,13 +36,12 @@ def test_no_multiples_data_source(self): } status_codes = set() for _ in range(10): - status_codes.add(self._make_request(params=params).status_code) + status_codes.add(self._make_request(params=params, endpoint="covidcast").status_code) self.assertEqual(status_codes, {200}) def test_no_multiples_source_signal(self): """Test requests with colon-delimited source-signal param presentation.""" params = { - "source": "covidcast", "signal": "fb-survey:smoothed_wcli", "time_type": "day", "geo_type": "state", @@ -53,13 +50,12 @@ def test_no_multiples_source_signal(self): } status_codes = set() for _ in range(10): - status_codes.add(self._make_request(params=params).status_code) + status_codes.add(self._make_request(params=params, endpoint="covidcast").status_code) self.assertEqual(status_codes, {200}) def test_multiples_allowed_signal_two_multiples(self): """Test requests with 2 multiples and allowed dashboard signal""" params = { - "source": "covidcast", "signal": "fb-survey:smoothed_wcli", "time_type": "day", "geo_type": "state", @@ -68,13 +64,12 @@ def test_multiples_allowed_signal_two_multiples(self): } status_codes = set() for _ in range(10): - status_codes.add(self._make_request(params=params).status_code) + status_codes.add(self._make_request(params=params, endpoint="covidcast").status_code) self.assertEqual(status_codes, {200}) def test_multiples_non_allowed_signal(self): """Test requests with 2 multiples and non-allowed dashboard signal""" params = { - "source": "covidcast", "signal": "hospital-admissions:smoothed_adj_covid19_from_claims", "time_type": "day", "geo_type": "state", @@ -83,13 +78,12 @@ def test_multiples_non_allowed_signal(self): } status_codes = set() for _ in range(10): - status_codes.add(self._make_request(params=params).status_code) + status_codes.add(self._make_request(params=params, endpoint="covidcast").status_code) self.assertEqual(status_codes, {200, 429}) def test_multiples_mixed_allowed_signal_two_multiples(self): """Test requests with 2 multiples and mixed-allowed dashboard signal""" params = { - "source": "covidcast", "signal": "fb-survey:smoothed_wcli,hospital-admissions:smoothed_adj_covid19_from_claims", "time_type": "day", "geo_type": "state", @@ -98,13 +92,12 @@ def test_multiples_mixed_allowed_signal_two_multiples(self): } status_codes = set() for _ in range(10): - status_codes.add(self._make_request(params=params).status_code) + status_codes.add(self._make_request(params=params, endpoint="covidcast").status_code) self.assertEqual(status_codes, {200, 429}) def test_multiples_allowed_signal_three_multiples(self): """Test requests with 3 multiples and allowed dashboard signal""" params = { - "source": "covidcast", "signal": "fb-survey:smoothed_wcli,fb-survey:smoothed_wcli", "time_type": "day", "geo_type": "state", @@ -113,13 +106,12 @@ def test_multiples_allowed_signal_three_multiples(self): } status_codes = set() for _ in range(10): - status_codes.add(self._make_request(params=params).status_code) + status_codes.add(self._make_request(params=params, endpoint="covidcast").status_code) self.assertEqual(status_codes, {401}) def test_multiples_mixed_allowed_signal_three_multiples(self): """Test requests with 3 multiples and mixed-allowed dashboard signal""" params = { - "source": "covidcast", "signal": "fb-survey:smoothed_wcli,fb-survey:smoothed_wcli1", "time_type": "day", "geo_type": "state", @@ -128,13 +120,12 @@ def test_multiples_mixed_allowed_signal_three_multiples(self): } status_codes = set() for _ in range(10): - status_codes.add(self._make_request(params=params).status_code) + status_codes.add(self._make_request(params=params, endpoint="covidcast").status_code) self.assertEqual(status_codes, {401}) def test_multiples_mixed_allowed_signal_api_key(self): """Test requests with 3 multiples and mixed-allowed dashboard signal + valid API Key""" params = { - "source": "covidcast", "signal": "fb-survey:smoothed_wcli,fb-survey:smoothed_wcli1", "time_type": "day", "geo_type": "state", @@ -144,14 +135,13 @@ def test_multiples_mixed_allowed_signal_api_key(self): status_codes = set() for _ in range(10): status_codes.add( - self._make_request(params=params, auth=self.epidata_client.auth).status_code + self._make_request(params=params, auth=self.epidata_client.auth, endpoint="covidcast").status_code ) self.assertEqual(status_codes, {200}) def test_multiples_allowed_signal_api_key(self): """Test requests with 3 multiples and allowed dashboard signal + valid API Key""" params = { - "source": "covidcast", "signal": "fb-survey:smoothed_wcli,fb-survey:smoothed_wcli", "time_type": "day", "geo_type": "state", @@ -161,14 +151,13 @@ def test_multiples_allowed_signal_api_key(self): status_codes = set() for _ in range(10): status_codes.add( - self._make_request(params=params, auth=self.epidata_client.auth).status_code + self._make_request(params=params, auth=self.epidata_client.auth, endpoint="covidcast").status_code ) self.assertEqual(status_codes, {200}) def test_no_multiples_allowed_signal_api_key(self): """Test requests with no multiples and allowed dashboard signal + valid API Key""" params = { - "source": "covidcast", "signal": "fb-survey:smoothed_wcli", "time_type": "day", "geo_type": "state", @@ -178,14 +167,13 @@ def test_no_multiples_allowed_signal_api_key(self): status_codes = set() for _ in range(10): status_codes.add( - self._make_request(params=params, auth=self.epidata_client.auth).status_code + self._make_request(params=params, auth=self.epidata_client.auth, endpoint="covidcast").status_code ) self.assertEqual(status_codes, {200}) def test_no_multiples_allowed_signal_bad_api_key(self): """Test requests with no multiples and allowed dashboard signal + bad API Key""" params = { - "source": "covidcast", "signal": "fb-survey:smoothed_wcli", "time_type": "day", "geo_type": "state", @@ -196,54 +184,51 @@ def test_no_multiples_allowed_signal_bad_api_key(self): for _ in range(10): status_codes.add( self._make_request( - params=params, auth=("bad_key", "bad_email") + params=params, auth=("bad_key", "bad_email"), endpoint="covidcast" ).status_code ) self.assertEqual(status_codes, {200}) def test_restricted_endpoint_no_key(self): """Test restricted endpoint with no auth key""" - params = {"source": "cdc", "regions": "1as", "epiweeks": "202020"} + params = {"regions": "1as", "epiweeks": "202020"} status_codes = set() for _ in range(10): - status_codes.add(self._make_request(params=params).status_code) + status_codes.add(self._make_request(params=params, endpoint="cdc").status_code) self.assertEqual(status_codes, {401}) def test_restricted_endpoint_invalid_key(self): """Test restricted endpoint with invalid auth key""" params = { - "source": "cdc", "regions": "1as", "epiweeks": "202020", "auth": "invalid_key", } status_codes = set() for _ in range(10): - status_codes.add(self._make_request(params=params).status_code) + status_codes.add(self._make_request(params=params, endpoint="cdc").status_code) self.assertEqual(status_codes, {401}) def test_restricted_endpoint_no_roles_key(self): """Test restricted endpoint with no roles key""" params = { - "source": "cdc", "regions": "1as", "epiweeks": "202020", "auth": "key", } status_codes = set() for _ in range(10): - status_codes.add(self._make_request(params=params).status_code) + status_codes.add(self._make_request(params=params, endpoint="cdc").status_code) self.assertEqual(status_codes, {401}) def test_restricted_endpoint_valid_roles_key(self): """Test restricted endpoint with valid auth key with required role""" params = { - "source": "cdc", "regions": "1as", "epiweeks": "202020", "auth": "cdc_key", } status_codes = set() for _ in range(10): - status_codes.add(self._make_request(params=params).status_code) + status_codes.add(self._make_request(params=params, endpoint="cdc").status_code) self.assertEqual(status_codes, {200}) diff --git a/integrations/server/test_signal_dashboard.py b/integrations/server/test_signal_dashboard.py index e99a464fd..4b122f3d3 100644 --- a/integrations/server/test_signal_dashboard.py +++ b/integrations/server/test_signal_dashboard.py @@ -24,11 +24,7 @@ def setUp(self) -> None: def test_signal_dashboard_coverage(self): """Basic integration test for signal_dashboard_coverage endpoint""" - - params = { - "endpoint": "signal_dashboard_coverage", - } - response = self.epidata_client._request(params=params) + response = self.epidata_client._request(endpoint="signal_dashboard_coverage", params={}) self.assertEqual( response, { @@ -41,10 +37,7 @@ def test_signal_dashboard_coverage(self): def test_signal_dashboard_status(self): """Basic integration test for signal_dashboard_status endpoint""" - params = { - "endpoint": "signal_dashboard_status", - } - response = self.epidata_client._request(params=params) + response = self.epidata_client._request(endpoint="signal_dashboard_status", params={}) self.assertEqual( response, { diff --git a/src/common/integration_test_base_class.py b/src/common/integration_test_base_class.py index 387c0f617..47c9f68e4 100644 --- a/src/common/integration_test_base_class.py +++ b/src/common/integration_test_base_class.py @@ -19,7 +19,7 @@ def __init__(self, methodName: str = "runTest") -> None: self.create_tables_list = [] self.role_name = None self.epidata_client = Epidata - self.epidata_client.BASE_URL = "http://delphi_web_epidata/epidata/api.php" + self.epidata_client.BASE_URL = "http://delphi_web_epidata/epidata" self.epidata_client.auth = ("epidata", "key") def create_key_with_role(self, cur, role_name: str): From 425929479c81afd85391581c9f74c887e56e97ab Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Tue, 26 Sep 2023 11:59:37 +0300 Subject: [PATCH 25/55] Changed base test class, renamed class name --- integrations/acquisition/covidcast/test_db.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integrations/acquisition/covidcast/test_db.py b/integrations/acquisition/covidcast/test_db.py index 7b9d80770..780b318ee 100644 --- a/integrations/acquisition/covidcast/test_db.py +++ b/integrations/acquisition/covidcast/test_db.py @@ -1,13 +1,13 @@ from delphi_utils import Nans from delphi.epidata.acquisition.covidcast.database import DBLoadStateException -from delphi.epidata.acquisition.covidcast.test_utils import CovidcastBase, CovidcastTestRow +from delphi.epidata.common.covidcast_test_base import CovidcastTestBase, CovidcastTestRow # all the Nans we use here are just one value, so this is a shortcut to it: nmv = Nans.NOT_MISSING.value -class TestTest(CovidcastBase): +class TestDatabase(CovidcastTestBase): def _find_matches_for_row(self, row): # finds (if existing) row from both history and latest views that matches long-key of provided CovidcastTestRow From f1f97444f1f2f180344db1a6944d16b5e5b393ef Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Tue, 26 Sep 2023 12:01:05 +0300 Subject: [PATCH 26/55] Changed base class name. Replaced Epidata usage with local class object. --- integrations/client/test_delphi_epidata.py | 68 ++++++++-------------- 1 file changed, 23 insertions(+), 45 deletions(-) diff --git a/integrations/client/test_delphi_epidata.py b/integrations/client/test_delphi_epidata.py index 99473f0ac..bb4db8ce3 100644 --- a/integrations/client/test_delphi_epidata.py +++ b/integrations/client/test_delphi_epidata.py @@ -10,10 +10,8 @@ from aiohttp.client_exceptions import ClientResponseError # third party -import delphi.operations.secrets as secrets from delphi.epidata.maintenance.covidcast_meta_cache_updater import main as update_covidcast_meta_cache -from delphi.epidata.acquisition.covidcast.test_utils import CovidcastBase, CovidcastTestRow, FIPS, MSA -from delphi.epidata.client.delphi_epidata import Epidata +from delphi.epidata.common.covidcast_test_base import CovidcastTestBase, CovidcastTestRow, FIPS, MSA from delphi_utils import Nans # py3tester coverage target @@ -21,31 +19,10 @@ # all the Nans we use here are just one value, so this is a shortcut to it: nmv = Nans.NOT_MISSING.value -def fake_epidata_endpoint(func): - """This can be used as a decorator to enable a bogus Epidata endpoint to return 404 responses.""" - def wrapper(*args): - Epidata.BASE_URL = 'http://delphi_web_epidata/fake_epidata' - func(*args) - Epidata.BASE_URL = 'http://delphi_web_epidata/epidata' - return wrapper -class DelphiEpidataPythonClientTests(CovidcastBase): - """Tests the Python client.""" - - def localSetUp(self): - """Perform per-test setup.""" - - # reset the `covidcast_meta_cache` table (it should always have one row) - self._db._cursor.execute('update covidcast_meta_cache set timestamp = 0, epidata = "[]"') - - # use the local instance of the Epidata API - Epidata.BASE_URL = 'http://delphi_web_epidata/epidata' - Epidata.auth = ('epidata', 'key') - - # use the local instance of the epidata database - secrets.db.host = 'delphi_database_epidata' - secrets.db.epi = ('user', 'pass') +class DelphiEpidataPythonClientTests(CovidcastTestBase): + """Tests the Python client.""" def test_covidcast(self): """Test that the covidcast endpoint returns expected data.""" @@ -60,7 +37,7 @@ def test_covidcast(self): with self.subTest(name='request two signals'): # fetch data - response = Epidata.covidcast( + response = self.epidata_client.covidcast( **self.params_from_row(rows[0], signals=[rows[0].signal, rows[-1].signal]) ) @@ -79,7 +56,7 @@ def test_covidcast(self): with self.subTest(name='request two signals with tree format'): # fetch data - response = Epidata.covidcast( + response = self.epidata_client.covidcast( **self.params_from_row(rows[0], signals=[rows[0].signal, rows[-1].signal], format='tree') ) @@ -101,7 +78,7 @@ def test_covidcast(self): with self.subTest(name='request most recent'): # fetch data, without specifying issue or lag - response_1 = Epidata.covidcast( + response_1 = self.epidata_client.covidcast( **self.params_from_row(rows[0]) ) @@ -116,7 +93,7 @@ def test_covidcast(self): with self.subTest(name='request as-of a date'): # fetch data, specifying as_of - response_1a = Epidata.covidcast( + response_1a = self.epidata_client.covidcast( **self.params_from_row(rows[0], as_of=rows[1].issue) ) @@ -132,8 +109,8 @@ def test_covidcast(self): with self.subTest(name='request a range of issues'): # fetch data, specifying issue range, not lag - response_2 = Epidata.covidcast( - **self.params_from_row(rows[0], issues=Epidata.range(rows[0].issue, rows[1].issue)) + response_2 = self.epidata_client.covidcast( + **self.params_from_row(rows[0], issues=self.epidata_client.range(rows[0].issue, rows[1].issue)) ) expected = [ @@ -150,7 +127,7 @@ def test_covidcast(self): with self.subTest(name='request at a given lag'): # fetch data, specifying lag, not issue range - response_3 = Epidata.covidcast( + response_3 = self.epidata_client.covidcast( **self.params_from_row(rows[0], lag=2) ) @@ -165,7 +142,7 @@ def test_covidcast(self): with self.subTest(name='long request'): # fetch data, without specifying issue or lag # TODO should also trigger a post but doesn't due to the 414 issue - response_1 = Epidata.covidcast( + response_1 = self.epidata_client.covidcast( **self.params_from_row(rows[0], signals='sig'*1000) ) @@ -177,7 +154,7 @@ def test_covidcast(self): def test_request_method(self, get, post): """Test that a GET request is default and POST is used if a 414 is returned.""" with self.subTest(name='get request'): - Epidata.covidcast('src', 'sig', 'day', 'county', 20200414, '01234') + self.epidata_client.covidcast('src', 'sig', 'day', 'county', 20200414, '01234') get.assert_called_once() post.assert_not_called() with self.subTest(name='post request'): @@ -185,7 +162,7 @@ def test_request_method(self, get, post): mock_response = MagicMock() mock_response.status_code = 414 get.return_value = mock_response - Epidata.covidcast('src', 'sig', 'day', 'county', 20200414, '01234') + self.epidata_client.covidcast('src', 'sig', 'day', 'county', 20200414, '01234') get.assert_called_once() post.assert_called_once() @@ -196,7 +173,7 @@ def test_retry_request(self, get): mock_response = MagicMock() mock_response.status_code = 200 get.side_effect = [JSONDecodeError('Expecting value', "", 0), mock_response] - response = Epidata._request("") + response = self.epidata_client._request("") self.assertEqual(get.call_count, 2) self.assertEqual(response, mock_response.json()) @@ -207,7 +184,7 @@ def test_retry_request(self, get): get.side_effect = [JSONDecodeError('Expecting value', "", 0), JSONDecodeError('Expecting value', "", 0), mock_response] - response = Epidata._request("") + response = self.epidata_client._request("") self.assertEqual(get.call_count, 2) # 2 from previous test + 2 from this one self.assertEqual(response, {'result': 0, 'message': 'error: Expecting value: line 1 column 1 (char 0)'} @@ -232,7 +209,7 @@ def test_geo_value(self): ] def fetch(geo): - return Epidata.covidcast( + return self.epidata_client.covidcast( **self.params_from_row(rows[0], geo_value=geo) ) @@ -289,7 +266,7 @@ def test_covidcast_meta(self): update_covidcast_meta_cache(args=None) # fetch data - response = Epidata.covidcast_meta() + response = self.epidata_client.covidcast_meta() # make sure "last updated" time is recent: updated_time = response['epidata'][0]['last_update'] @@ -335,7 +312,7 @@ def test_async_epidata(self): ] self._insert_rows(rows) - test_output = Epidata.async_epidata('covidcast', [ + test_output = self.epidata_client.async_epidata('covidcast', [ self.params_from_row(rows[0]), self.params_from_row(rows[1]) ]*12, batch_size=10) @@ -344,15 +321,15 @@ def test_async_epidata(self): self.assertEqual( responses, [ - Epidata.covidcast(**self.params_from_row(rows[0])), - Epidata.covidcast(**self.params_from_row(rows[1])), + self.epidata_client.covidcast(**self.params_from_row(rows[0])), + self.epidata_client.covidcast(**self.params_from_row(rows[1])), ]*12 ) - @fake_epidata_endpoint def test_async_epidata_fail(self): + self.epidata_client.BASE_URL = "http://delphi_web_epidata/fake_epidata" with pytest.raises(ClientResponseError, match="404, message='NOT FOUND'"): - Epidata.async_epidata('covidcast', [ + self.epidata_client.async_epidata('covidcast', [ { 'data_source': 'src', 'signals': 'sig', @@ -362,3 +339,4 @@ def test_async_epidata_fail(self): 'time_values': '20200414' } ]) + self.epidata_client.BASE_URL = "http://delphi_web_epidata/epidata" From 853853262b2ed59b610da8477322ce12a5e081d6 Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Tue, 26 Sep 2023 12:03:28 +0300 Subject: [PATCH 27/55] Renamed test_utils.py to covidcast_test_base.py and moved it to src/common. --- .../covidcast_test_base.py} | 207 +++++++++++------- ...test_base_class.py => delphi_test_base.py} | 6 + 2 files changed, 130 insertions(+), 83 deletions(-) rename src/{acquisition/covidcast/test_utils.py => common/covidcast_test_base.py} (60%) rename src/common/{integration_test_base_class.py => delphi_test_base.py} (93%) diff --git a/src/acquisition/covidcast/test_utils.py b/src/common/covidcast_test_base.py similarity index 60% rename from src/acquisition/covidcast/test_utils.py rename to src/common/covidcast_test_base.py index 5a978f8cd..f318b14d9 100644 --- a/src/acquisition/covidcast/test_utils.py +++ b/src/common/covidcast_test_base.py @@ -1,17 +1,15 @@ from dataclasses import fields from datetime import date from typing import Any, Dict, Iterable, List, Optional, Sequence -import unittest import pandas as pd -from redis import Redis - -from delphi_utils import Nans -from delphi.epidata.common.covidcast_row import CovidcastRow from delphi.epidata.acquisition.covidcast.database import Database -from delphi.epidata.server._config import REDIS_HOST, REDIS_PASSWORD +from delphi.epidata.common.covidcast_row import CovidcastRow from delphi.epidata.server.utils.dates import day_to_time_value, time_value_to_day import delphi.operations.secrets as secrets +from delphi_utils import Nans + +from delphi.epidata.common.delphi_test_base import DelphiTestBase # all the Nans we use here are just one value, so this is a shortcut to it: nmv = Nans.NOT_MISSING.value @@ -19,8 +17,14 @@ # TODO replace these real geo_values with fake values, and use patch and mock to mock the return values of # delphi_utils.geomap.GeoMapper().get_geo_values(geo_type) in parse_geo_sets() of _params.py -FIPS = ['04019', '19143', '29063', '36083'] # Example list of valid FIPS codes as strings -MSA = ['40660', '44180', '48620', '49420'] # Example list of valid MSA codes as strings +FIPS = [ + "04019", + "19143", + "29063", + "36083", +] # Example list of valid FIPS codes as strings +MSA = ["40660", "44180", "48620", "49420"] # Example list of valid MSA codes as strings + class CovidcastTestRow(CovidcastRow): @staticmethod @@ -51,31 +55,88 @@ def __post_init__(self): if isinstance(self.issue, date): self.issue = day_to_time_value(self.issue) if isinstance(self.value_updated_timestamp, date): - self.value_updated_timestamp = day_to_time_value(self.value_updated_timestamp) + self.value_updated_timestamp = day_to_time_value( + self.value_updated_timestamp + ) def _sanitize_fields(self, extra_checks: bool = True): if self.issue and self.issue < self.time_value: self.issue = self.time_value if self.issue: - self.lag = (time_value_to_day(self.issue) - time_value_to_day(self.time_value)).days + self.lag = ( + time_value_to_day(self.issue) - time_value_to_day(self.time_value) + ).days else: self.lag = None # This sanity checking is already done in CsvImporter, but it's here so the testing class gets it too. if pd.isna(self.value) and self.missing_value == Nans.NOT_MISSING: - self.missing_value = Nans.NOT_APPLICABLE.value if extra_checks else Nans.OTHER.value + self.missing_value = ( + Nans.NOT_APPLICABLE.value if extra_checks else Nans.OTHER.value + ) if pd.isna(self.stderr) and self.missing_stderr == Nans.NOT_MISSING: - self.missing_stderr = Nans.NOT_APPLICABLE.value if extra_checks else Nans.OTHER.value + self.missing_stderr = ( + Nans.NOT_APPLICABLE.value if extra_checks else Nans.OTHER.value + ) if pd.isna(self.sample_size) and self.missing_sample_size == Nans.NOT_MISSING: - self.missing_sample_size = Nans.NOT_APPLICABLE.value if extra_checks else Nans.OTHER.value + self.missing_sample_size = ( + Nans.NOT_APPLICABLE.value if extra_checks else Nans.OTHER.value + ) return self -def covidcast_rows_from_args(sanitize_fields: bool = False, test_mode: bool = True, **kwargs: Dict[str, Iterable]) -> List[CovidcastTestRow]: +class CovidcastTestBase(DelphiTestBase): + def localSetUp(self): + # use the local test instance of the database + secrets.db.host = 'delphi_database_epidata' + secrets.db.epi = ('user', 'pass') + + self._db = Database() + self._db.connect() + + # empty all of the data tables + for ( + table + ) in "epimetric_load epimetric_latest epimetric_full geo_dim signal_dim".split(): + self._db._cursor.execute(f"TRUNCATE TABLE {table};") + + # reset the `covidcast_meta_cache` table (it should always have one row) + self._db._cursor.execute( + 'update covidcast_meta_cache set timestamp = 0, epidata = "[]"' + ) + self._db._connection.commit() + + def localTearDown(self): + self._db.disconnect(False) + del self._db + + def _insert_rows(self, rows: Sequence[CovidcastTestRow]): + # inserts rows into the database using the full acquisition process, including 'dbjobs' load into history & latest tables + n = self._db.insert_or_update_bulk(rows) + print(f"{n} rows added to load table & dispatched to v4 schema") + # NOTE: this isnt expressly needed for our test cases, but would be if using external access (like through client lib) to ensure changes are visible outside of this db session + self._db._connection.commit() + + def params_from_row(self, row: CovidcastTestRow, **kwargs): + ret = { + "data_source": row.source, + "signals": row.signal, + "time_type": row.time_type, + "geo_type": row.geo_type, + "time_values": row.time_value, + "geo_value": row.geo_value, + } + ret.update(kwargs) + return ret + + +def covidcast_rows_from_args( + sanitize_fields: bool = False, test_mode: bool = True, **kwargs: Dict[str, Iterable] +) -> List[CovidcastTestRow]: """A convenience constructor for test rows. Example: @@ -89,12 +150,22 @@ def covidcast_rows_from_args(sanitize_fields: bool = False, test_mode: bool = Tr assert len(set(len(lst) for lst in kwargs.values())) == 1 if sanitize_fields: - return [CovidcastTestRow.make_default_row(**_kwargs)._sanitize_fields(extra_checks=test_mode) for _kwargs in transpose_dict(kwargs)] + return [ + CovidcastTestRow.make_default_row(**_kwargs)._sanitize_fields( + extra_checks=test_mode + ) + for _kwargs in transpose_dict(kwargs) + ] else: - return [CovidcastTestRow.make_default_row(**_kwargs) for _kwargs in transpose_dict(kwargs)] + return [ + CovidcastTestRow.make_default_row(**_kwargs) + for _kwargs in transpose_dict(kwargs) + ] -def covidcast_rows_from_records(records: Iterable[dict], sanity_check: bool = False) -> List[CovidcastTestRow]: +def covidcast_rows_from_records( + records: Iterable[dict], sanity_check: bool = False +) -> List[CovidcastTestRow]: """A convenience constructor. Default is different from from_args, because from_records is usually called on faux-API returns in tests, @@ -103,36 +174,60 @@ def covidcast_rows_from_records(records: Iterable[dict], sanity_check: bool = Fa You can use csv.DictReader before this to read a CSV file. """ records = list(records) - return [CovidcastTestRow.make_default_row(**record) if not sanity_check else CovidcastTestRow.make_default_row(**record)._sanitize_fields() for record in records] + return [ + CovidcastTestRow.make_default_row(**record) + if not sanity_check + else CovidcastTestRow.make_default_row(**record)._sanitize_fields() + for record in records + ] -def covidcast_rows_as_dicts(rows: Iterable[CovidcastTestRow], ignore_fields: Optional[List[str]] = None) -> List[dict]: +def covidcast_rows_as_dicts( + rows: Iterable[CovidcastTestRow], ignore_fields: Optional[List[str]] = None +) -> List[dict]: return [row.as_dict(ignore_fields=ignore_fields) for row in rows] -def covidcast_rows_as_dataframe(rows: Iterable[CovidcastTestRow], ignore_fields: Optional[List[str]] = None) -> pd.DataFrame: +def covidcast_rows_as_dataframe( + rows: Iterable[CovidcastTestRow], ignore_fields: Optional[List[str]] = None +) -> pd.DataFrame: if ignore_fields is None: ignore_fields = [] - columns = [field.name for field in fields(CovidcastTestRow) if field.name not in ignore_fields] + columns = [ + field.name + for field in fields(CovidcastTestRow) + if field.name not in ignore_fields + ] if rows: - df = pd.concat([row.as_dataframe(ignore_fields=ignore_fields) for row in rows], ignore_index=True) + df = pd.concat( + [row.as_dataframe(ignore_fields=ignore_fields) for row in rows], + ignore_index=True, + ) return df[columns] else: return pd.DataFrame(columns=columns) def covidcast_rows_as_api_row_df(rows: Iterable[CovidcastTestRow]) -> pd.DataFrame: - return covidcast_rows_as_dataframe(rows, ignore_fields=CovidcastTestRow._api_row_ignore_fields) + return covidcast_rows_as_dataframe( + rows, ignore_fields=CovidcastTestRow._api_row_ignore_fields + ) -def covidcast_rows_as_api_compatibility_row_df(rows: Iterable[CovidcastTestRow]) -> pd.DataFrame: - return covidcast_rows_as_dataframe(rows, ignore_fields=CovidcastTestRow._api_row_compatibility_ignore_fields) +def covidcast_rows_as_api_compatibility_row_df( + rows: Iterable[CovidcastTestRow], +) -> pd.DataFrame: + return covidcast_rows_as_dataframe( + rows, ignore_fields=CovidcastTestRow._api_row_compatibility_ignore_fields + ) def covidcast_rows_as_db_row_df(rows: Iterable[CovidcastTestRow]) -> pd.DataFrame: - return covidcast_rows_as_dataframe(rows, ignore_fields=CovidcastTestRow._db_row_ignore_fields) + return covidcast_rows_as_dataframe( + rows, ignore_fields=CovidcastTestRow._db_row_ignore_fields + ) def transpose_dict(d: Dict[Any, List[Any]]) -> List[Dict[Any, Any]]: @@ -145,65 +240,11 @@ def transpose_dict(d: Dict[Any, List[Any]]) -> List[Dict[Any, Any]]: return [dict(zip(d.keys(), values)) for values in zip(*d.values())] -def assert_frame_equal_no_order(df1: pd.DataFrame, df2: pd.DataFrame, index: List[str], **kwargs: Any) -> None: +def assert_frame_equal_no_order( + df1: pd.DataFrame, df2: pd.DataFrame, index: List[str], **kwargs: Any +) -> None: """Assert that two DataFrames are equal, ignoring the order of rows.""" # Remove any existing index. If it wasn't named, drop it. Set a new index and sort it. df1 = df1.reset_index().drop(columns="index").set_index(index).sort_index() df2 = df2.reset_index().drop(columns="index").set_index(index).sort_index() pd.testing.assert_frame_equal(df1, df2, **kwargs) - - -class CovidcastBase(unittest.TestCase): - def setUp(self): - # use the local test instance of the database - secrets.db.host = 'delphi_database_epidata' - secrets.db.epi = ('user', 'pass') - - self._db = Database() - self._db.connect() - - # empty all of the data tables - for table in "epimetric_load epimetric_latest epimetric_full geo_dim signal_dim".split(): - self._db._cursor.execute(f"TRUNCATE TABLE {table};") - self.localSetUp() - self._db._connection.commit() - - # clear all rate-limiting info from redis - r = Redis(host=REDIS_HOST, password=REDIS_PASSWORD) - for k in r.keys("LIMITER/*"): - r.delete(k) - - - def tearDown(self): - # close and destroy conenction to the database - self.localTearDown() - self._db.disconnect(False) - del self._db - - def localSetUp(self): - # stub; override in subclasses to perform custom setup. - # runs after tables have been truncated but before database changes have been committed - pass - - def localTearDown(self): - # stub; override in subclasses to perform custom teardown. - # runs after database changes have been committed - pass - - def _insert_rows(self, rows: Sequence[CovidcastTestRow]): - # inserts rows into the database using the full acquisition process, including 'dbjobs' load into history & latest tables - n = self._db.insert_or_update_bulk(rows) - print(f"{n} rows added to load table & dispatched to v4 schema") - self._db._connection.commit() # NOTE: this isnt expressly needed for our test cases, but would be if using external access (like through client lib) to ensure changes are visible outside of this db session - - def params_from_row(self, row: CovidcastTestRow, **kwargs): - ret = { - 'data_source': row.source, - 'signals': row.signal, - 'time_type': row.time_type, - 'geo_type': row.geo_type, - 'time_values': row.time_value, - 'geo_value': row.geo_value, - } - ret.update(kwargs) - return ret diff --git a/src/common/integration_test_base_class.py b/src/common/delphi_test_base.py similarity index 93% rename from src/common/integration_test_base_class.py rename to src/common/delphi_test_base.py index 47c9f68e4..28e76b675 100644 --- a/src/common/integration_test_base_class.py +++ b/src/common/delphi_test_base.py @@ -66,12 +66,18 @@ def localSetUp(self): # runs after user/api_key tables have been truncated, but before test-specific tables are created/deleted/truncated and before database changes have been committed pass + def localTearDown(self): + # stub; override in subclasses to perform custom teardown. + # runs after database changes have been committed + pass + @staticmethod def _clear_limits() -> None: limiter.storage.reset() def tearDown(self) -> None: """Perform per-test teardown.""" + self.localTearDown() self.cur.close() self.cnx.close() self._clear_limits() From e0c370d5eecc52e79e61f391b817f98f15c47c0e Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Tue, 26 Sep 2023 12:50:46 +0300 Subject: [PATCH 28/55] Replaced localSetUp with setUp in order to uselocalSetUp in covidcast tests --- src/common/covidcast_test_base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/common/covidcast_test_base.py b/src/common/covidcast_test_base.py index f318b14d9..6754bdc45 100644 --- a/src/common/covidcast_test_base.py +++ b/src/common/covidcast_test_base.py @@ -90,8 +90,9 @@ def _sanitize_fields(self, extra_checks: bool = True): class CovidcastTestBase(DelphiTestBase): - def localSetUp(self): + def setUp(self): # use the local test instance of the database + super().setUp() secrets.db.host = 'delphi_database_epidata' secrets.db.epi = ('user', 'pass') From 6fa0072d0b67cf33d59bc3790c75245eb8ccd4c4 Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Tue, 26 Sep 2023 12:52:27 +0300 Subject: [PATCH 29/55] Replaced base class to CovidcastTestBase, removed unnecessary code from setUp method. Removed few blank lines. --- .../server/test_covidcast_endpoints.py | 23 +++---------------- 1 file changed, 3 insertions(+), 20 deletions(-) diff --git a/integrations/server/test_covidcast_endpoints.py b/integrations/server/test_covidcast_endpoints.py index e79b6f32f..e16dbf406 100644 --- a/integrations/server/test_covidcast_endpoints.py +++ b/integrations/server/test_covidcast_endpoints.py @@ -10,7 +10,7 @@ import pandas as pd from delphi.epidata.maintenance.covidcast_meta_cache_updater import main as update_cache -from delphi.epidata.acquisition.covidcast.test_utils import CovidcastBase, CovidcastTestRow +from delphi.epidata.common.covidcast_test_base import CovidcastTestBase, CovidcastTestRow # use the local instance of the Epidata API BASE_URL = "http://delphi_web_epidata/epidata/covidcast" @@ -18,25 +18,12 @@ AUTH = ('epidata', 'key') -class CovidcastEndpointTests(CovidcastBase): +class CovidcastEndpointTests(CovidcastTestBase): """Tests the `covidcast/*` endpoint.""" def localSetUp(self): """Perform per-test setup.""" - # reset the `covidcast_meta_cache` table (it should always have one row) - self._db._cursor.execute('update covidcast_meta_cache set timestamp = 0, epidata = "[]"') - - cur = self._db._cursor - # NOTE: we must specify the db schema "epidata" here because the cursor/connection are bound to schema "covid" - cur.execute("TRUNCATE TABLE epidata.api_user") - cur.execute("TRUNCATE TABLE epidata.user_role") - cur.execute("TRUNCATE TABLE epidata.user_role_link") - cur.execute("INSERT INTO epidata.api_user (api_key, email) VALUES ('quidel_key', 'quidel_email')") - cur.execute("INSERT INTO epidata.user_role (name) VALUES ('quidel')") - cur.execute( - "INSERT INTO epidata.user_role_link (user_id, role_id) SELECT api_user.id, user_role.id FROM epidata.api_user JOIN epidata.user_role WHERE api_key='quidel_key' and user_role.name='quidel'" - ) - cur.execute("INSERT INTO epidata.api_user (api_key, email) VALUES ('key', 'email')") + self.role_name = "quidel" def _fetch(self, endpoint="/", is_compatibility=False, auth=AUTH, **params): # make the request @@ -145,7 +132,6 @@ def test_trend(self): out = self._fetch("/trend", signal=first.signal_pair(), geo=first.geo_pair(), date=last.time_value, window="20200401-20201212", basis=ref.time_value) - self.assertEqual(out["result"], 1) self.assertEqual(len(out["epidata"]), 1) trend = out["epidata"][0] @@ -168,7 +154,6 @@ def test_trend(self): self.assertEqual(trend["max_value"], last.value) self.assertEqual(trend["max_trend"], "steady") - def test_trendseries(self): """Request a signal from the /trendseries endpoint.""" @@ -235,7 +220,6 @@ def match_row(trend, row): self.assertEqual(trend["max_value"], first.value) self.assertEqual(trend["max_trend"], "decreasing") - def test_csv(self): """Request a signal from the /csv endpoint.""" @@ -253,7 +237,6 @@ def test_csv(self): self.assertEqual(df.shape, (len(rows), 10)) self.assertEqual(list(df.columns), ["geo_value", "signal", "time_value", "issue", "lag", "value", "stderr", "sample_size", "geo_type", "data_source"]) - def test_backfill(self): """Request a signal from the /backfill endpoint.""" From dcc31b120ceda6436a36c41e3f3578a33dd3211b Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Tue, 26 Sep 2023 13:05:47 +0300 Subject: [PATCH 30/55] Moved super().setUp() call to the end of the method. --- src/common/covidcast_test_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/covidcast_test_base.py b/src/common/covidcast_test_base.py index 6754bdc45..1ab58a911 100644 --- a/src/common/covidcast_test_base.py +++ b/src/common/covidcast_test_base.py @@ -92,7 +92,6 @@ def _sanitize_fields(self, extra_checks: bool = True): class CovidcastTestBase(DelphiTestBase): def setUp(self): # use the local test instance of the database - super().setUp() secrets.db.host = 'delphi_database_epidata' secrets.db.epi = ('user', 'pass') @@ -110,6 +109,7 @@ def setUp(self): 'update covidcast_meta_cache set timestamp = 0, epidata = "[]"' ) self._db._connection.commit() + super().setUp() def localTearDown(self): self._db.disconnect(False) From 0466420ad10ce06a548ad534e69675d8130c6e71 Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Tue, 26 Sep 2023 13:06:50 +0300 Subject: [PATCH 31/55] Replaced CovidcastBase with new CovidcastTestBase. Removed duplicated code from localSetUp() method. --- integrations/server/test_covidcast_meta.py | 59 +++------------------- 1 file changed, 8 insertions(+), 51 deletions(-) diff --git a/integrations/server/test_covidcast_meta.py b/integrations/server/test_covidcast_meta.py index 857422a41..9d7f62a1d 100644 --- a/integrations/server/test_covidcast_meta.py +++ b/integrations/server/test_covidcast_meta.py @@ -1,15 +1,12 @@ """Integration tests for the `covidcast_meta` endpoint.""" -# standard library -import unittest - # third party import mysql.connector import requests #first party from delphi_utils import Nans -from delphi.epidata.acquisition.covidcast.test_utils import CovidcastBase, CovidcastTestRow +from delphi.epidata.common.covidcast_test_base import CovidcastTestBase, CovidcastTestRow from delphi.epidata.maintenance.covidcast_meta_cache_updater import main as update_cache import delphi.operations.secrets as secrets @@ -18,7 +15,7 @@ AUTH = ('epidata', 'key') -class CovidcastMetaTests(CovidcastBase): +class CovidcastMetaTests(CovidcastTestBase): """Tests the `covidcast_meta` endpoint.""" src_sig_lookups = { @@ -52,63 +49,23 @@ class CovidcastMetaTests(CovidcastBase): def localSetUp(self): """Perform per-test setup.""" - # connect to the `epidata` database and clear the `covidcast` table - cnx = mysql.connector.connect( - user='user', - password='pass', - host='delphi_database_epidata', - database='covid') - cur = cnx.cursor() - - # clear all tables - cur.execute("truncate table epimetric_load") - cur.execute("truncate table epimetric_full") - cur.execute("truncate table epimetric_latest") - cur.execute("truncate table geo_dim") - cur.execute("truncate table signal_dim") - # reset the `covidcast_meta_cache` table (it should always have one row) - cur.execute('update covidcast_meta_cache set timestamp = 0, epidata = "[]"') - - # NOTE: we must specify the db schema "epidata" here because the cursor/connection are bound to schema "covid" - cur.execute("TRUNCATE TABLE epidata.api_user") - cur.execute("TRUNCATE TABLE epidata.user_role") - cur.execute("TRUNCATE TABLE epidata.user_role_link") - cur.execute("INSERT INTO epidata.api_user (api_key, email) VALUES ('quidel_key', 'quidel_email')") - cur.execute("INSERT INTO epidata.user_role (name) VALUES ('quidel')") - cur.execute( - "INSERT INTO epidata.user_role_link (user_id, role_id) SELECT api_user.id, user_role.id FROM epidata.api_user JOIN epidata.user_role WHERE api_key='quidel_key' and user_role.name='quidel'" - ) - cur.execute("INSERT INTO epidata.api_user (api_key, email) VALUES ('key', 'email')") + self.role_name = "quidel" # populate dimension tables for (src,sig) in self.src_sig_lookups: - cur.execute(''' + self._db._cursor.execute(''' INSERT INTO `signal_dim` (`signal_key_id`, `source`, `signal`) VALUES (%d, '%s', '%s'); ''' % ( self.src_sig_lookups[(src,sig)], src, sig )) for (gt,gv) in self.geo_lookups: - cur.execute(''' + self._db._cursor.execute(''' INSERT INTO `geo_dim` (`geo_key_id`, `geo_type`, `geo_value`) VALUES (%d, '%s', '%s'); ''' % ( self.geo_lookups[(gt,gv)], gt, gv )) - cnx.commit() - cur.close() + self._db._connection.commit() # initialize counter for tables without non-autoincrement id self.id_counter = 666 - # make connection and cursor available to test cases - self.cnx = cnx - self.cur = cnx.cursor() - - # use the local instance of the epidata database - secrets.db.host = 'delphi_database_epidata' - secrets.db.epi = ('user', 'pass') - - - def localTearDown(self): - """Perform per-test teardown.""" - self.cur.close() - self.cnx.close() def insert_placeholder_data(self): expected = [] @@ -135,13 +92,13 @@ def insert_placeholder_data(self): }) for tv in (1, 2): for gv, v in zip(('geo1', 'geo2'), (10, 20)): - self.cur.execute(self.template % ( + self._db._cursor.execute(self.template % ( self._get_id(), self.src_sig_lookups[(src,sig)], self.geo_lookups[(gt,gv)], tt, tv, v, tv, # re-use time value for issue Nans.NOT_MISSING, Nans.NOT_MISSING, Nans.NOT_MISSING )) - self.cnx.commit() + self._db._connection.commit() update_cache(args=None) return expected From 9e601c4dd5e166e72c0d35622979b08b0b71d2c8 Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Tue, 26 Sep 2023 15:37:56 +0300 Subject: [PATCH 32/55] Changed base class for CovidcastTestBase. Removed unnecessary code from setUp and changed it to the localSetUp --- integrations/server/test_covidcast_nowcast.py | 34 +++---------------- 1 file changed, 4 insertions(+), 30 deletions(-) diff --git a/integrations/server/test_covidcast_nowcast.py b/integrations/server/test_covidcast_nowcast.py index 32445afdf..267bddf04 100644 --- a/integrations/server/test_covidcast_nowcast.py +++ b/integrations/server/test_covidcast_nowcast.py @@ -1,45 +1,19 @@ """Integration tests for the `covidcast_nowcast` endpoint.""" - -# standard library -import unittest - -# third party -import mysql.connector import requests +from delphi.epidata.common.covidcast_test_base import CovidcastTestBase # use the local instance of the Epidata API BASE_URL = 'http://delphi_web_epidata/epidata' AUTH = ('epidata', 'key') -class CovidcastTests(unittest.TestCase): +class CovidcastTests(CovidcastTestBase): """Tests the `covidcast` endpoint.""" - def setUp(self): + def localSetUp(self): """Perform per-test setup.""" - - # connect to the `epidata` database and clear the `covidcast` table - cnx = mysql.connector.connect( - user='user', - password='pass', - host='delphi_database_epidata', - database='epidata') - cur = cnx.cursor() - cur.execute('truncate table covidcast_nowcast') - cur.execute('delete from api_user') - cur.execute('insert into api_user(api_key, email) values("key", "email")') - cnx.commit() - cur.close() - - # make connection and cursor available to test cases - self.cnx = cnx - self.cur = cnx.cursor() - - def tearDown(self): - """Perform per-test teardown.""" - self.cur.close() - self.cnx.close() + self.truncate_tables_list = ["covidcast_nowcast"] @staticmethod def _make_request(params: dict): From c5f74d64d55fafa74bf1d853d3df40974a16fc38 Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Tue, 26 Sep 2023 15:39:52 +0300 Subject: [PATCH 33/55] Removed unused import --- integrations/server/test_covidcast_meta.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/integrations/server/test_covidcast_meta.py b/integrations/server/test_covidcast_meta.py index 9d7f62a1d..0ea98493b 100644 --- a/integrations/server/test_covidcast_meta.py +++ b/integrations/server/test_covidcast_meta.py @@ -8,7 +8,6 @@ from delphi_utils import Nans from delphi.epidata.common.covidcast_test_base import CovidcastTestBase, CovidcastTestRow from delphi.epidata.maintenance.covidcast_meta_cache_updater import main as update_cache -import delphi.operations.secrets as secrets # use the local instance of the Epidata API BASE_URL = 'http://delphi_web_epidata/epidata' @@ -223,4 +222,3 @@ def test_filter(self): self.assertEqual(res['result'], 1) self.assertEqual(len(res['epidata']), len(expected)) self.assertEqual(res['epidata'][0], {}) - From 7ecae272a2d19b8f1c4f08c80d53eb30fe5002d8 Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Tue, 26 Sep 2023 15:50:29 +0300 Subject: [PATCH 34/55] Replaced base class, removed unused imports, replaced Epidata client with internal self.epidata_client from the base class. --- integrations/server/test_covidcast.py | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/integrations/server/test_covidcast.py b/integrations/server/test_covidcast.py index 4963ca9a8..e2b9f44e8 100644 --- a/integrations/server/test_covidcast.py +++ b/integrations/server/test_covidcast.py @@ -1,31 +1,20 @@ """Integration tests for the `covidcast` endpoint.""" -# standard library -from typing import Callable import unittest -# third party -import mysql.connector - # first party from delphi_utils import Nans -from delphi.epidata.acquisition.covidcast.test_utils import CovidcastBase, CovidcastTestRow, FIPS, MSA -from delphi.epidata.client.delphi_epidata import Epidata +from delphi.epidata.common.covidcast_test_base import CovidcastTestBase, CovidcastTestRow, FIPS, MSA -class CovidcastTests(CovidcastBase): +class CovidcastTests(CovidcastTestBase): """Tests the `covidcast` endpoint.""" - def localSetUp(self): - """Perform per-test setup.""" - self._db._cursor.execute('update covidcast_meta_cache set timestamp = 0, epidata = "[]"') - def request_based_on_row(self, row: CovidcastTestRow, **kwargs): params = self.params_from_row(row, endpoint='covidcast', **kwargs) # use the local instance of the Epidata API - Epidata.BASE_URL = 'http://delphi_web_epidata/epidata' - Epidata.auth = ('epidata', 'key') - response = Epidata.covidcast(**params) + self.epidata_client.auth = ('epidata', 'key') + response = self.epidata_client.covidcast(**params) return response From ae07c0f9f5224b9852591997c0ffe8598ad9bdc9 Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Tue, 26 Sep 2023 15:51:49 +0300 Subject: [PATCH 35/55] Fixed import --- integrations/server/test_delphi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integrations/server/test_delphi.py b/integrations/server/test_delphi.py index fc3e3bc7e..36e31e2b2 100644 --- a/integrations/server/test_delphi.py +++ b/integrations/server/test_delphi.py @@ -1,7 +1,7 @@ import json # first party -from delphi.epidata.common.integration_test_base_class import DelphiTestBase +from delphi.epidata.common.delphi_test_base import DelphiTestBase class DelphiTest(DelphiTestBase): From d3fa119218ed096d3805e8d8097dba49bd09c8d8 Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Tue, 26 Sep 2023 15:54:08 +0300 Subject: [PATCH 36/55] Fixed import --- integrations/server/test_dengue_nowcast.py | 2 +- integrations/server/test_dengue_sensors.py | 2 +- integrations/server/test_ecdc_ili.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/integrations/server/test_dengue_nowcast.py b/integrations/server/test_dengue_nowcast.py index 79f9765f4..8762472b8 100644 --- a/integrations/server/test_dengue_nowcast.py +++ b/integrations/server/test_dengue_nowcast.py @@ -1,5 +1,5 @@ # first party -from delphi.epidata.common.integration_test_base_class import DelphiTestBase +from delphi.epidata.common.delphi_test_base import DelphiTestBase class DengueNowcastTest(DelphiTestBase): diff --git a/integrations/server/test_dengue_sensors.py b/integrations/server/test_dengue_sensors.py index 55d103367..424bc24d6 100644 --- a/integrations/server/test_dengue_sensors.py +++ b/integrations/server/test_dengue_sensors.py @@ -1,5 +1,5 @@ # first party -from delphi.epidata.common.integration_test_base_class import DelphiTestBase +from delphi.epidata.common.delphi_test_base import DelphiTestBase class DengueSensorsTest(DelphiTestBase): diff --git a/integrations/server/test_ecdc_ili.py b/integrations/server/test_ecdc_ili.py index 5fe24bcd3..e49e0bcfa 100644 --- a/integrations/server/test_ecdc_ili.py +++ b/integrations/server/test_ecdc_ili.py @@ -1,5 +1,5 @@ # first party -from delphi.epidata.common.integration_test_base_class import DelphiTestBase +from delphi.epidata.common.delphi_test_base import DelphiTestBase class EcdcIliTest(DelphiTestBase): From d01df1fc14378a961e482c65026981897fdb20d6 Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Tue, 26 Sep 2023 16:08:48 +0300 Subject: [PATCH 37/55] Fixed wrong import --- integrations/server/test_flusurv.py | 2 +- integrations/server/test_fluview_clinical.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/integrations/server/test_flusurv.py b/integrations/server/test_flusurv.py index 33f0f00b8..586b28d1c 100644 --- a/integrations/server/test_flusurv.py +++ b/integrations/server/test_flusurv.py @@ -1,5 +1,5 @@ # first party -from delphi.epidata.common.integration_test_base_class import DelphiTestBase +from delphi.epidata.common.delphi_test_base import DelphiTestBase class FlusurvTest(DelphiTestBase): diff --git a/integrations/server/test_fluview_clinical.py b/integrations/server/test_fluview_clinical.py index 8ebccfc9e..300e0cb73 100644 --- a/integrations/server/test_fluview_clinical.py +++ b/integrations/server/test_fluview_clinical.py @@ -1,5 +1,5 @@ # first party -from delphi.epidata.common.integration_test_base_class import DelphiTestBase +from delphi.epidata.common.delphi_test_base import DelphiTestBase class FluviewClinicalTest(DelphiTestBase): From eba110b24b8427e0c98e9124edd6a0269b184b49 Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Tue, 26 Sep 2023 16:10:27 +0300 Subject: [PATCH 38/55] Changed base class. Removed unncessary code. Replaced Epidata calls with internal self.epidata_client --- integrations/server/test_fluview_meta.py | 45 +++--------------------- 1 file changed, 5 insertions(+), 40 deletions(-) diff --git a/integrations/server/test_fluview_meta.py b/integrations/server/test_fluview_meta.py index 6f81c1859..b23d79f9d 100644 --- a/integrations/server/test_fluview_meta.py +++ b/integrations/server/test_fluview_meta.py @@ -1,50 +1,15 @@ """Integration tests for the `fluview_meta` endpoint.""" -# standard library -import unittest - -# third party -import mysql.connector - # first party -from delphi.epidata.client.delphi_epidata import Epidata +from delphi.epidata.common.delphi_test_base import DelphiTestBase -class FluviewMetaTests(unittest.TestCase): +class FluviewMetaTests(DelphiTestBase): """Tests the `fluview_meta` endpoint.""" - @classmethod - def setUpClass(cls): - """Perform one-time setup.""" - - # use the local instance of the Epidata API - Epidata.BASE_URL = 'http://delphi_web_epidata/epidata' - Epidata.auth = ('epidata', 'key') - - def setUp(self): + def localSetUp(self): """Perform per-test setup.""" - - # connect to the `epidata` database and clear the `fluview` table - cnx = mysql.connector.connect( - user='user', - password='pass', - host='delphi_database_epidata', - database='epidata') - cur = cnx.cursor() - cur.execute('truncate table fluview') - cur.execute('delete from api_user') - cur.execute('insert into api_user(api_key, email) values ("key", "email")') - cnx.commit() - cur.close() - - # make connection and cursor available to test cases - self.cnx = cnx - self.cur = cnx.cursor() - - def tearDown(self): - """Perform per-test teardown.""" - self.cur.close() - self.cnx.close() + self.truncate_tables_list = ["fluview"] def test_round_trip(self): """Make a simple round-trip with some sample data.""" @@ -64,7 +29,7 @@ def test_round_trip(self): self.cnx.commit() # make the request - response = Epidata.fluview_meta() + response = self.epidata_client.fluview_meta() # assert that the right data came back self.assertEqual(response, { From 236e87d022a5af7e5b23835cff402643aa52ebf6 Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Tue, 26 Sep 2023 16:39:17 +0300 Subject: [PATCH 39/55] Removed epidata client auth set --- integrations/server/test_covidcast.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/integrations/server/test_covidcast.py b/integrations/server/test_covidcast.py index e2b9f44e8..3d3fadcac 100644 --- a/integrations/server/test_covidcast.py +++ b/integrations/server/test_covidcast.py @@ -12,8 +12,6 @@ class CovidcastTests(CovidcastTestBase): def request_based_on_row(self, row: CovidcastTestRow, **kwargs): params = self.params_from_row(row, endpoint='covidcast', **kwargs) - # use the local instance of the Epidata API - self.epidata_client.auth = ('epidata', 'key') response = self.epidata_client.covidcast(**params) return response From 92fcefd82dd3921c9057f987a10b08cdea756c28 Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Tue, 26 Sep 2023 16:39:51 +0300 Subject: [PATCH 40/55] Changed base class. Removed unnecessary code. --- integrations/server/test_fluview.py | 38 ++++------------------------- 1 file changed, 5 insertions(+), 33 deletions(-) diff --git a/integrations/server/test_fluview.py b/integrations/server/test_fluview.py index 48d9585fd..caf6ab75d 100644 --- a/integrations/server/test_fluview.py +++ b/integrations/server/test_fluview.py @@ -7,44 +7,16 @@ import mysql.connector # first party -from delphi.epidata.client.delphi_epidata import Epidata +from delphi.epidata.common.delphi_test_base import DelphiTestBase -class FluviewTests(unittest.TestCase): +class FluviewTests(DelphiTestBase): """Tests the `fluview` endpoint.""" - @classmethod - def setUpClass(cls): - """Perform one-time setup.""" - - # use the local instance of the Epidata API - Epidata.BASE_URL = 'http://delphi_web_epidata/epidata' - Epidata.auth = ('epidata', 'key') - - def setUp(self): + def localSetUp(self): """Perform per-test setup.""" - # connect to the `epidata` database and clear the `fluview` table - cnx = mysql.connector.connect( - user='user', - password='pass', - host='delphi_database_epidata', - database='epidata') - cur = cnx.cursor() - cur.execute('truncate table fluview') - cur.execute('delete from api_user') - cur.execute('insert into api_user(api_key, email) values ("key", "email")') - cnx.commit() - cur.close() - - # make connection and cursor available to test cases - self.cnx = cnx - self.cur = cnx.cursor() - - def tearDown(self): - """Perform per-test teardown.""" - self.cur.close() - self.cnx.close() + self.truncate_tables_list = ["fluview"] def test_round_trip(self): """Make a simple round-trip with some sample data.""" @@ -62,7 +34,7 @@ def test_round_trip(self): self.cnx.commit() # make the request - response = Epidata.fluview('nat', 202020) + response = self.epidata_client.fluview('nat', 202020) # assert that the right data came back self.assertEqual(response, { From 105be043a5f19388156e2f183f2b0cda8ac757e7 Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Tue, 26 Sep 2023 17:02:14 +0300 Subject: [PATCH 41/55] Fix wrong import --- integrations/server/test_gft.py | 2 +- integrations/server/test_ght.py | 2 +- integrations/server/test_kcdc_ili.py | 2 +- integrations/server/test_meta.py | 2 +- integrations/server/test_nidss_dengue.py | 2 +- integrations/server/test_nidss_flu.py | 2 +- integrations/server/test_norostat.py | 2 +- integrations/server/test_nowcast.py | 2 +- integrations/server/test_paho_dengue.py | 2 +- integrations/server/test_quidel.py | 2 +- integrations/server/test_sensors.py | 2 +- integrations/server/test_signal_dashboard.py | 2 +- integrations/server/test_twitter.py | 2 +- integrations/server/test_wiki.py | 2 +- tests/common/test_covidcast_row.py | 7 ++++--- 15 files changed, 18 insertions(+), 17 deletions(-) diff --git a/integrations/server/test_gft.py b/integrations/server/test_gft.py index 3f2a68526..80a0dc984 100644 --- a/integrations/server/test_gft.py +++ b/integrations/server/test_gft.py @@ -1,5 +1,5 @@ # first party -from delphi.epidata.common.integration_test_base_class import DelphiTestBase +from delphi.epidata.common.delphi_test_base import DelphiTestBase class GftTest(DelphiTestBase): diff --git a/integrations/server/test_ght.py b/integrations/server/test_ght.py index 67b135aef..115f20b28 100644 --- a/integrations/server/test_ght.py +++ b/integrations/server/test_ght.py @@ -1,5 +1,5 @@ # first party -from delphi.epidata.common.integration_test_base_class import DelphiTestBase +from delphi.epidata.common.delphi_test_base import DelphiTestBase class GhtTest(DelphiTestBase): diff --git a/integrations/server/test_kcdc_ili.py b/integrations/server/test_kcdc_ili.py index aab40baa6..48c1e5160 100644 --- a/integrations/server/test_kcdc_ili.py +++ b/integrations/server/test_kcdc_ili.py @@ -1,5 +1,5 @@ # first party -from delphi.epidata.common.integration_test_base_class import DelphiTestBase +from delphi.epidata.common.delphi_test_base import DelphiTestBase class KcdcIliTest(DelphiTestBase): diff --git a/integrations/server/test_meta.py b/integrations/server/test_meta.py index 295a5cd98..e0cd1013f 100644 --- a/integrations/server/test_meta.py +++ b/integrations/server/test_meta.py @@ -1,5 +1,5 @@ # first party -from delphi.epidata.common.integration_test_base_class import DelphiTestBase +from delphi.epidata.common.delphi_test_base import DelphiTestBase class MetaTest(DelphiTestBase): diff --git a/integrations/server/test_nidss_dengue.py b/integrations/server/test_nidss_dengue.py index 679937ea6..1b08048ce 100644 --- a/integrations/server/test_nidss_dengue.py +++ b/integrations/server/test_nidss_dengue.py @@ -1,5 +1,5 @@ # first party -from delphi.epidata.common.integration_test_base_class import DelphiTestBase +from delphi.epidata.common.delphi_test_base import DelphiTestBase class NiddsDengueTest(DelphiTestBase): diff --git a/integrations/server/test_nidss_flu.py b/integrations/server/test_nidss_flu.py index 0b13ee67f..678017502 100644 --- a/integrations/server/test_nidss_flu.py +++ b/integrations/server/test_nidss_flu.py @@ -1,5 +1,5 @@ # first party -from delphi.epidata.common.integration_test_base_class import DelphiTestBase +from delphi.epidata.common.delphi_test_base import DelphiTestBase class NiddsFluTest(DelphiTestBase): diff --git a/integrations/server/test_norostat.py b/integrations/server/test_norostat.py index a7866a91d..bfc2c2b80 100644 --- a/integrations/server/test_norostat.py +++ b/integrations/server/test_norostat.py @@ -1,5 +1,5 @@ # first party -from delphi.epidata.common.integration_test_base_class import DelphiTestBase +from delphi.epidata.common.delphi_test_base import DelphiTestBase class NorostatTest(DelphiTestBase): diff --git a/integrations/server/test_nowcast.py b/integrations/server/test_nowcast.py index 2b48dd0da..38897f0ae 100644 --- a/integrations/server/test_nowcast.py +++ b/integrations/server/test_nowcast.py @@ -1,5 +1,5 @@ # first party -from delphi.epidata.common.integration_test_base_class import DelphiTestBase +from delphi.epidata.common.delphi_test_base import DelphiTestBase class NowcastTest(DelphiTestBase): diff --git a/integrations/server/test_paho_dengue.py b/integrations/server/test_paho_dengue.py index bbe8953f8..911986897 100644 --- a/integrations/server/test_paho_dengue.py +++ b/integrations/server/test_paho_dengue.py @@ -1,5 +1,5 @@ # first party -from delphi.epidata.common.integration_test_base_class import DelphiTestBase +from delphi.epidata.common.delphi_test_base import DelphiTestBase class PahoDengueTest(DelphiTestBase): diff --git a/integrations/server/test_quidel.py b/integrations/server/test_quidel.py index 696a7ee41..30902eeae 100644 --- a/integrations/server/test_quidel.py +++ b/integrations/server/test_quidel.py @@ -1,5 +1,5 @@ # first party -from delphi.epidata.common.integration_test_base_class import DelphiTestBase +from delphi.epidata.common.delphi_test_base import DelphiTestBase class QuidelTest(DelphiTestBase): diff --git a/integrations/server/test_sensors.py b/integrations/server/test_sensors.py index 835b53893..c5afa329e 100644 --- a/integrations/server/test_sensors.py +++ b/integrations/server/test_sensors.py @@ -1,5 +1,5 @@ # first party -from delphi.epidata.common.integration_test_base_class import DelphiTestBase +from delphi.epidata.common.delphi_test_base import DelphiTestBase class SensorsTest(DelphiTestBase): diff --git a/integrations/server/test_signal_dashboard.py b/integrations/server/test_signal_dashboard.py index 4b122f3d3..30492e88b 100644 --- a/integrations/server/test_signal_dashboard.py +++ b/integrations/server/test_signal_dashboard.py @@ -1,5 +1,5 @@ # first party -from delphi.epidata.common.integration_test_base_class import DelphiTestBase +from delphi.epidata.common.delphi_test_base import DelphiTestBase class SignalDashboardTest(DelphiTestBase): diff --git a/integrations/server/test_twitter.py b/integrations/server/test_twitter.py index 7ba162480..18891c009 100644 --- a/integrations/server/test_twitter.py +++ b/integrations/server/test_twitter.py @@ -1,5 +1,5 @@ # first party -from delphi.epidata.common.integration_test_base_class import DelphiTestBase +from delphi.epidata.common.delphi_test_base import DelphiTestBase class TwitterTest(DelphiTestBase): diff --git a/integrations/server/test_wiki.py b/integrations/server/test_wiki.py index d53cef0c9..21638b43e 100644 --- a/integrations/server/test_wiki.py +++ b/integrations/server/test_wiki.py @@ -1,5 +1,5 @@ # first party -from delphi.epidata.common.integration_test_base_class import DelphiTestBase +from delphi.epidata.common.delphi_test_base import DelphiTestBase class WikiTest(DelphiTestBase): diff --git a/tests/common/test_covidcast_row.py b/tests/common/test_covidcast_row.py index 273596ebb..77e2e441e 100644 --- a/tests/common/test_covidcast_row.py +++ b/tests/common/test_covidcast_row.py @@ -5,20 +5,21 @@ from delphi_utils.nancodes import Nans from delphi.epidata.common.covidcast_row import CovidcastRow, set_df_dtypes -from delphi.epidata.acquisition.covidcast.test_utils import ( +from delphi.epidata.common.covidcast_test_base import ( CovidcastTestRow, covidcast_rows_as_api_compatibility_row_df, covidcast_rows_as_api_row_df, covidcast_rows_from_args, transpose_dict, + MSA, + CovidcastTestBase ) -from delphi.epidata.acquisition.covidcast.test_utils import MSA # py3tester coverage target (equivalent to `import *`) __test_target__ = "delphi.epidata.common.covidcast_row" -class TestCovidcastRows(unittest.TestCase): +class TestCovidcastRows(CovidcastTestBase): expected_df = set_df_dtypes( DataFrame( { From 7bdf683c90dfa7fe147a334ed5969a5a45d48f7a Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Wed, 27 Sep 2023 17:25:24 +0300 Subject: [PATCH 42/55] Replace unittest.TestCase with custom base class. Remove unnecessary code. Adjust test case's code. --- .../covidcast/test_covidcast_meta_caching.py | 88 ++++--------------- .../covidcast/test_csv_uploading.py | 77 +++------------- .../covidcast/test_delete_batch.py | 27 +----- .../covidcast_nowcast/test_csv_uploading.py | 44 ++-------- integrations/client/test_nowcast.py | 56 ++---------- integrations/server/test_api_keys.py | 2 +- integrations/server/test_cdc.py | 2 +- tests/server/test_params.py | 2 +- tests/server/test_query.py | 2 +- 9 files changed, 46 insertions(+), 254 deletions(-) diff --git a/integrations/acquisition/covidcast/test_covidcast_meta_caching.py b/integrations/acquisition/covidcast/test_covidcast_meta_caching.py index 30b6bbcf2..02f81fe22 100644 --- a/integrations/acquisition/covidcast/test_covidcast_meta_caching.py +++ b/integrations/acquisition/covidcast/test_covidcast_meta_caching.py @@ -2,17 +2,13 @@ # standard library import json -import unittest # third party -import mysql.connector import requests # first party from delphi_utils import Nans -from delphi.epidata.client.delphi_epidata import Epidata -import delphi.operations.secrets as secrets -import delphi.epidata.acquisition.covidcast.database as live +from delphi.epidata.common.covidcast_test_base import CovidcastTestBase from delphi.epidata.maintenance.covidcast_meta_cache_updater import main # py3tester coverage target (equivalent to `import *`) @@ -23,67 +19,16 @@ # use the local instance of the Epidata API BASE_URL = 'http://delphi_web_epidata/epidata' +AUTH = ("epidata", "key") -class CovidcastMetaCacheTests(unittest.TestCase): +class CovidcastMetaCacheTests(CovidcastTestBase): """Tests covidcast metadata caching.""" - def setUp(self): - """Perform per-test setup.""" - - # connect to the `epidata` database - cnx = mysql.connector.connect( - user='user', - password='pass', - host='delphi_database_epidata', - database='covid') - cur = cnx.cursor() - - # clear all tables - cur.execute("truncate table epimetric_load") - cur.execute("truncate table epimetric_full") - cur.execute("truncate table epimetric_latest") - cur.execute("truncate table geo_dim") - cur.execute("truncate table signal_dim") - # reset the `covidcast_meta_cache` table (it should always have one row) - cur.execute('update covidcast_meta_cache set timestamp = 0, epidata = "[]"') - cnx.commit() - cur.close() - - # make connection and cursor available to test cases - self.cnx = cnx - self.cur = cnx.cursor() - - # use the local instance of the epidata database - secrets.db.host = 'delphi_database_epidata' - secrets.db.epi = ('user', 'pass') - - epidata_cnx = mysql.connector.connect( - user='user', - password='pass', - host='delphi_database_epidata', - database='epidata') - epidata_cur = epidata_cnx.cursor() - - epidata_cur.execute("DELETE FROM `api_user`") - epidata_cur.execute('INSERT INTO `api_user`(`api_key`, `email`) VALUES("key", "email")') - epidata_cnx.commit() - epidata_cur.close() - epidata_cnx.close() - - # use the local instance of the Epidata API - Epidata.BASE_URL = BASE_URL - Epidata.auth = ('epidata', 'key') - - def tearDown(self): - """Perform per-test teardown.""" - self.cur.close() - self.cnx.close() - @staticmethod def _make_request(): params = {'cached': 'true'} - response = requests.get(f"{Epidata.BASE_URL}/covidcast_meta", params=params, auth=Epidata.auth) + response = requests.get(f"{BASE_URL}/covidcast_meta", params=params, auth=AUTH) response.raise_for_status() return response.json() @@ -91,18 +36,18 @@ def test_caching(self): """Populate, query, cache, query, and verify the cache.""" # insert dummy data - self.cur.execute(f''' + self._db._cursor.execute(f''' INSERT INTO `signal_dim` (`signal_key_id`, `source`, `signal`) VALUES (42, 'src', 'sig'); ''') - self.cur.execute(f''' + self._db._cursor.execute(f''' INSERT INTO `geo_dim` (`geo_key_id`, `geo_type`, `geo_value`) VALUES (96, 'state', 'pa'), (97, 'state', 'wa'); ''') - self.cur.execute(f''' + self._db._cursor.execute(f''' INSERT INTO `epimetric_latest` (`epimetric_id`, `signal_key_id`, `geo_key_id`, `time_type`, `time_value`, `value_updated_timestamp`, @@ -115,13 +60,10 @@ def test_caching(self): (16, 42, 97, 'day', 20200422, 789, 1, 2, 3, 20200423, 1, {Nans.NOT_MISSING}, {Nans.NOT_MISSING}, {Nans.NOT_MISSING}) ''') - self.cnx.commit() + self._db._connection.commit() # make sure the live utility is serving something sensible - cvc_database = live.Database() - cvc_database.connect() - epidata1 = cvc_database.compute_covidcast_meta() - cvc_database.disconnect(False) + epidata1 = self._db.compute_covidcast_meta() self.assertEqual(len(epidata1),1) self.assertEqual(epidata1, [ { @@ -146,7 +88,7 @@ def test_caching(self): # make sure the API covidcast_meta is still blank, since it only serves # the cached version and we haven't cached anything yet - epidata2 = Epidata.covidcast_meta() + epidata2 = self.epidata_client.covidcast_meta() self.assertEqual(epidata2['result'], -2, json.dumps(epidata2)) # update the cache @@ -154,18 +96,18 @@ def test_caching(self): main(args) # fetch the cached version - epidata3 = Epidata.covidcast_meta() + epidata3 = self.epidata_client.covidcast_meta() # cached version should now equal live version self.assertEqual(epidata1, epidata3) # insert dummy data timestamped as of now - self.cur.execute(''' + self._db._cursor.execute(''' update covidcast_meta_cache set timestamp = UNIX_TIMESTAMP(NOW()), epidata = '[{"hello": "world"}]' ''') - self.cnx.commit() + self._db._connection.commit() # fetch the cached version (manually) epidata4 = self._make_request() @@ -180,12 +122,12 @@ def test_caching(self): }) # insert dummy data timestamped as 2 hours old - self.cur.execute(''' + self._db._cursor.execute(''' update covidcast_meta_cache set timestamp = UNIX_TIMESTAMP(NOW()) - 3600 * 2, epidata = '[{"hello": "world"}]' ''') - self.cnx.commit() + self._db._connection.commit() # fetch the cached version (manually) epidata5 = self._make_request() diff --git a/integrations/acquisition/covidcast/test_csv_uploading.py b/integrations/acquisition/covidcast/test_csv_uploading.py index dab35f414..bad0e7eb2 100644 --- a/integrations/acquisition/covidcast/test_csv_uploading.py +++ b/integrations/acquisition/covidcast/test_csv_uploading.py @@ -3,80 +3,23 @@ # standard library from datetime import date import os -import unittest -import argparse # third party -import mysql.connector import pandas as pd import numpy as np # first party from delphi_utils import Nans -from delphi.epidata.client.delphi_epidata import Epidata from delphi.epidata.acquisition.covidcast.csv_to_database import main, get_argument_parser -import delphi.operations.secrets as secrets +from delphi.epidata.common.covidcast_test_base import CovidcastTestBase # py3tester coverage target (equivalent to `import *`) __test_target__ = 'delphi.epidata.acquisition.covidcast.csv_to_database' -class CsvUploadingTests(unittest.TestCase): +class CsvUploadingTests(CovidcastTestBase): """Tests covidcast CSV uploading.""" - def setUp(self): - """Perform per-test setup.""" - - # connect to the `epidata` database and clear the `covidcast` table - cnx = mysql.connector.connect( - user='user', - password='pass', - host='delphi_database_epidata', - database='covid') - cur = cnx.cursor() - - # clear all tables - cur.execute("truncate table epimetric_load") - cur.execute("truncate table epimetric_full") - cur.execute("truncate table epimetric_latest") - cur.execute("truncate table geo_dim") - cur.execute("truncate table signal_dim") - # reset the `covidcast_meta_cache` table (it should always have one row) - cur.execute('update covidcast_meta_cache set timestamp = 0, epidata = "[]"') - - cnx.commit() - cur.close() - - # make connection and cursor available to test cases - self.cnx = cnx - self.cur = cnx.cursor() - - # use the local instance of the epidata database - secrets.db.host = 'delphi_database_epidata' - secrets.db.epi = ('user', 'pass') - - epidata_cnx = mysql.connector.connect( - user='user', - password='pass', - host='delphi_database_epidata', - database='epidata') - epidata_cur = epidata_cnx.cursor() - - epidata_cur.execute("DELETE FROM `api_user`") - epidata_cur.execute('INSERT INTO `api_user`(`api_key`, `email`) VALUES("key", "email")') - epidata_cnx.commit() - epidata_cur.close() - epidata_cnx.close() - - # use the local instance of the Epidata API - Epidata.BASE_URL = 'http://delphi_web_epidata/epidata' - Epidata.auth = ('epidata', 'key') - - def tearDown(self): - """Perform per-test teardown.""" - self.cur.close() - self.cnx.close() - @staticmethod def apply_lag(expected_epidata): expected_issue_day=date.today() @@ -91,11 +34,11 @@ def apply_lag(expected_epidata): return expected_epidata def verify_timestamps_and_defaults(self): - self.cur.execute(''' + self._db._cursor.execute(''' select value_updated_timestamp from epimetric_full UNION ALL select value_updated_timestamp from epimetric_latest''') - for (value_updated_timestamp,) in self.cur: + for (value_updated_timestamp,) in self._db._cursor: self.assertGreater(value_updated_timestamp, 0) def test_uploading(self): @@ -130,7 +73,7 @@ def test_uploading(self): # upload CSVs main(args) - response = Epidata.covidcast('src-name', signal_name, 'day', 'state', 20200419, '*') + response = self.epidata_client.covidcast('src-name', signal_name, 'day', 'state', 20200419, '*') expected_values = pd.concat([values, pd.DataFrame({ "geo_type": "state", "source": "src-name", "time_type": "day", "time_value": [20200419] * 3, "signal": [signal_name] * 3, "direction": [None] * 3})], axis=1).rename(columns=uploader_column_rename).to_dict(orient="records") expected_response = {'result': 1, 'epidata': self.apply_lag(expected_values), 'message': 'success'} @@ -158,7 +101,7 @@ def test_uploading(self): # upload CSVs main(args) - response = Epidata.covidcast('src-name', signal_name, 'day', 'state', 20200419, '*') + response = self.epidata_client.covidcast('src-name', signal_name, 'day', 'state', 20200419, '*') expected_values = pd.concat([values, pd.DataFrame({ "geo_type": "state", @@ -195,7 +138,7 @@ def test_uploading(self): # upload CSVs main(args) - response = Epidata.covidcast('src-name', signal_name, 'day', 'state', 20200419, '*') + response = self.epidata_client.covidcast('src-name', signal_name, 'day', 'state', 20200419, '*') expected_response = {'epidata': [], 'result': -2, 'message': 'no results'} @@ -220,7 +163,7 @@ def test_uploading(self): # upload CSVs main(args) - response = Epidata.covidcast('src-name', signal_name, 'day', 'state', 20200419, '*') + response = self.epidata_client.covidcast('src-name', signal_name, 'day', 'state', 20200419, '*') expected_values_df = pd.concat([values, pd.DataFrame({ "geo_type": "state", @@ -256,7 +199,7 @@ def test_uploading(self): # upload CSVs main(args) - response = Epidata.covidcast('src-name', signal_name, 'day', 'state', 20200419, '*') + response = self.epidata_client.covidcast('src-name', signal_name, 'day', 'state', 20200419, '*') expected_values = pd.concat([values, pd.DataFrame({ "geo_type": "state", @@ -290,7 +233,7 @@ def test_uploading(self): # upload CSVs main(args) - response = Epidata.covidcast('src-name', signal_name, 'day', 'state', 20200419, '*') + response = self.epidata_client.covidcast('src-name', signal_name, 'day', 'state', 20200419, '*') expected_response = {'epidata': [], 'result': -2, 'message': 'no results'} diff --git a/integrations/acquisition/covidcast/test_delete_batch.py b/integrations/acquisition/covidcast/test_delete_batch.py index 4624df27c..c68b8a2b6 100644 --- a/integrations/acquisition/covidcast/test_delete_batch.py +++ b/integrations/acquisition/covidcast/test_delete_batch.py @@ -6,39 +6,16 @@ from os import path # first party -import delphi.operations.secrets as secrets -from delphi.epidata.acquisition.covidcast.database import Database -from delphi.epidata.acquisition.covidcast.test_utils import covidcast_rows_from_args +from delphi.epidata.common.covidcast_test_base import covidcast_rows_from_args, CovidcastTestBase # py3tester coverage target (equivalent to `import *`) __test_target__ = 'delphi.epidata.acquisition.covidcast.database' Example = namedtuple("example", "given expected") -class DeleteBatch(unittest.TestCase): +class DeleteBatch(CovidcastTestBase): """Tests batch deletions""" - - def setUp(self): - """Perform per-test setup.""" - - # use the local instance of the epidata database - secrets.db.host = 'delphi_database_epidata' - secrets.db.epi = ('user', 'pass') - - # will use secrets as set above - self._db = Database() - self._db.connect() - - for table in "epimetric_load epimetric_latest epimetric_full geo_dim signal_dim".split(): - self._db._cursor.execute(f"TRUNCATE TABLE {table}") - - - def tearDown(self): - """Perform per-test teardown.""" - self._db.disconnect(False) - del self._db - @unittest.skip("Database user would require FILE privileges") def test_delete_from_file(self): self._test_delete_batch(path.join(path.dirname(__file__), "delete_batch.csv")) diff --git a/integrations/acquisition/covidcast_nowcast/test_csv_uploading.py b/integrations/acquisition/covidcast_nowcast/test_csv_uploading.py index bf1a0f9a0..a46ddaa40 100644 --- a/integrations/acquisition/covidcast_nowcast/test_csv_uploading.py +++ b/integrations/acquisition/covidcast_nowcast/test_csv_uploading.py @@ -3,21 +3,18 @@ # standard library from datetime import date import os -import unittest from unittest.mock import patch from functools import partialmethod from datetime import date # third party -import mysql.connector import epiweeks as epi # first party -from delphi.epidata.client.delphi_epidata import Epidata from delphi.epidata.acquisition.covidcast_nowcast.load_sensors import main from delphi.epidata.acquisition.covidcast.csv_importer import CsvImporter -import delphi.operations.secrets as secrets +from delphi.epidata.common.covidcast_test_base import CovidcastTestBase # py3tester coverage target (equivalent to `import *`) __test_target__ = 'delphi.epidata.acquisition.covidcast_nowcast.load_sensors' @@ -27,41 +24,12 @@ ) -class CsvUploadingTests(unittest.TestCase): +class CsvUploadingTests(CovidcastTestBase): """Tests covidcast nowcast CSV uploading.""" - def setUp(self): + def localSetUp(self): """Perform per-test setup.""" - - # connect to the `epidata` database and clear the `covidcast` table - cnx = mysql.connector.connect( - user='user', - password='pass', - host='delphi_database_epidata', - database='epidata') - cur = cnx.cursor() - cur.execute('truncate table covidcast_nowcast') - cur.execute('delete from api_user') - cur.execute('insert into api_user(api_key, email) values ("key", "email")') - cnx.commit() - cur.close() - - # make connection and cursor available to test cases - self.cnx = cnx - self.cur = cnx.cursor() - - # use the local instance of the epidata database - secrets.db.host = 'delphi_database_epidata' - secrets.db.epi = ('user', 'pass') - - # use the local instance of the Epidata API - Epidata.BASE_URL = 'http://delphi_web_epidata/epidata' - Epidata.auth = ('epidata', 'key') - - def tearDown(self): - """Perform per-test teardown.""" - self.cur.close() - self.cnx.close() + self.truncate_tables_list = ["covidcast_nowcast"] @patch('delphi.epidata.acquisition.covidcast_nowcast.load_sensors.CsvImporter.find_csv_files', new=FIXED_ISSUE_IMPORTER) @@ -105,7 +73,7 @@ def test_uploading(self): ) # check data uploaded - response = Epidata.covidcast_nowcast( + response = self.epidata_client.covidcast_nowcast( 'src', 'sig', 'testsensor', 'day', 'state', 20200419, 'ca') self.assertEqual(response, { 'result': 1, @@ -141,7 +109,7 @@ def test_duplicate_row(self): main() # most most recent value is the one stored - response = Epidata.covidcast_nowcast( + response = self.epidata_client.covidcast_nowcast( 'src', 'sig', 'testsensor', 'day', 'state', 20200419, 'ca') self.assertEqual(response, { 'result': 1, diff --git a/integrations/client/test_nowcast.py b/integrations/client/test_nowcast.py index 84fc0e080..5645e8108 100644 --- a/integrations/client/test_nowcast.py +++ b/integrations/client/test_nowcast.py @@ -1,55 +1,17 @@ """Integration tests for delphi_epidata.py.""" -# standard library -import unittest - -# third party -import mysql.connector - # first party -from delphi.epidata.client.delphi_epidata import Epidata -import delphi.operations.secrets as secrets +from delphi.epidata.common.delphi_test_base import DelphiTestBase # py3tester coverage target __test_target__ = 'delphi.epidata.client.delphi_epidata' -class DelphiEpidataPythonClientNowcastTests(unittest.TestCase): +class DelphiEpidataPythonClientNowcastTests(DelphiTestBase): """Tests the Python client.""" - def setUp(self): + def localSetUp(self): """Perform per-test setup.""" - - # connect to the `epidata` database and clear relevant tables - cnx = mysql.connector.connect( - user='user', - password='pass', - host='delphi_database_epidata', - database='epidata') - cur = cnx.cursor() - - cur.execute('truncate table covidcast_nowcast') - cur.execute('delete from api_user') - cur.execute('insert into api_user(api_key, email) values ("key", "email")') - - cnx.commit() - cur.close() - - # make connection and cursor available to test cases - self.cnx = cnx - self.cur = cnx.cursor() - - # use the local instance of the Epidata API - Epidata.BASE_URL = 'http://delphi_web_epidata/epidata' - Epidata.auth = ('epidata', 'key') - - # use the local instance of the epidata database - secrets.db.host = 'delphi_database_epidata' - secrets.db.epi = ('user', 'pass') - - def tearDown(self): - """Perform per-test teardown.""" - self.cur.close() - self.cnx.close() + self.truncate_tables_list = ["covidcast_nowcast"] def test_covidcast_nowcast(self): """Test that the covidcast_nowcast endpoint returns expected data.""" @@ -62,7 +24,7 @@ def test_covidcast_nowcast(self): self.cnx.commit() # fetch data - response = Epidata.covidcast_nowcast( + response = self.epidata_client.covidcast_nowcast( 'src', ['sig1', 'sig2'], 'sensor', 'day', 'county', 20200101, '01001') # request two signals @@ -87,9 +49,9 @@ def test_covidcast_nowcast(self): }) # request range of issues - response = Epidata.covidcast_nowcast( + response = self.epidata_client.covidcast_nowcast( 'src', 'sig1', 'sensor', 'day', 'county', 20200101, '01001', - issues=Epidata.range(20200101, 20200102)) + issues=self.epidata_client.range(20200101, 20200102)) self.assertEqual(response, { 'result': 1, @@ -112,7 +74,7 @@ def test_covidcast_nowcast(self): }) # request as_of - response = Epidata.covidcast_nowcast( + response = self.epidata_client.covidcast_nowcast( 'src', 'sig1', 'sensor', 'day', 'county', 20200101, '01001', as_of=20200101) @@ -130,7 +92,7 @@ def test_covidcast_nowcast(self): }) # request unavailable data - response = Epidata.covidcast_nowcast( + response = self.epidata_client.covidcast_nowcast( 'src', 'sig1', 'sensor', 'day', 'county', 22222222, '01001') self.assertEqual(response, {'epidata': [], 'result': -2, 'message': 'no results'}) diff --git a/integrations/server/test_api_keys.py b/integrations/server/test_api_keys.py index 3ba67e324..d686220c1 100644 --- a/integrations/server/test_api_keys.py +++ b/integrations/server/test_api_keys.py @@ -2,7 +2,7 @@ import requests # first party -from delphi.epidata.common.integration_test_base_class import DelphiTestBase +from delphi.epidata.common.delphi_test_base import DelphiTestBase class APIKeysTets(DelphiTestBase): diff --git a/integrations/server/test_cdc.py b/integrations/server/test_cdc.py index d468bd162..7f60ddca1 100644 --- a/integrations/server/test_cdc.py +++ b/integrations/server/test_cdc.py @@ -1,5 +1,5 @@ # first party -from delphi.epidata.common.integration_test_base_class import DelphiTestBase +from delphi.epidata.common.delphi_test_base import DelphiTestBase class CdcTest(DelphiTestBase): diff --git a/tests/server/test_params.py b/tests/server/test_params.py index 6d6de4fdc..cd7d701c2 100644 --- a/tests/server/test_params.py +++ b/tests/server/test_params.py @@ -28,7 +28,7 @@ from delphi.epidata.server._exceptions import ( ValidationFailedException, ) -from delphi.epidata.acquisition.covidcast.test_utils import FIPS, MSA +from delphi.epidata.common.covidcast_test_base import FIPS, MSA # py3tester coverage target __test_target__ = "delphi.epidata.server._params" diff --git a/tests/server/test_query.py b/tests/server/test_query.py index ec07d3e8b..d3d904fab 100644 --- a/tests/server/test_query.py +++ b/tests/server/test_query.py @@ -21,7 +21,7 @@ TimeSet, SourceSignalSet, ) -from delphi.epidata.acquisition.covidcast.test_utils import FIPS, MSA +from delphi.epidata.common.covidcast_test_base import FIPS, MSA # py3tester coverage target __test_target__ = "delphi.epidata.server._query" From 987d9901eb3afd3c9e8a32574c0442f91b454d03 Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Thu, 28 Sep 2023 23:31:34 +0300 Subject: [PATCH 43/55] Final adjustments --- .../covidcast/test_covidcast_meta_caching.py | 18 +- integrations/server/test_api_keys.py | 30 +-- .../server/test_covidcast_endpoints.py | 203 +++++++++++------- integrations/server/test_covidcast_meta.py | 93 +++++--- integrations/server/test_covidcast_nowcast.py | 19 +- integrations/server/test_fluview.py | 6 - integrations/server/test_ght.py | 3 +- integrations/server/test_signal_dashboard.py | 2 + src/common/delphi_test_base.py | 33 ++- 9 files changed, 237 insertions(+), 170 deletions(-) diff --git a/integrations/acquisition/covidcast/test_covidcast_meta_caching.py b/integrations/acquisition/covidcast/test_covidcast_meta_caching.py index 02f81fe22..4111561eb 100644 --- a/integrations/acquisition/covidcast/test_covidcast_meta_caching.py +++ b/integrations/acquisition/covidcast/test_covidcast_meta_caching.py @@ -3,9 +3,6 @@ # standard library import json -# third party -import requests - # first party from delphi_utils import Nans from delphi.epidata.common.covidcast_test_base import CovidcastTestBase @@ -17,21 +14,10 @@ 'covidcast_meta_cache_updater' ) -# use the local instance of the Epidata API -BASE_URL = 'http://delphi_web_epidata/epidata' -AUTH = ("epidata", "key") - class CovidcastMetaCacheTests(CovidcastTestBase): """Tests covidcast metadata caching.""" - @staticmethod - def _make_request(): - params = {'cached': 'true'} - response = requests.get(f"{BASE_URL}/covidcast_meta", params=params, auth=AUTH) - response.raise_for_status() - return response.json() - def test_caching(self): """Populate, query, cache, query, and verify the cache.""" @@ -110,7 +96,7 @@ def test_caching(self): self._db._connection.commit() # fetch the cached version (manually) - epidata4 = self._make_request() + epidata4 = self._make_request(endpoint="covidcast_meta", json=True, params={'cached': 'true'}, auth=self.epidata_client.auth, raise_for_status=True) # make sure the cache was actually served self.assertEqual(epidata4, { @@ -130,7 +116,7 @@ def test_caching(self): self._db._connection.commit() # fetch the cached version (manually) - epidata5 = self._make_request() + epidata5 = self._make_request(endpoint="covidcast_meta", json=True, params={'cached': 'true'}, auth=self.epidata_client.auth, raise_for_status=True) # make sure the cache was returned anyhow self.assertEqual(epidata4, epidata5) diff --git a/integrations/server/test_api_keys.py b/integrations/server/test_api_keys.py index d686220c1..449c5b72d 100644 --- a/integrations/server/test_api_keys.py +++ b/integrations/server/test_api_keys.py @@ -1,6 +1,4 @@ """Integration tests for the API Keys""" -import requests - # first party from delphi.epidata.common.delphi_test_base import DelphiTestBase @@ -11,12 +9,6 @@ class APIKeysTets(DelphiTestBase): def localSetUp(self): self.role_name = "cdc" - def _make_request(self, url: str = None, endpoint: str = None, params: dict = {}, auth: tuple = None): - if not url: - url = self.epidata_client.BASE_URL - response = requests.get(f"{url}/{endpoint}", params=params, auth=auth) - return response - def test_public_route(self): """Test public route""" status_codes = set() @@ -36,7 +28,7 @@ def test_no_multiples_data_source(self): } status_codes = set() for _ in range(10): - status_codes.add(self._make_request(params=params, endpoint="covidcast").status_code) + status_codes.add(self._make_request(params=params).status_code) self.assertEqual(status_codes, {200}) def test_no_multiples_source_signal(self): @@ -50,7 +42,7 @@ def test_no_multiples_source_signal(self): } status_codes = set() for _ in range(10): - status_codes.add(self._make_request(params=params, endpoint="covidcast").status_code) + status_codes.add(self._make_request(params=params).status_code) self.assertEqual(status_codes, {200}) def test_multiples_allowed_signal_two_multiples(self): @@ -64,7 +56,7 @@ def test_multiples_allowed_signal_two_multiples(self): } status_codes = set() for _ in range(10): - status_codes.add(self._make_request(params=params, endpoint="covidcast").status_code) + status_codes.add(self._make_request(params=params).status_code) self.assertEqual(status_codes, {200}) def test_multiples_non_allowed_signal(self): @@ -78,7 +70,7 @@ def test_multiples_non_allowed_signal(self): } status_codes = set() for _ in range(10): - status_codes.add(self._make_request(params=params, endpoint="covidcast").status_code) + status_codes.add(self._make_request(params=params).status_code) self.assertEqual(status_codes, {200, 429}) def test_multiples_mixed_allowed_signal_two_multiples(self): @@ -92,7 +84,7 @@ def test_multiples_mixed_allowed_signal_two_multiples(self): } status_codes = set() for _ in range(10): - status_codes.add(self._make_request(params=params, endpoint="covidcast").status_code) + status_codes.add(self._make_request(params=params).status_code) self.assertEqual(status_codes, {200, 429}) def test_multiples_allowed_signal_three_multiples(self): @@ -106,7 +98,7 @@ def test_multiples_allowed_signal_three_multiples(self): } status_codes = set() for _ in range(10): - status_codes.add(self._make_request(params=params, endpoint="covidcast").status_code) + status_codes.add(self._make_request(params=params).status_code) self.assertEqual(status_codes, {401}) def test_multiples_mixed_allowed_signal_three_multiples(self): @@ -120,7 +112,7 @@ def test_multiples_mixed_allowed_signal_three_multiples(self): } status_codes = set() for _ in range(10): - status_codes.add(self._make_request(params=params, endpoint="covidcast").status_code) + status_codes.add(self._make_request(params=params).status_code) self.assertEqual(status_codes, {401}) def test_multiples_mixed_allowed_signal_api_key(self): @@ -135,7 +127,7 @@ def test_multiples_mixed_allowed_signal_api_key(self): status_codes = set() for _ in range(10): status_codes.add( - self._make_request(params=params, auth=self.epidata_client.auth, endpoint="covidcast").status_code + self._make_request(params=params, auth=self.epidata_client.auth).status_code ) self.assertEqual(status_codes, {200}) @@ -151,7 +143,7 @@ def test_multiples_allowed_signal_api_key(self): status_codes = set() for _ in range(10): status_codes.add( - self._make_request(params=params, auth=self.epidata_client.auth, endpoint="covidcast").status_code + self._make_request(params=params, auth=self.epidata_client.auth).status_code ) self.assertEqual(status_codes, {200}) @@ -167,7 +159,7 @@ def test_no_multiples_allowed_signal_api_key(self): status_codes = set() for _ in range(10): status_codes.add( - self._make_request(params=params, auth=self.epidata_client.auth, endpoint="covidcast").status_code + self._make_request(params=params, auth=self.epidata_client.auth).status_code ) self.assertEqual(status_codes, {200}) @@ -184,7 +176,7 @@ def test_no_multiples_allowed_signal_bad_api_key(self): for _ in range(10): status_codes.add( self._make_request( - params=params, auth=("bad_key", "bad_email"), endpoint="covidcast" + params=params, auth=("bad_key", "bad_email") ).status_code ) self.assertEqual(status_codes, {200}) diff --git a/integrations/server/test_covidcast_endpoints.py b/integrations/server/test_covidcast_endpoints.py index e16dbf406..9d75e5a6a 100644 --- a/integrations/server/test_covidcast_endpoints.py +++ b/integrations/server/test_covidcast_endpoints.py @@ -6,17 +6,11 @@ # third party from more_itertools import windowed -import requests import pandas as pd from delphi.epidata.maintenance.covidcast_meta_cache_updater import main as update_cache from delphi.epidata.common.covidcast_test_base import CovidcastTestBase, CovidcastTestRow -# use the local instance of the Epidata API -BASE_URL = "http://delphi_web_epidata/epidata/covidcast" -BASE_URL_OLD = "http://delphi_web_epidata/epidata/api.php" -AUTH = ('epidata', 'key') - class CovidcastEndpointTests(CovidcastTestBase): """Tests the `covidcast/*` endpoint.""" @@ -25,21 +19,6 @@ def localSetUp(self): """Perform per-test setup.""" self.role_name = "quidel" - def _fetch(self, endpoint="/", is_compatibility=False, auth=AUTH, **params): - # make the request - if is_compatibility: - url = BASE_URL_OLD - # only set endpoint if it's not already set - # only set endpoint if it's not already set - params.setdefault("endpoint", "covidcast") - if params.get("source"): - params.setdefault("data_source", params.get("source")) - else: - url = f"{BASE_URL}{endpoint}" - response = requests.get(url, params=params, auth=auth) - response.raise_for_status() - return response.json() - def _diff_rows(self, rows: Sequence[float]): return [ float(x - y) if x is not None and y is not None else None @@ -59,11 +38,16 @@ def test_basic(self): self._insert_rows(rows) with self.subTest("validation"): - out = self._fetch("/") + out = self._make_request(auth=self.epidata_client.auth, json=True, raise_for_status=True) self.assertEqual(out["result"], -1) with self.subTest("simple"): - out = self._fetch("/", signal=first.signal_pair(), geo=first.geo_pair(), time="day:*") + params = { + "signal": first.signal_pair(), + "geo": first.geo_pair(), + "time": "day:*" + } + out = self._make_request(auth=self.epidata_client.auth, json=True, raise_for_status=True, params=params) self.assertEqual(len(out["epidata"]), len(rows)) def test_basic_restricted_source(self): @@ -73,52 +57,71 @@ def test_basic_restricted_source(self): self._insert_rows(rows) with self.subTest("validation"): - out = self._fetch("/") + out = self._make_request(auth=self.epidata_client.auth, json=True, raise_for_status=True) self.assertEqual(out["result"], -1) with self.subTest("no_roles"): - out = self._fetch("/", signal=first.signal_pair(), geo=first.geo_pair(), time="day:*") + params = { + "signal": first.signal_pair(), + "geo": first.geo_pair(), + "time": "day:*" + } + out = self._make_request(auth=self.epidata_client.auth, json=True, raise_for_status=True, params=params) self.assertEqual(len(out["epidata"]), 0) with self.subTest("no_api_key"): - out = self._fetch("/", auth=None, signal=first.signal_pair(), geo=first.geo_pair(), time="day:*") + params = { + "signal": first.signal_pair(), + "geo": first.geo_pair(), + "time": "day:*" + } + out = self._make_request(json=True, raise_for_status=True, params=params) self.assertEqual(len(out["epidata"]), 0) with self.subTest("quidel_role"): - out = self._fetch("/", auth=("epidata", "quidel_key"), signal=first.signal_pair(), geo=first.geo_pair(), time="day:*") + params = { + "signal": first.signal_pair(), + "geo": first.geo_pair(), + "time": "day:*", + "auth": "quidel_key" + } + out = self._make_request(json=True, raise_for_status=True, params=params) self.assertEqual(len(out["epidata"]), len(rows)) - def test_compatibility(self): - """Request at the /api.php endpoint.""" - rows = [CovidcastTestRow.make_default_row(source="src", signal="sig", time_value=2020_04_01 + i, value=i) for i in range(10)] - first = rows[0] - self._insert_rows(rows) - - with self.subTest("validation"): - out = self._fetch("/", is_compatibility=True) - self.assertEqual(out["result"], -1) - - with self.subTest("simple"): - out = self._fetch("/", signal=first.signal_pair(), geo=first.geo_pair(), time="day:*", is_compatibility=True) - self.assertEqual(out["epidata"], [row.as_api_compatibility_row_dict() for row in rows]) - - def test_compatibility_restricted_source(self): - """Restricted request at the /api.php endpoint.""" - rows = [CovidcastTestRow.make_default_row(time_value=2020_04_01 + i, value=i, source="quidel") for i in range(10)] - first = rows[0] - self._insert_rows(rows) - - with self.subTest("no_roles"): - out = self._fetch("/", signal=first.signal_pair(), geo=first.geo_pair(), time="day:*", is_compatibility=True) - self.assertTrue("epidata" not in out) - - with self.subTest("no_api_key"): - out = self._fetch("/", auth=None, signal=first.signal_pair(), geo=first.geo_pair(), time="day:*", is_compatibility=True) - self.assertTrue("epidata" not in out) - - with self.subTest("quidel_role"): - out = self._fetch("/", auth=("epidata", "quidel_key"), signal=first.signal_pair(), geo=first.geo_pair(), time="day:*", is_compatibility=True) - self.assertEqual(out["epidata"], [row.as_api_compatibility_row_dict() for row in rows]) + # Commented out this section as we want go get rid of legacy .php code + # ----------------------------------------- + # def test_compatibility(self): + # """Request at the /api.php endpoint.""" + # rows = [CovidcastTestRow.make_default_row(source="src", signal="sig", time_value=2020_04_01 + i, value=i) for i in range(10)] + # first = rows[0] + # self._insert_rows(rows) + + # with self.subTest("validation"): + # out = self._fetch("/", is_compatibility=True) + # self.assertEqual(out["result"], -1) + + # with self.subTest("simple"): + # out = self._fetch("/", signal=first.signal_pair(), geo=first.geo_pair(), time="day:*", is_compatibility=True) + # self.assertEqual(out["epidata"], [row.as_api_compatibility_row_dict() for row in rows]) + + # def test_compatibility_restricted_source(self): + # """Restricted request at the /api.php endpoint.""" + # rows = [CovidcastTestRow.make_default_row(time_value=2020_04_01 + i, value=i, source="quidel") for i in range(10)] + # first = rows[0] + # self._insert_rows(rows) + + # with self.subTest("no_roles"): + # out = self._fetch("/", signal=first.signal_pair(), geo=first.geo_pair(), time="day:*", is_compatibility=True) + # self.assertTrue("epidata" not in out) + + # with self.subTest("no_api_key"): + # out = self._fetch("/", auth=None, signal=first.signal_pair(), geo=first.geo_pair(), time="day:*", is_compatibility=True) + # self.assertTrue("epidata" not in out) + + # with self.subTest("quidel_role"): + # out = self._fetch("/", auth=("epidata", "quidel_key"), signal=first.signal_pair(), geo=first.geo_pair(), time="day:*", is_compatibility=True) + # self.assertEqual(out["epidata"], [row.as_api_compatibility_row_dict() for row in rows]) + # ----------------------------------------- def test_trend(self): """Request a signal from the /trend endpoint.""" @@ -130,7 +133,14 @@ def test_trend(self): ref = rows[num_rows // 2] self._insert_rows(rows) - out = self._fetch("/trend", signal=first.signal_pair(), geo=first.geo_pair(), date=last.time_value, window="20200401-20201212", basis=ref.time_value) + params = { + "signal": first.signal_pair(), + "geo": first.geo_pair(), + "date": last.time_value, + "window": "20200401-20201212", + "basis": ref.time_value + } + out = self._make_request(endpoint="covidcast/trend", auth=self.epidata_client.auth, json=True, raise_for_status=True, params=params) self.assertEqual(out["result"], 1) self.assertEqual(len(out["epidata"]), 1) @@ -163,7 +173,14 @@ def test_trendseries(self): last = rows[-1] self._insert_rows(rows) - out = self._fetch("/trendseries", signal=first.signal_pair(), geo=first.geo_pair(), date=last.time_value, window="20200401-20200410", basis=1) + params = { + "signal": first.signal_pair(), + "geo": first.geo_pair(), + "date": last.time_value, + "window": "20200401-20200410", + "basis": 1 + } + out = self._make_request(endpoint="covidcast/trendseries", auth=self.epidata_client.auth, json=True, raise_for_status=True, params=params) self.assertEqual(out["result"], 1) self.assertEqual(len(out["epidata"]), 3) @@ -227,11 +244,13 @@ def test_csv(self): first = rows[0] self._insert_rows(rows) - response = requests.get( - f"{BASE_URL}/csv", - params=dict(signal=first.signal_pair(), start_day="2020-04-01", end_day="2020-12-12", geo_type=first.geo_type), - ) - response.raise_for_status() + params = { + "signal": first.signal_pair(), + "start_day": "2020-04-01", + "end_day": "2020-12-12", + "geo_type": first.geo_type + } + response = self._make_request(endpoint="covidcast/csv", raise_for_status=True, params=params) out = response.text df = pd.read_csv(StringIO(out), index_col=0) self.assertEqual(df.shape, (len(rows), 10)) @@ -248,7 +267,13 @@ def test_backfill(self): self._insert_rows([*issue_0, *issue_1, *last_issue]) first = issue_0[0] - out = self._fetch("/backfill", signal=first.signal_pair(), geo=first.geo_pair(), time="day:20200401-20201212", anchor_lag=3) + params = { + "signal": first.signal_pair(), + "geo": first.geo_pair(), + "time": "day:20200401-20201212", + "anchor_lag": 3 + } + out = self._make_request(endpoint="covidcast/backfill", auth=self.epidata_client.auth, json=True, raise_for_status=True, params=params) self.assertEqual(out["result"], 1) df = pd.DataFrame(out["epidata"]) self.assertEqual(len(df), 3 * num_rows) # num issues @@ -278,7 +303,7 @@ def test_meta(self): update_cache(args=None) with self.subTest("plain"): - out = self._fetch("/meta") + out = self._make_request(endpoint="covidcast/meta", auth=self.epidata_client.auth, json=True, raise_for_status=True) self.assertEqual(len(out), 1) data_source = out[0] self.assertEqual(data_source["source"], first.source) @@ -296,20 +321,26 @@ def test_meta(self): self.assertEqual(stats_g["mean"], sum(r.value for r in rows) / len(rows)) with self.subTest("filtered"): - out = self._fetch("/meta", signal=f"{first.source}:*") + params = { + "signal": f"{first.source}:*" + } + out = self._make_request(endpoint="covidcast/meta", auth=self.epidata_client.auth, json=True, raise_for_status=True, params=params) self.assertEqual(len(out), 1) data_source = out[0] self.assertEqual(data_source["source"], first.source) self.assertEqual(len(data_source["signals"]), 1) stats = data_source["signals"][0] self.assertEqual(stats["source"], first.source) - out = self._fetch("/meta", signal=f"{first.source}:X") + params = { + "signal": f"{first.source}:X" + } + out = self._make_request(endpoint="covidcast/meta", auth=self.epidata_client.auth, json=True, raise_for_status=True, params=params) self.assertEqual(len(out), 0) def test_meta_restricted(self): """Request 'restricted' signals from the /meta endpoint.""" # NOTE: this method is nearly identical to ./test_covidcast_meta.py:test_restricted_sources() - # ...except the self._fetch() methods are different, as is the format of those methods' outputs + # ...except the self._make_request() methods are different, as is the format of those methods' outputs # (the other covidcast_meta endpoint uses APrinter, this one returns its own unadulterated json). # additionally, the sample data used here must match entries (that is, named sources and signals) # from covidcast_utils.model.data_sources (the `data_sources` variable from file @@ -328,12 +359,15 @@ def test_meta_restricted(self): update_cache(args=None) # verify unauthenticated (no api key) or unauthorized (user w/o privilege) only see metadata for one source - self.assertEqual(len(self._fetch("/meta", auth=None)), 1) - self.assertEqual(len(self._fetch("/meta", auth=AUTH)), 1) + unauthenticated_request = self._make_request(endpoint="covidcast/meta", json=True, raise_for_status=True) + unauthorized_request = self._make_request(endpoint="covidcast/meta", auth=self.epidata_client.auth, json=True, raise_for_status=True) + self.assertEqual(len(unauthenticated_request), 1) + self.assertEqual(len(unauthorized_request), 1) # verify authorized user sees metadata for both sources qauth = ('epidata', 'quidel_key') - self.assertEqual(len(self._fetch("/meta", auth=qauth)), 2) + authorized_request = self._make_request(endpoint="covidcast/meta", auth=qauth, json=True, raise_for_status=True) + self.assertEqual(len(authorized_request), 2) def test_coverage(self): """Request a signal from the /coverage endpoint.""" @@ -345,17 +379,34 @@ def test_coverage(self): first = rows[0] with self.subTest("default"): - out = self._fetch("/coverage", signal=first.signal_pair(), geo_type=first.geo_type, latest=dates[-1], format="json") + params = { + "signal": first.signal_pair(), + "geo_type": first.geo_type, + "latest": dates[-1], + "format": "json" + } + out = self._make_request(endpoint="covidcast/coverage", auth=self.epidata_client.auth, json=True, raise_for_status=True, params=params) self.assertEqual(len(out), len(num_geos_per_date)) self.assertEqual([o["time_value"] for o in out], dates) self.assertEqual([o["count"] for o in out], num_geos_per_date) with self.subTest("specify window"): - out = self._fetch("/coverage", signal=first.signal_pair(), geo_type=first.geo_type, window=f"{dates[0]}-{dates[1]}", format="json") + params = { + "signal": first.signal_pair(), + "geo_type": first.geo_type, + "window": f"{dates[0]}-{dates[1]}", + "format": "json" + } + out = self._make_request("covidcast/coverage", auth=self.epidata_client.auth, json=True, raise_for_status=True, params=params) self.assertEqual(len(out), 2) self.assertEqual([o["time_value"] for o in out], dates[:2]) self.assertEqual([o["count"] for o in out], num_geos_per_date[:2]) with self.subTest("invalid geo_type"): - out = self._fetch("/coverage", signal=first.signal_pair(), geo_type="doesnt_exist", format="json") + params = { + "signal": first.signal_pair(), + "geo_type": "doesnt_exist", + "format": "json" + } + out = self._make_request(endpoint="covidcast/coverage", auth=self.epidata_client.auth, json=True, raise_for_status=True, params=params) self.assertEqual(len(out), 0) diff --git a/integrations/server/test_covidcast_meta.py b/integrations/server/test_covidcast_meta.py index 0ea98493b..e79ec523c 100644 --- a/integrations/server/test_covidcast_meta.py +++ b/integrations/server/test_covidcast_meta.py @@ -1,18 +1,10 @@ """Integration tests for the `covidcast_meta` endpoint.""" -# third party -import mysql.connector -import requests - #first party from delphi_utils import Nans from delphi.epidata.common.covidcast_test_base import CovidcastTestBase, CovidcastTestRow from delphi.epidata.maintenance.covidcast_meta_cache_updater import main as update_cache -# use the local instance of the Epidata API -BASE_URL = 'http://delphi_web_epidata/epidata' -AUTH = ('epidata', 'key') - class CovidcastMetaTests(CovidcastTestBase): """Tests the `covidcast_meta` endpoint.""" @@ -104,13 +96,6 @@ def insert_placeholder_data(self): def _get_id(self): self.id_counter += 1 return self.id_counter - - @staticmethod - def _fetch(auth=AUTH, **kwargs): - params = kwargs.copy() - response = requests.get(f"{BASE_URL}/covidcast_meta", params=params, auth=auth) - response.raise_for_status() - return response.json() def test_round_trip(self): """Make a simple round-trip with some sample data.""" @@ -119,7 +104,7 @@ def test_round_trip(self): expected = self.insert_placeholder_data() # make the request - response = self._fetch() + response = self._make_request(endpoint="covidcast_meta", auth=self.epidata_client.auth, json=True, raise_for_status=True) # assert that the right data came back self.assertEqual(response, { @@ -141,12 +126,15 @@ def test_restricted_sources(self): update_cache(args=None) # verify unauthenticated (no api key) or unauthorized (user w/o privilege) only see metadata for one source - self.assertEqual(len(self._fetch(auth=None)['epidata']), 1) - self.assertEqual(len(self._fetch(auth=AUTH)['epidata']), 1) + unauthenticated_request = self._make_request(endpoint="covidcast_meta", json=True, raise_for_status=True) + self.assertEqual(len(unauthenticated_request['epidata']), 1) + unauthorized_request = self._make_request(endpoint="covidcast_meta", auth=self.epidata_client.auth, json=True, raise_for_status=True) + self.assertEqual(len(unauthorized_request['epidata']), 1) # verify authorized user sees metadata for both sources qauth = ('epidata', 'quidel_key') - self.assertEqual(len(self._fetch(auth=qauth)['epidata']), 2) + authorized_request = self._make_request(endpoint="covidcast_meta", auth=qauth, json=True, raise_for_status=True) + self.assertEqual(len(authorized_request['epidata']), 2) def test_filter(self): """Test filtering options some sample data.""" @@ -154,63 +142,99 @@ def test_filter(self): # insert placeholder data and accumulate expected results (in sort order) expected = self.insert_placeholder_data() - res = self._fetch() + res = self._make_request(endpoint="covidcast_meta", auth=self.epidata_client.auth, json=True, raise_for_status=True) self.assertEqual(res['result'], 1) self.assertEqual(len(res['epidata']), len(expected)) # time types - res = self._fetch(time_types='day') + params = { + "time_types": "day" + } + res = self._make_request(endpoint="covidcast_meta", auth=self.epidata_client.auth, json=True, raise_for_status=True, params=params) self.assertEqual(res['result'], 1) self.assertEqual(len(res['epidata']), sum([1 for s in expected if s['time_type'] == 'day'])) - res = self._fetch(time_types='day,week') + params = { + "time_types": "day,week" + } + res = self._make_request(endpoint="covidcast_meta", auth=self.epidata_client.auth, json=True, raise_for_status=True, params=params) self.assertEqual(res['result'], 1) self.assertTrue(isinstance(res['epidata'], list)) self.assertEqual(len(res['epidata']), len(expected)) - res = self._fetch(time_types='sec') + params = { + "time_types": "sec" + } + res = self._make_request(endpoint="covidcast_meta", auth=self.epidata_client.auth, json=True, raise_for_status=True, params=params) self.assertEqual(res['result'], -2) # geo types - res = self._fetch(geo_types='hrr') + params = { + "geo_types": "hrr" + } + res = self._make_request(endpoint="covidcast_meta", auth=self.epidata_client.auth, json=True, raise_for_status=True, params=params) self.assertEqual(res['result'], 1) self.assertTrue(isinstance(res['epidata'], list)) self.assertEqual(len(res['epidata']), sum([1 for s in expected if s['geo_type'] == 'hrr'])) - res = self._fetch(geo_types='hrr,msa') + params = { + "geo_types": "hrr,msa" + } + res = self._make_request(endpoint="covidcast_meta", auth=self.epidata_client.auth, json=True, raise_for_status=True, params=params) self.assertEqual(res['result'], 1) self.assertTrue(isinstance(res['epidata'], list)) self.assertEqual(len(res['epidata']), len(expected)) - res = self._fetch(geo_types='state') + params = { + "geo_types": "state" + } + res = self._make_request(endpoint="covidcast_meta", auth=self.epidata_client.auth, json=True, raise_for_status=True, params=params) self.assertEqual(res['result'], -2) # signals - res = self._fetch(signals='src1:sig1') + params = { + "signals": "src1:sig1" + } + res = self._make_request(endpoint="covidcast_meta", auth=self.epidata_client.auth, json=True, raise_for_status=True, params=params) self.assertEqual(res['result'], 1) self.assertTrue(isinstance(res['epidata'], list)) self.assertEqual(len(res['epidata']), sum([1 for s in expected if s['data_source'] == 'src1' and s['signal'] == 'sig1'])) - res = self._fetch(signals='src1') + params = { + "signals": "src1" + } + res = self._make_request(endpoint="covidcast_meta", auth=self.epidata_client.auth, json=True, raise_for_status=True, params=params) self.assertEqual(res['result'], 1) self.assertTrue(isinstance(res['epidata'], list)) self.assertEqual(len(res['epidata']), sum([1 for s in expected if s['data_source'] == 'src1'])) - res = self._fetch(signals='src1:*') + params = { + "signals": "src1:*" + } + res = self._make_request(endpoint="covidcast_meta", auth=self.epidata_client.auth, json=True, raise_for_status=True, params=params) self.assertEqual(res['result'], 1) self.assertTrue(isinstance(res['epidata'], list)) self.assertEqual(len(res['epidata']), sum([1 for s in expected if s['data_source'] == 'src1'])) - res = self._fetch(signals='src1:src4') + params = { + "signals": "src1:src4" + } + res = self._make_request(endpoint="covidcast_meta", auth=self.epidata_client.auth, json=True, raise_for_status=True, params=params) self.assertEqual(res['result'], -2) - res = self._fetch(signals='src1:*,src2:*') + params = { + "signals": "src1:*,src2:*" + } + res = self._make_request(endpoint="covidcast_meta", auth=self.epidata_client.auth, json=True, raise_for_status=True, params=params) self.assertEqual(res['result'], 1) self.assertTrue(isinstance(res['epidata'], list)) self.assertEqual(len(res['epidata']), len(expected)) # filter fields - res = self._fetch(fields='data_source,min_time') + params = { + "fields": "data_source,min_time" + } + res = self._make_request(endpoint="covidcast_meta", auth=self.epidata_client.auth, json=True, raise_for_status=True, params=params) self.assertEqual(res['result'], 1) self.assertEqual(len(res['epidata']), len(expected)) self.assertTrue('data_source' in res['epidata'][0]) @@ -218,7 +242,10 @@ def test_filter(self): self.assertFalse('max_time' in res['epidata'][0]) self.assertFalse('signal' in res['epidata'][0]) - res = self._fetch(fields='xx') + params = { + "fields": "xx" + } + res = self._make_request(endpoint="covidcast_meta", auth=self.epidata_client.auth, json=True, raise_for_status=True, params=params) self.assertEqual(res['result'], 1) self.assertEqual(len(res['epidata']), len(expected)) self.assertEqual(res['epidata'][0], {}) diff --git a/integrations/server/test_covidcast_nowcast.py b/integrations/server/test_covidcast_nowcast.py index 267bddf04..a33d6b0b8 100644 --- a/integrations/server/test_covidcast_nowcast.py +++ b/integrations/server/test_covidcast_nowcast.py @@ -1,12 +1,7 @@ """Integration tests for the `covidcast_nowcast` endpoint.""" -import requests from delphi.epidata.common.covidcast_test_base import CovidcastTestBase -# use the local instance of the Epidata API -BASE_URL = 'http://delphi_web_epidata/epidata' -AUTH = ('epidata', 'key') - class CovidcastTests(CovidcastTestBase): """Tests the `covidcast` endpoint.""" @@ -15,12 +10,6 @@ def localSetUp(self): """Perform per-test setup.""" self.truncate_tables_list = ["covidcast_nowcast"] - @staticmethod - def _make_request(params: dict): - response = requests.get(f"{BASE_URL}/covidcast_nowcast", params=params, auth=AUTH) - response.raise_for_status() - return response.json() - def test_query(self): """Query nowcasts using default and specified issue.""" @@ -42,7 +31,7 @@ def test_query(self): 'geo_value': '01001', 'issues': 20200101 } - response = self._make_request(params=params) + response = self._make_request(endpoint="covidcast_nowcast", auth=self.epidata_client.auth, json=True, raise_for_status=True, params=params) self.assertEqual(response, { 'result': 1, 'epidata': [{ @@ -58,7 +47,6 @@ def test_query(self): # make request without specific issue date params={ - 'source': 'covidcast_nowcast', 'data_source': 'src', 'signals': 'sig', 'sensor_names': 'sensor', @@ -67,7 +55,7 @@ def test_query(self): 'time_values': 20200101, 'geo_value': '01001', } - response = self._make_request(params=params) + response = self._make_request(endpoint="covidcast_nowcast", auth=self.epidata_client.auth, json=True, raise_for_status=True, params=params) self.assertEqual(response, { 'result': 1, @@ -83,7 +71,6 @@ def test_query(self): }) params={ - 'source': 'covidcast_nowcast', 'data_source': 'src', 'signals': 'sig', 'sensor_names': 'sensor', @@ -93,7 +80,7 @@ def test_query(self): 'geo_value': '01001', 'as_of': 20200101 } - response = self._make_request(params=params) + response = self._make_request(endpoint="covidcast_nowcast", auth=self.epidata_client.auth, json=True, raise_for_status=True, params=params) self.assertEqual(response, { 'result': 1, diff --git a/integrations/server/test_fluview.py b/integrations/server/test_fluview.py index caf6ab75d..eb6c1ec0f 100644 --- a/integrations/server/test_fluview.py +++ b/integrations/server/test_fluview.py @@ -1,11 +1,5 @@ """Integration tests for the `fluview` endpoint.""" -# standard library -import unittest - -# third party -import mysql.connector - # first party from delphi.epidata.common.delphi_test_base import DelphiTestBase diff --git a/integrations/server/test_ght.py b/integrations/server/test_ght.py index 115f20b28..44b3497a1 100644 --- a/integrations/server/test_ght.py +++ b/integrations/server/test_ght.py @@ -16,7 +16,8 @@ def test_ght(self): ("/n/query", "US", "200101", "12345"), ) self.cnx.commit() - response = self.epidata_client.ght(locations="US", epiweeks="200101", query="/n/query", auth="ght_key") + ghtauth = ("epidata", "ght_key") + response = self.epidata_client.ght(locations="US", epiweeks="200101", query="/n/query", auth=ghtauth) self.assertEqual( response, {"epidata": [{"location": "US", "epiweek": 200101, "value": 12345.0}], "result": 1, "message": "success"}, diff --git a/integrations/server/test_signal_dashboard.py b/integrations/server/test_signal_dashboard.py index 30492e88b..18627601d 100644 --- a/integrations/server/test_signal_dashboard.py +++ b/integrations/server/test_signal_dashboard.py @@ -5,6 +5,8 @@ class SignalDashboardTest(DelphiTestBase): """Basic integration tests for signal_dashboard_coverage and signal_dashboard_status endpints.""" + # NOTE: In all other tests localSetUp() method was used. But it is not applicable for this test + # due to order of commands, so thats why method reload + calling super was required. def setUp(self) -> None: """Perform per-test setup.""" diff --git a/src/common/delphi_test_base.py b/src/common/delphi_test_base.py index 28e76b675..675f024a3 100644 --- a/src/common/delphi_test_base.py +++ b/src/common/delphi_test_base.py @@ -3,6 +3,7 @@ # third party import mysql.connector +import requests # first party from delphi.epidata.client.delphi_epidata import Epidata @@ -23,23 +24,49 @@ def __init__(self, methodName: str = "runTest") -> None: self.epidata_client.auth = ("epidata", "key") def create_key_with_role(self, cur, role_name: str): - cur.execute(f'INSERT INTO `api_user`(`api_key`, `email`) VALUES("{role_name}_key", "{role_name}_email")') + cur.execute( + f'INSERT INTO `api_user`(`api_key`, `email`) VALUES("{role_name}_key", "{role_name}_email")' + ) cur.execute(f'INSERT INTO `user_role`(`name`) VALUES("{role_name}")') cur.execute( f'INSERT INTO `user_role_link`(`user_id`, `role_id`) SELECT `api_user`.`id`, `user_role`.`id` FROM `api_user` JOIN `user_role` WHERE `api_user`.`api_key`="{role_name}_key" AND `user_role`.`name`="{role_name}"' ) + def _make_request( + self, + endpoint: str = "covidcast", + auth: tuple = None, + json: bool = False, + raise_for_status: bool = False, + params: dict = None, + ): + response = requests.get( + f"{self.epidata_client.BASE_URL}/{endpoint}", params=params, auth=auth + ) + if raise_for_status: + response.raise_for_status() + if json: + return response.json() + return response + def setUp(self) -> None: """Perform per-test setup.""" # connect to the `epidata` database - cnx = mysql.connector.connect(user="user", password="pass", host="delphi_database_epidata", database="epidata") + cnx = mysql.connector.connect( + user="user", + password="pass", + host="delphi_database_epidata", + database="epidata", + ) cur = cnx.cursor() cur.execute("DELETE FROM `api_user`") cur.execute("TRUNCATE TABLE `user_role`") cur.execute("TRUNCATE TABLE `user_role_link`") - cur.execute('INSERT INTO `api_user`(`api_key`, `email`) VALUES ("key", "email")') + cur.execute( + 'INSERT INTO `api_user`(`api_key`, `email`) VALUES ("key", "email")' + ) self.localSetUp() From a2377f4e76768d9a8e31d171e3c880f2edc9b92b Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Mon, 2 Oct 2023 20:26:31 +0300 Subject: [PATCH 44/55] Fixed ght test --- integrations/server/test_ght.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integrations/server/test_ght.py b/integrations/server/test_ght.py index 44b3497a1..370a08f41 100644 --- a/integrations/server/test_ght.py +++ b/integrations/server/test_ght.py @@ -16,7 +16,7 @@ def test_ght(self): ("/n/query", "US", "200101", "12345"), ) self.cnx.commit() - ghtauth = ("epidata", "ght_key") + ghtauth = "ght_key" response = self.epidata_client.ght(locations="US", epiweeks="200101", query="/n/query", auth=ghtauth) self.assertEqual( response, From 9eb559cfcbaa5a8c7bc8de625fcc97a7b8ba66c5 Mon Sep 17 00:00:00 2001 From: george haff Date: Mon, 2 Oct 2023 14:02:35 -0400 Subject: [PATCH 45/55] revert to previous behavior, and mark deprecated ( not fully tested) --- src/client/delphi_epidata.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/client/delphi_epidata.py b/src/client/delphi_epidata.py index e773bdc50..99d8f4abc 100644 --- a/src/client/delphi_epidata.py +++ b/src/client/delphi_epidata.py @@ -691,14 +691,17 @@ def covidcast_nowcast( return Epidata._request("covidcast_nowcast", params) @staticmethod - def async_epidata(endpoint, param_list, batch_size=50): - """Make asynchronous Epidata calls for a list of parameters.""" + def async_epidata(param_list, batch_size=50): + """[DEPRECATED] Make asynchronous Epidata calls for a list of parameters.""" - request_url = f"{Epidata.BASE_URL}/{endpoint}" + import warnings + warnings.filterwarnings("once", category=DeprecationWarning, module="delphi_epidata") + warnings.warn("Method `Epidata.async_epidata()` is deprecated and will be removed in a future version.", + category=DeprecationWarning) async def async_get(params, session): """Helper function to make Epidata GET requests.""" - async with session.get(request_url, params=params) as response: + async with session.get(f"{Epidata.BASE_URL}/api.php", params=params) as response: response.raise_for_status() return await response.json(), params From 3f5e1a6fc8ceeddb4904dea5e433189924df82e1 Mon Sep 17 00:00:00 2001 From: george haff Date: Mon, 2 Oct 2023 14:13:30 -0400 Subject: [PATCH 46/55] also revert tests --- integrations/client/test_delphi_epidata.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/integrations/client/test_delphi_epidata.py b/integrations/client/test_delphi_epidata.py index 99473f0ac..361105b22 100644 --- a/integrations/client/test_delphi_epidata.py +++ b/integrations/client/test_delphi_epidata.py @@ -335,9 +335,9 @@ def test_async_epidata(self): ] self._insert_rows(rows) - test_output = Epidata.async_epidata('covidcast', [ - self.params_from_row(rows[0]), - self.params_from_row(rows[1]) + test_output = Epidata.async_epidata([ + self.params_from_row(rows[0], source='covidcast'), + self.params_from_row(rows[1], source='covidcast') ]*12, batch_size=10) responses = [i[0] for i in test_output] # check response is same as standard covidcast call, using 24 calls to test batch sizing @@ -352,8 +352,9 @@ def test_async_epidata(self): @fake_epidata_endpoint def test_async_epidata_fail(self): with pytest.raises(ClientResponseError, match="404, message='NOT FOUND'"): - Epidata.async_epidata('covidcast', [ + Epidata.async_epidata([ { + 'source': 'covidcast', 'data_source': 'src', 'signals': 'sig', 'time_type': 'day', From 5f0b5211522d9d7dc08227d12344e754703358d4 Mon Sep 17 00:00:00 2001 From: Rostyslav Zatserkovnyi Date: Tue, 3 Oct 2023 18:57:30 +0300 Subject: [PATCH 47/55] test_api_keys tweak --- integrations/server/test_api_keys.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integrations/server/test_api_keys.py b/integrations/server/test_api_keys.py index 0c5be4c0b..d5c742d3d 100644 --- a/integrations/server/test_api_keys.py +++ b/integrations/server/test_api_keys.py @@ -21,7 +21,7 @@ def test_public_route(self): public_route = "http://delphi_web_epidata/epidata/version" status_codes = set() for _ in range(10): - status_codes.add(self._make_request("version", public_route).status_code) + status_codes.add(self._make_request("version").status_code) self.assertEqual(status_codes, {200}) def test_no_multiples_data_source(self): From cd665bd39ee93d4b13df1aa0e8cb2e76a2a263a9 Mon Sep 17 00:00:00 2001 From: Rostyslav Zatserkovnyi Date: Tue, 3 Oct 2023 19:42:56 +0300 Subject: [PATCH 48/55] Fix failing test --- integrations/client/test_delphi_epidata.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/integrations/client/test_delphi_epidata.py b/integrations/client/test_delphi_epidata.py index 361105b22..ff32d1ac6 100644 --- a/integrations/client/test_delphi_epidata.py +++ b/integrations/client/test_delphi_epidata.py @@ -339,13 +339,15 @@ def test_async_epidata(self): self.params_from_row(rows[0], source='covidcast'), self.params_from_row(rows[1], source='covidcast') ]*12, batch_size=10) - responses = [i[0] for i in test_output] - # check response is same as standard covidcast call, using 24 calls to test batch sizing + responses = [i[0]["epidata"] for i in test_output] + # check response is same as standard covidcast call (minus fields omitted by the api.php endpoint), + # using 24 calls to test batch sizing + ignore_fields = ["source", "time_type", "geo_type"] self.assertEqual( responses, [ - Epidata.covidcast(**self.params_from_row(rows[0])), - Epidata.covidcast(**self.params_from_row(rows[1])), + [{k: v for k, v in row.items() if k not in ignore_fields} for row in Epidata.covidcast(**self.params_from_row(rows[0]))["epidata"]], + [{k: v for k, v in row.items() if k not in ignore_fields} for row in Epidata.covidcast(**self.params_from_row(rows[1]))["epidata"]], ]*12 ) From 65d1587e154931bc29e8278336ea15ded4c76fdd Mon Sep 17 00:00:00 2001 From: Rostyslav Zatserkovnyi Date: Wed, 4 Oct 2023 18:18:42 +0300 Subject: [PATCH 49/55] More test tweaks --- integrations/client/test_delphi_epidata.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/integrations/client/test_delphi_epidata.py b/integrations/client/test_delphi_epidata.py index ff32d1ac6..8c2f26803 100644 --- a/integrations/client/test_delphi_epidata.py +++ b/integrations/client/test_delphi_epidata.py @@ -342,12 +342,12 @@ def test_async_epidata(self): responses = [i[0]["epidata"] for i in test_output] # check response is same as standard covidcast call (minus fields omitted by the api.php endpoint), # using 24 calls to test batch sizing - ignore_fields = ["source", "time_type", "geo_type"] + ignore_fields = CovidcastTestRow._api_row_compatibility_ignore_fields self.assertEqual( responses, [ - [{k: v for k, v in row.items() if k not in ignore_fields} for row in Epidata.covidcast(**self.params_from_row(rows[0]))["epidata"]], - [{k: v for k, v in row.items() if k not in ignore_fields} for row in Epidata.covidcast(**self.params_from_row(rows[1]))["epidata"]], + [{k: row[k] for k in row.keys() - ignore_fields} for row in Epidata.covidcast(**self.params_from_row(rows[0]))["epidata"]], + [{k: row[k] for k in row.keys() - ignore_fields} for row in Epidata.covidcast(**self.params_from_row(rows[1]))["epidata"]], ]*12 ) From e701cb6d8d81687f0be58e4e065b8981e3bc9f54 Mon Sep 17 00:00:00 2001 From: george haff Date: Thu, 5 Oct 2023 14:55:31 -0400 Subject: [PATCH 50/55] changelog update --- src/client/packaging/pypi/CHANGELOG.md | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/client/packaging/pypi/CHANGELOG.md b/src/client/packaging/pypi/CHANGELOG.md index ec322b6ae..f138604ad 100644 --- a/src/client/packaging/pypi/CHANGELOG.md +++ b/src/client/packaging/pypi/CHANGELOG.md @@ -3,12 +3,16 @@ All notable future changes to the `delphi_epidata` python client will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/). -## [4.2.0] - 2023-09-27 +## [4.1.11] - 2023-10-12 + +https://github.com/cmu-delphi/delphi-epidata/pull/1288 ### Changed - -- Modify the signatures for several methods: endpoint no longer needs to be specified in the params dict under the "source" key, but becomes a mandatory parameter for the method: - - `async_epidata(param_list, batch_size=50)` → `async_epidata(endpoint, param_list, batch_size=50)` +- Internally uses newer endpoint-specific URLs instead of an older compatibility alias URL. +- Upper limit on the number of rows returned per request has been increased to 1M (was 3650). +- Method `Epidata.async_epidata()` is now deprecated and will be removed in a future version. +- The dict object returned from data request methods will now always have an entry for the "`epidata`" key, potentially with an empty list as its value. previously, if there were no results for the request, the "`epidata`" entry was not present. +- Results from the `Epidata.covidcast()` method (or potentially the `covidcast` endpoint via the `async_epidata()` method) will now include the keys "`source`", "`geo_type`", and "`time_type`" (and their associated values). ## [4.1.10] - 2023-09-28 From fefc0889b97e7c97397e4cde73662d9caa8db94f Mon Sep 17 00:00:00 2001 From: george haff Date: Thu, 5 Oct 2023 14:58:31 -0400 Subject: [PATCH 51/55] changelog update (formatting) --- src/client/packaging/pypi/CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/client/packaging/pypi/CHANGELOG.md b/src/client/packaging/pypi/CHANGELOG.md index f138604ad..ee95128f7 100644 --- a/src/client/packaging/pypi/CHANGELOG.md +++ b/src/client/packaging/pypi/CHANGELOG.md @@ -5,7 +5,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/). ## [4.1.11] - 2023-10-12 -https://github.com/cmu-delphi/delphi-epidata/pull/1288 +### Includes +- https://github.com/cmu-delphi/delphi-epidata/pull/1288 ### Changed - Internally uses newer endpoint-specific URLs instead of an older compatibility alias URL. From b5273472774f286f7b2c2ec738dd50b018c36fb9 Mon Sep 17 00:00:00 2001 From: Rostyslav Zatserkovnyi Date: Thu, 5 Oct 2023 22:15:58 +0300 Subject: [PATCH 52/55] Remove public_route --- integrations/server/test_api_keys.py | 1 - 1 file changed, 1 deletion(-) diff --git a/integrations/server/test_api_keys.py b/integrations/server/test_api_keys.py index d5c742d3d..8cb50016a 100644 --- a/integrations/server/test_api_keys.py +++ b/integrations/server/test_api_keys.py @@ -18,7 +18,6 @@ def _make_request(self, endpoint: str, params: dict = {}, auth: tuple = None): def test_public_route(self): """Test public route""" - public_route = "http://delphi_web_epidata/epidata/version" status_codes = set() for _ in range(10): status_codes.add(self._make_request("version").status_code) From c97adadefb2b62d8818d30403c27f723befc6794 Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Wed, 11 Oct 2023 23:15:30 +0300 Subject: [PATCH 53/55] Removed unnecessary tests debugging options --- dev/local/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/local/Makefile b/dev/local/Makefile index 3264502ec..fb02668ee 100644 --- a/dev/local/Makefile +++ b/dev/local/Makefile @@ -171,7 +171,7 @@ test: --mount type=bind,source=$(CWD)repos/delphi/delphi-epidata/src,target=/usr/src/app/delphi/epidata,readonly \ --env "SQLALCHEMY_DATABASE_URI=$(sqlalchemy_uri)" \ --env "FLASK_SECRET=abc" \ - delphi_web_python python -m pytest -vvl --import-mode importlib $(pdb) $(test) | tee test_output_$(NOW).log + delphi_web_python python -m pytest --import-mode importlib $(pdb) $(test) | tee test_output_$(NOW).log .PHONY=r-test r-test: From 3dfdea6d5a33393c572ba9585edad0d50310a3f3 Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Thu, 12 Oct 2023 16:39:12 +0300 Subject: [PATCH 54/55] Removed comment --- integrations/server/test_covidcast_endpoints.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/integrations/server/test_covidcast_endpoints.py b/integrations/server/test_covidcast_endpoints.py index c45e7ef3b..650a18d1e 100644 --- a/integrations/server/test_covidcast_endpoints.py +++ b/integrations/server/test_covidcast_endpoints.py @@ -88,8 +88,6 @@ def test_basic_restricted_source(self): out = self._make_request(json=True, raise_for_status=True, params=params) self.assertEqual(len(out["epidata"]), len(rows)) - # Commented out this section as we want go get rid of legacy .php code - # ----------------------------------------- def test_compatibility(self): """Request at the /api.php endpoint.""" rows = [CovidcastTestRow.make_default_row(source="src", signal="sig", time_value=2020_04_01 + i, value=i) for i in range(10)] From 16776294cc0fd8523689acb03f8d157ae022a5ed Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Thu, 12 Oct 2023 16:50:38 +0300 Subject: [PATCH 55/55] Moved params to the top of the test case --- .../server/test_covidcast_endpoints.py | 23 +++++-------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/integrations/server/test_covidcast_endpoints.py b/integrations/server/test_covidcast_endpoints.py index 650a18d1e..497a65471 100644 --- a/integrations/server/test_covidcast_endpoints.py +++ b/integrations/server/test_covidcast_endpoints.py @@ -55,37 +55,26 @@ def test_basic_restricted_source(self): rows = [CovidcastTestRow.make_default_row(time_value=2020_04_01 + i, value=i, source="quidel") for i in range(10)] first = rows[0] self._insert_rows(rows) + params = { + "signal": first.signal_pair(), + "geo": first.geo_pair(), + "time": "day:*" + } with self.subTest("validation"): out = self._make_request(auth=self.epidata_client.auth, json=True, raise_for_status=True) self.assertEqual(out["result"], -1) with self.subTest("no_roles"): - params = { - "signal": first.signal_pair(), - "geo": first.geo_pair(), - "time": "day:*" - } out = self._make_request(auth=self.epidata_client.auth, json=True, raise_for_status=True, params=params) self.assertEqual(len(out["epidata"]), 0) with self.subTest("no_api_key"): - params = { - "signal": first.signal_pair(), - "geo": first.geo_pair(), - "time": "day:*" - } out = self._make_request(json=True, raise_for_status=True, params=params) self.assertEqual(len(out["epidata"]), 0) with self.subTest("quidel_role"): - params = { - "signal": first.signal_pair(), - "geo": first.geo_pair(), - "time": "day:*", - "auth": "quidel_key" - } - out = self._make_request(json=True, raise_for_status=True, params=params) + out = self._make_request(json=True, raise_for_status=True, params=params, auth=("epidata", "quidel_key")) self.assertEqual(len(out["epidata"]), len(rows)) def test_compatibility(self):