Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Signal Documentation Coverage Endpoint #1584

Merged
merged 30 commits into from
Feb 3, 2025
Merged
Changes from 1 commit
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
b0934b5
first pass at coverage endpoint
nolangormley Jan 27, 2025
1a06e2e
added SQL to schemas
nolangormley Jan 27, 2025
55d13cb
added load table and function to recompute data
nolangormley Jan 27, 2025
16fe7b7
modified endpoint to use geo_sets, allowing multiple geos in filter
nolangormley Jan 27, 2025
85a972e
removed indexes from load table, fixed index declaration on main table
nolangormley Jan 27, 2025
c2baf02
added CLI and tests
nolangormley Jan 29, 2025
233ccd2
fixed sonar suggestions
nolangormley Jan 29, 2025
65c040a
more sonar issues
nolangormley Jan 29, 2025
3c404b2
ignore sonar test
aysim319 Jan 30, 2025
a5555a5
removed unessesary index rebuild, fixed indexes in ddl sql file
nolangormley Jan 30, 2025
31200bd
Merge branch 'sig_doc_coverage' of github.com:cmu-delphi/delphi-epida…
nolangormley Jan 30, 2025
26c6450
updated SQL to use transaction instead of load table
nolangormley Jan 31, 2025
1f26061
updated integration test, removed SQL alias
nolangormley Jan 31, 2025
a0f631f
adding alias back in
nolangormley Feb 1, 2025
8da0326
Update src/maintenance/coverage_crossref_updater.py
nolangormley Feb 1, 2025
8a46b84
Update src/acquisition/covidcast/database.py
nolangormley Feb 1, 2025
f0e60f0
Update src/acquisition/covidcast/database.py
nolangormley Feb 1, 2025
d9f9260
Update src/acquisition/covidcast/database.py
nolangormley Feb 1, 2025
38b9e7b
Update src/maintenance/coverage_crossref_updater.py
nolangormley Feb 1, 2025
59d0ec8
Update integrations/acquisition/covidcast/test_coverage_crossref_upda…
nolangormley Feb 1, 2025
0bf61b7
Update src/server/endpoints/covidcast.py
nolangormley Feb 3, 2025
8a59022
Update integrations/acquisition/covidcast/test_coverage_crossref_upda…
nolangormley Feb 3, 2025
d8c5efa
Merge branch 'sig_doc_coverage' of github.com:cmu-delphi/delphi-epida…
nolangormley Feb 3, 2025
a458730
added test coverage and fixed groupby
nolangormley Feb 3, 2025
62b7475
Update src/maintenance/coverage_crossref_updater.py
nolangormley Feb 3, 2025
269c108
Update src/maintenance/coverage_crossref_updater.py
nolangormley Feb 3, 2025
b0be91f
Update src/maintenance/coverage_crossref_updater.py
nolangormley Feb 3, 2025
3b137d8
Update integrations/acquisition/covidcast/test_coverage_crossref_upda…
nolangormley Feb 3, 2025
e50624c
Merge branch 'sig_doc_coverage' of github.com:cmu-delphi/delphi-epida…
nolangormley Feb 3, 2025
1604845
added some test cases
nolangormley Feb 3, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
added load table and function to recompute data
nolangormley committed Jan 27, 2025
commit 55d13cb5a9f1d1ecdf3e4aebd6b8f05fce668749
42 changes: 42 additions & 0 deletions src/acquisition/covidcast/database.py
Original file line number Diff line number Diff line change
@@ -561,3 +561,45 @@ def retrieve_covidcast_meta_cache(self):
for entry in cache:
cache_hash[(entry['data_source'], entry['signal'], entry['time_type'], entry['geo_type'])] = entry
return cache_hash

def compute_coverage_crossref(self):
"""Compute coverage_crossref table."""
nolangormley marked this conversation as resolved.
Show resolved Hide resolved
logger = get_structured_logger("compute_coverage_crossref")

coverage_crossref_load_delete_sql = f'''
DELETE FROM coverage_crossref_load WHERE 1;
'''

coverage_crossref_compute_sql = f'''
INSERT INTO coverage_crossref_load
SELECT
el.signal_key_id,
el.geo_key_id,
MIN(el.time_value) as min_time_value,
MAX(el.time_value) as max_time_value
FROM epimetric_latest el
GROUP BY el.signal_key_id, el.geo_key_id;
'''

coverage_crossref_delete_sql = f'''
DELETE FROM coverage_crossref WHERE 1;
'''

coverage_crossref_update_sql = f'''
INSERT INTO coverage_crossref
SELECT * FROM coverage_crossref_load;
'''

self._cursor.execute(coverage_crossref_load_delete_sql)
logger.info(f"coverage_crossref_load_delete_sql:{self._cursor.rowcount}")

self._cursor.execute(coverage_crossref_compute_sql)
logger.info(f"coverage_crossref_compute_sql:{self._cursor.rowcount}")

self._cursor.execute(coverage_crossref_delete_sql)
logger.info(f"coverage_crossref_delete_sql:{self._cursor.rowcount}")

self._cursor.execute(coverage_crossref_update_sql)
logger.info(f"coverage_crossref_update_sql:{self._cursor.rowcount}")

return self._cursor.rowcount
9 changes: 9 additions & 0 deletions src/ddl/v4_schema.sql
Original file line number Diff line number Diff line change
@@ -165,6 +165,15 @@ CREATE TABLE `covidcast_meta_cache` (
) ENGINE=InnoDB;
INSERT INTO covidcast_meta_cache VALUES (0, '[]');

CREATE TABLE `coverage_crossref_load` (
`signal_key_id` bigint NOT NULL,
`geo_key_id` bigint NOT NULL,
`min_time_value` int NOT NULL,
`max_time_value` int NOT NULL,
UNIQUE INDEX coverage_crossref_signal_key_id ON coverage_crossref (signal_key_id),
UNIQUE INDEX coverage_crossref_geo_key_id ON coverage_crossref (geo_key_id)
) ENGINE=InnoDB;

CREATE TABLE `coverage_crossref` (
`signal_key_id` bigint NOT NULL,
`geo_key_id` bigint NOT NULL,
2 changes: 1 addition & 1 deletion src/ddl/v4_schema_aliases.sql
Original file line number Diff line number Diff line change
@@ -8,4 +8,4 @@
CREATE VIEW `epidata`.`epimetric_full_v` AS SELECT * FROM `covid`.`epimetric_full_v`;
CREATE VIEW `epidata`.`epimetric_latest_v` AS SELECT * FROM `covid`.`epimetric_latest_v`;
CREATE VIEW `epidata`.`covidcast_meta_cache` AS SELECT * FROM `covid`.`covidcast_meta_cache`;
CREATE VIEW `epidata`.`coverage_crossref_v` AS SELECT * FROM `covid`.`coverage_crossref_v`;
CREATE VIEW `epidata`.`coverage_crossref_v` AS SELECT * FROM `covid`.`coverage_crossref_v`;