diff --git a/brasilio/settings.py b/brasilio/settings.py index 2feccd3b..f37282e4 100644 --- a/brasilio/settings.py +++ b/brasilio/settings.py @@ -1,4 +1,5 @@ from urllib.parse import urlparse +from pathlib import Path import environ import sentry_sdk @@ -35,6 +36,7 @@ "corsheaders", "django_extensions", "rest_framework", + "rest_framework.authtoken", "markdownx", "django_rq", # Project apps @@ -224,6 +226,7 @@ def get_neo4j_config_dict(neo4j_uri): # Covid19 import settings COVID_IMPORT_PERMISSION_PREFIX = "can_import_covid_state_" +SAMPLE_SPREADSHEETS_DATA_DIR = Path(BASE_DIR).joinpath("covid19", "tests", "data") # RockecChat config ROCKETCHAT_BASE_URL = env("ROCKETCHAT_BASE_URL") diff --git a/brazil_data/cities.py b/brazil_data/cities.py index f400811a..d20d260a 100644 --- a/brazil_data/cities.py +++ b/brazil_data/cities.py @@ -1,7 +1,6 @@ from collections import namedtuple from functools import lru_cache from itertools import groupby -from pathlib import Path import rows import rows.utils diff --git a/covid19/stats.py b/covid19/stats.py index b0be014d..5bc6085a 100644 --- a/covid19/stats.py +++ b/covid19/stats.py @@ -85,7 +85,7 @@ class Covid19Stats: "new_deaths_indeterminate_2019": (Sum, "new_deaths_indeterminate_2019"), "new_deaths_others_2019": (Sum, "new_deaths_others_2019"), "new_deaths_pneumonia_2019": (Sum, "new_deaths_pneumonia_2019"), - "new_deaths_respiratory_failure_2019": (Sum, "new_deaths_respiratory_failure_2019"), + "new_deaths_respiratory_failure_2019": (Sum, "new_deaths_respiratory_failure_2019",), "new_deaths_sars_2019": (Sum, "new_deaths_sars_2019"), "new_deaths_septicemia_2019": (Sum, "new_deaths_septicemia_2019"), "new_deaths_total_2019": (Sum, "new_deaths_total_2019"), @@ -337,7 +337,9 @@ def aggregate_state_data(self, select_columns, groupby_columns, state=None): return list(qs.order_by(*groupby_columns).values(*groupby_columns).annotate(**annotate_dict)) def aggregate_epiweek(self, data, group_key="epidemiological_week"): - row_key = lambda row: row[group_key] + def row_key(row): + return row[group_key] + result = [] data.sort(key=row_key) for epiweek, group in groupby(data, key=row_key): @@ -352,7 +354,7 @@ def aggregate_epiweek(self, data, group_key="epidemiological_week"): def historical_case_data_for_state_per_day(self, state): return self.aggregate_state_data( - groupby_columns=["date"], select_columns=self.graph_daily_cases_columns, state=state + groupby_columns=["date"], select_columns=self.graph_daily_cases_columns, state=state, ) def historical_case_data_for_state_per_epiweek(self, state): @@ -374,7 +376,7 @@ def aggregate_registry_data(self, select_columns, groupby_columns, state=None): def historical_registry_data_for_state_per_day(self, state=None): # If state = None, return data for Brazil return self.aggregate_registry_data( - groupby_columns=["date"], select_columns=self.graph_daily_registry_deaths_columns, state=state + groupby_columns=["date"], select_columns=self.graph_daily_registry_deaths_columns, state=state, ) def excess_deaths_registry_data_for_state_per_day(self, state=None): diff --git a/covid19/tests/test_google_data.py b/covid19/tests/test_google_data.py index 09e0f40e..d73d3965 100644 --- a/covid19/tests/test_google_data.py +++ b/covid19/tests/test_google_data.py @@ -9,7 +9,6 @@ class TestGoogleDataIntegration(TestCase): - @skip("This test won't work with Django's DummyCache, which is enabled for development") def test_cache_general_spreadsheet(self): cache.clear() diff --git a/covid19/tests/test_spreadsheet_api.py b/covid19/tests/test_spreadsheet_api.py new file mode 100644 index 00000000..f657c2f2 --- /dev/null +++ b/covid19/tests/test_spreadsheet_api.py @@ -0,0 +1,110 @@ +import shutil +from unittest.mock import patch, Mock, PropertyMock +from datetime import date, timedelta +from pathlib import Path +from model_bakery import baker + +from django.conf import settings +from django.urls import reverse +from django.contrib.auth import get_user_model +from django.contrib.auth.models import Group +from django.core.files.uploadedfile import SimpleUploadedFile +from rest_framework.test import APITestCase +from rest_framework.authtoken.models import Token + +from covid19.exceptions import SpreadsheetValidationErrors + + +class ImportSpreadsheetByDateAPIViewTests(APITestCase): + def setUp(self): + valid_csv = settings.SAMPLE_SPREADSHEETS_DATA_DIR / "sample-PR.csv" + assert valid_csv.exists() + + self.data = { + "date": date.today(), + "boletim_urls": "http://google.com\r\n\r http://brasil.io", + "boletim_notes": "notes", + } + + self.filename = f"sample.csv" + + self.file_data = self.gen_file(self.filename, valid_csv.read_bytes()) + self.data["file"] = self.file_data + self.setUp_user_credentials() + + def setUp_user_credentials(self): + self.user = baker.make(get_user_model()) + self.user.groups.add(Group.objects.get(name__endswith="Rio de Janeiro")) + self.user.groups.add(Group.objects.get(name__endswith="Paraná")) + + self.token = baker.make(Token, user=self.user) + self.headers = {"Authorization": f"Token {self.token.key}"} + self.client.credentials(HTTP_AUTHORIZATION=self.headers["Authorization"]) + self.client.force_login(user=self.user) + + def gen_file(self, name, content): + if isinstance(content, str): + content = str.encode(content) + return SimpleUploadedFile(name, content) + + def tearDown(self): + if Path(settings.MEDIA_ROOT).exists(): + shutil.rmtree(settings.MEDIA_ROOT) + + @patch( + "covid19.spreadsheet_validator.validate_historical_data", Mock(return_value=["warning 1", "warning 2"]), + ) + @patch("covid19.models.StateSpreadsheet.admin_url", new_callable=PropertyMock) + def test_import_data_from_a_valid_state_spreadsheet_request(self, mock_admin_url): + mock_admin_url.return_value = "https://brasil.io/covid19/dataset/PR" + + expected_status = 200 + expected_response = { + "warnings": ["warning 1", "warning 2"], + "detail_url": "https://brasil.io/covid19/dataset/PR", + } + + reverse_name = "covid19:statespreadsheet-list" + self.url = reverse(reverse_name, args=["PR"]) + + response = self.client.post(self.url, data=self.data, format="json") + + assert expected_status == response.status_code + assert expected_response == response.json() + + def test_403_login_required(self): + expected_status = 403 + reverse_name = "covid19:statespreadsheet-list" + + self.url = reverse(reverse_name, args=["PR"]) + + self.client.force_authenticate(user=None) + + response = self.client.post(self.url, data=self.data, format="json") + assert expected_status == response.status_code + + @patch("covid19.spreadsheet_validator.validate_historical_data", autospec=True) + def test_400_if_spreadsheet_error_on_import_data(self, mock_merge): + exception = SpreadsheetValidationErrors() + exception.new_error("error 1") + exception.new_error("error 2") + mock_merge.side_effect = exception + + expected_status = 400 + expected_exception_messages = ["error 1", "error 2"] + expected_response = {"errors": {"date": ["Campo não aceita datas futuras."]}} + + tomorrow = date.today() + timedelta(days=1) + tomorrow = tomorrow.isoformat() + self.data["date"] = tomorrow + + reverse_name = "covid19:statespreadsheet-list" + + self.url = reverse(reverse_name, args=["RJ"]) + + response = self.client.post(self.url, data=self.data, format="json") + + assert len(exception.error_messages) == 2 + assert expected_exception_messages == sorted(exception.error_messages) + assert expected_status == response.status_code + assert expected_response == response.json() diff --git a/covid19/urls.py b/covid19/urls.py index 405a2f1f..961c8851 100644 --- a/covid19/urls.py +++ b/covid19/urls.py @@ -12,6 +12,7 @@ path("historical/weekly/", views.historical_weekly, name="historical-weekly"), path("cities/geo/", views.cities_geojson, name="cities-geo"), path("states/geo/", views.states_geojson, name="states-geo"), - path("import-data//", views.import_spreadsheet_proxy, name="spreadsheet_proxy"), + path("import-data//", views.import_spreadsheet_proxy, name="spreadsheet_proxy",), + path("api/import-data//", views.state_spreadsheet_view_list, name="statespreadsheet-list",), path("/", views.dashboard, name="dashboard"), ] diff --git a/covid19/views.py b/covid19/views.py index f9363243..47d46de4 100644 --- a/covid19/views.py +++ b/covid19/views.py @@ -1,20 +1,28 @@ import datetime import random +import json +from django.core.files.uploadedfile import SimpleUploadedFile from django.http import Http404, HttpResponse, JsonResponse from django.shortcuts import render +from django.db import transaction + +from rest_framework import permissions +from rest_framework.decorators import api_view, permission_classes from brazil_data.cities import get_state_info from brazil_data.states import STATE_BY_ACRONYM, STATES from core.middlewares import disable_non_logged_user_cache from core.util import cached_http_get_json from covid19.exceptions import SpreadsheetValidationErrors +from covid19.forms import StateSpreadsheetForm from covid19.geo import city_geojson, state_geojson from covid19.models import StateSpreadsheet from covid19.spreadsheet import create_merged_state_spreadsheet from covid19.stats import Covid19Stats, max_values from covid19.util import row_to_column from covid19.epiweek import get_epiweek +from covid19.signals import new_spreadsheet_imported_signal stats = Covid19Stats() @@ -61,11 +69,7 @@ def clean_weekly_data(data, skip=0, diff_days=-14): now = datetime.datetime.now() today = datetime.date(now.year, now.month, now.day) _, until_epiweek = get_epiweek(today + datetime.timedelta(days=diff_days)) - return [ - row - for index, row in enumerate(data) - if index >= skip and row["epidemiological_week"] < until_epiweek - ] + return [row for index, row in enumerate(data) if index >= skip and row["epidemiological_week"] < until_epiweek] def historical_data(request, period): @@ -300,3 +304,35 @@ def status(request): data.append(table_entry) return render(request, "covid-status.html", {"import_data": data}) + + +@api_view(["POST"]) +@permission_classes([permissions.IsAuthenticated]) +def state_spreadsheet_view_list(request, *args, **kwargs): + def gen_file(name, content): + if isinstance(content, str): + content = str.encode(content) + return SimpleUploadedFile(name, content) + + if request.method == "POST": + data = json.loads(request.body) + + data["state"] = kwargs["state"] # sempre terá um state dado que ele está na URL + + state = data["state"] + date = data["date"] + filename = f"{state}-{date}-import.csv" + + file_data = {"file": gen_file(filename, "".join(data["file"]))} + + form = StateSpreadsheetForm(data, file_data, user=request.user) + + if form.is_valid(): + transaction.on_commit(lambda: new_spreadsheet_imported_signal.send(sender=self, spreadsheet=spreadsheet)) + + spreadsheet = form.save() + spreadsheet.refresh_from_db() + + return JsonResponse({"warnings": spreadsheet.warnings, "detail_url": spreadsheet.admin_url}) + + return JsonResponse({"errors": form.errors}, status=400)