Merge pull request #466 from turicas/feature/organize-fields-metadata

[WIP] Organiza metadados de campos

core/management/commands/export_fields_to_csv.py

@@ -36,6 +36,7 @@ def handle(self, *args, **kwargs):
                         {
                             "dataset_slug": dataset_slug,
                             "description": field.description,
+                            "searchable": field.searchable,
                             "frontend_filter": field.frontend_filter,
                             "has_choices": field.has_choices,
                             "link_template": field.link_template,

core/management/commands/update_data.py

@@ -64,8 +64,8 @@ def str_to_list(data):
 
 def table_update_data(row):
     row["ordering"] = str_to_list(row["ordering"])
-    row["filtering"] = str_to_list(row["filtering"])
-    row["search"] = str_to_list(row["search"])
+    row["filtering_fields"] = str_to_list(row["filtering"])
+    row["search_fields"] = str_to_list(row["search"])
     return {"dataset": row["dataset"], "version": row["version"], "name": row["name"], "defaults": row}
 
 
@@ -165,4 +165,10 @@ def handle(self, *args, **kwargs):
                 data_table.save()
             else:  # Same table as before, so no need to update
                 total_skipped += 1
+
+            if table.filtering_fields:  # avoid None
+                table.fields.filter(name__in=table.filtering_fields).update(frontend_filter=True)
+            if table.search_fields:
+                table.fields.filter(name__in=table.search_fields__isnull).update(searchable=True)
+
         print(" created: {}, updated: {}, skipped: {}.".format(total_created, total_updated, total_skipped))

core/management/commands/update_fields_from_csv.py

@@ -18,7 +18,7 @@ def handle(self, *args, **kwargs):
         expected_field_names = set(
             "dataset_slug description frontend_filter has_choices "
             "link_template order null name options obfuscate show "
-            "show_on_frontend table_name title type version_name".split()
+            "show_on_frontend table_name title type version_name searchable".split()
         )
         if set(table.field_names) != expected_field_names:
             raise ValueError("Field names didn't match")

core/migrations/0023_auto_20201006_1435.py

@@ -0,0 +1,14 @@
+# Generated by Django 3.1.1 on 2020-10-06 17:35
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("core", "0022_auto_20200918_1805"),
+    ]
+
+    operations = [
+        migrations.RenameField(model_name="table", old_name="filtering", new_name="filtering_fields",),
+    ]

core/migrations/0024_auto_20201006_1439.py

@@ -0,0 +1,31 @@
+# Generated by Django 3.1.1 on 2020-10-06 17:39
+
+from django.db import migrations
+
+
+def ensure_filtering_fields_config(apps, schema_editor):
+    Table = apps.get_model("core.Table")
+
+    for table in Table.objects.all():
+        fields = table.field_set.all()
+
+        fields_maps = {f.name: f for f in fields}
+        for fieldname in table.filtering_fields or []:
+            field = fields.get(name=fieldname)
+            if not field.frontend_filter:
+                field.frontend_filter = True
+                field.save()
+                print(f"{table.dataset.slug}.{table.name}.{fieldname} atualizado como frontend_filter")
+
+
+def rollback(*args, **kwargs):
+    pass
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("core", "0023_auto_20201006_1435"),
+    ]
+
+    operations = [migrations.RunPython(ensure_filtering_fields_config, rollback)]

core/migrations/0025_auto_20201007_1702.py

@@ -0,0 +1,16 @@
+# Generated by Django 3.1.1 on 2020-10-07 20:02
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("core", "0024_auto_20201006_1439"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="field", name="searchable", field=models.BooleanField(blank=True, default=False),
+        ),
+    ]

core/migrations/0026_auto_20201007_1704.py

@@ -0,0 +1,14 @@
+# Generated by Django 3.1.1 on 2020-10-07 20:04
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("core", "0025_auto_20201007_1702"),
+    ]
+
+    operations = [
+        migrations.RenameField(model_name="table", old_name="search", new_name="search_fields",),
+    ]

core/migrations/0027_auto_20201007_1707.py

@@ -0,0 +1,19 @@
+# Generated by Django 3.1.1 on 2020-10-07 20:07
+
+from django.db import migrations
+
+
+def update_searchable_fields_flags(apps, schema_editor):
+    Table = apps.get_model("core", "Table")
+
+    for table in Table.objects.exclude(search_fields__isnull=True):
+        table.field_set.filter(name__in=table.search_fields).update(searchable=True)
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("core", "0026_auto_20201007_1704"),
+    ]
+
+    operations = [migrations.RunPython(update_searchable_fields_flags)]

core/models.py

@@ -295,8 +295,8 @@ class Table(models.Model):
     name = models.CharField(max_length=255, null=False, blank=False)
     options = models.JSONField(null=True, blank=True)
     ordering = ArrayField(models.CharField(max_length=63), null=False, blank=False)
-    filtering = ArrayField(models.CharField(max_length=63), null=True, blank=True)
-    search = ArrayField(models.CharField(max_length=63), null=True, blank=True)
+    filtering_fields = ArrayField(models.CharField(max_length=63), null=True, blank=True)
+    search_fields = ArrayField(models.CharField(max_length=63), null=True, blank=True)
     version = models.ForeignKey(Version, on_delete=models.CASCADE, null=False, blank=False)
     import_date = models.DateTimeField(null=True, blank=True)
     description = MarkdownxField(null=True, blank=True)
@@ -306,6 +306,10 @@ class Table(models.Model):
     def __str__(self):
         return "{}.{}.{}".format(self.dataset.slug, self.version.name, self.name)
 
+    @property
+    def filtering(self):
+        return [f.name for f in self.fields.frontend_filters()]
+
     @property
     def collect_date(self):
         return self.version.collected_at
@@ -322,6 +326,10 @@ def db_table(self):
     def fields(self):
         return self.field_set.all()
 
+    @property
+    def search(self):
+        return [f.name for f in self.fields.searchable()]
+
     @property
     def enabled(self):
         return not self.hidden
@@ -470,7 +478,13 @@ def for_table(self, table):
         return self.filter(table=table)
 
     def choiceables(self):
-        return self.filter(has_choices=True, frontend_filter=True)
+        return self.frontend_filters().filter(has_choices=True)
+
+    def frontend_filters(self):
+        return self.filter(frontend_filter=True)
+
+    def searchable(self):
+        return self.filter(searchable=True)
 
 
 class Field(models.Model):
@@ -482,6 +496,7 @@ class Field(models.Model):
     dataset = models.ForeignKey(Dataset, on_delete=models.CASCADE, null=False, blank=False)
     description = models.TextField(null=True, blank=True)
     frontend_filter = models.BooleanField(null=False, blank=True, default=False)
+    searchable = models.BooleanField(null=False, blank=True, default=False)
     has_choices = models.BooleanField(null=False, blank=True, default=False)
     link_template = models.TextField(max_length=2000, null=True, blank=True)
     order = models.PositiveIntegerField(null=False, blank=False)

core/tests/test_forms.py

@@ -14,7 +14,7 @@ class DynamicModelFormTests(BaseTestCaseWithSampleDataset):
     ]
 
     def setUp(self):
-        self.table.filtering = []
+        self.table.field_set.all().update(frontend_filter=False)
         self.table.save()
 
     def get_model_field(self, name):
@@ -26,8 +26,7 @@ def test_table_without_filters_gets_empty_form(self):
         assert 0 == len(form.fields)
 
     def test_generate_form_based_in_table_filtering(self):
-        self.table.filtering = ["uf", "city"]
-        self.table.save()
+        self.table.field_set.filter(name__in=["uf", "city"]).update(frontend_filter=True)
 
         DynamicFormClasss = get_table_dynamic_form(self.table, cache=False)
         form = DynamicFormClasss()
@@ -38,8 +37,7 @@ def test_generate_form_based_in_table_filtering(self):
         assert isinstance(form.fields["city"], type(self.get_model_field("city").formfield()))
 
     def test_filter_form_does_not_invalidate_if_no_data(self):
-        self.table.filtering = ["uf", "city"]
-        self.table.save()
+        self.table.field_set.filter(name__in=["uf", "city"]).update(frontend_filter=True)
 
         DynamicFormClasss = get_table_dynamic_form(self.table, cache=False)
         form = DynamicFormClasss(data={})
@@ -48,8 +46,7 @@ def test_filter_form_does_not_invalidate_if_no_data(self):
         assert {"uf": "", "city": ""} == form.cleaned_data
 
     def test_validate_form_against_field_choices(self):
-        self.table.filtering = ["uf", "city"]
-        self.table.save()
+        self.table.field_set.filter(name__in=["uf", "city"]).update(frontend_filter=True)
         uf_field = self.table.get_field("uf")
         uf_field.has_choices = True
         uf_field.choices = {"data": ["RJ", "SP", "MG"]}
@@ -76,8 +73,7 @@ def test_validate_form_against_field_choices(self):
         assert {"city": "Rio de Janeiro"} == form.cleaned_data
 
     def test_choice_failback_to_default_type_if_has_choice_field_but_no_data(self):
-        self.table.filtering = ["uf", "city"]
-        self.table.save()
+        self.table.field_set.filter(name__in=["uf", "city"]).update(frontend_filter=True)
         uf_field = self.table.get_field("uf")
         uf_field.has_choices = True
         uf_field.save()

core/tests/test_models.py

@@ -9,7 +9,7 @@
 from rows import fields
 
 from core.dynamic_models import DynamicModelMixin
-from core.models import Dataset, DataTable, Table, TableFile, Version
+from core.models import Dataset, DataTable, Table, TableFile, Version, Field
 from utils.file_info import human_readable_size
 
 
@@ -79,6 +79,36 @@ def test_table_default_manager_excludes_hidden_tables(self):
         assert hidden_table in tables
 
 
+class FieldModelTests(TestCase):
+    def test_searchable_queryset(self):
+        field = baker.make(Field, searchable=True)
+        baker.make(Field, searchable=False)
+
+        qs = Field.objects.searchable()
+
+        assert field in qs
+        assert 1 == qs.count()
+
+    def test_frontend_filters(self):
+        field = baker.make(Field, frontend_filter=True)
+        baker.make(Field, frontend_filter=False)
+
+        qs = Field.objects.frontend_filters()
+
+        assert field in qs
+        assert 1 == qs.count()
+
+    def test_choiceables_queryset(self):
+        field = baker.make(Field, frontend_filter=True, has_choices=True)
+        baker.make(Field, frontend_filter=True, has_choices=False)
+        baker.make(Field, frontend_filter=False)
+
+        qs = Field.objects.choiceables()
+
+        assert field in qs
+        assert 1 == qs.count()
+
+
 class DataTableModelTests(TestCase):
     def setUp(self):
         self.table = baker.make(Table, dataset__slug="ds-slug", name="table_name")