Merge pull request #20 from beveradb/separate-deprivation-filters

Add support for separate deprivation filters

requirements.txt

@@ -17,4 +17,6 @@ geopy
 flask
 requests-cache
 backoff
-flask_sslify
+flask_sslify
+pyunpack
+patool

run_server.py

@@ -19,10 +19,10 @@
 logging.getLogger().setLevel(logging.DEBUG if app_debug else logging.INFO)
 
 # Download dataset files and pre-seeded API call / compute cache to reduce slug size
-preload_files('https://github.com/beveradb/home-area-helper/releases/download/v0.5/', [
+preload_files('https://github.com/beveradb/home-area-helper/releases/download/v0.6/', [
     {'dir': 'datasets/uk/', 'file': 'uk-wgs84-imd-shapefiles.zip'},
     {'dir': 'caches/', 'file': 'requests_cache.sqlite.zip'},
-    {'dir': 'caches/', 'file': 'static_cache.sqlite.zip'},
+    {'dir': 'caches/', 'file': 'static_cache.sqlite'},
 ])
 
 # Set up disk caching for HTTP requests (e.g. API calls), pre-seeded from above download file

src/imd_tools.py

@@ -10,46 +10,84 @@
 from src.multi_polygons import filter_multipoly_by_bounding_box
 from src.utils import timeit
 
+rank_type_properties = {
+    'deprivation': {'england': 'IMDDec0', 'scotland': 'Decile'},
+    'income': {'england': 'IncDec', 'scotland': 'IncRank'},
+    'crime': {'england': 'CriDec', 'scotland': 'CrimeRank'},
+    'health': {'england': 'HDDDec', 'scotland': 'HlthRank'},
+    'education': {'england': 'EduDec', 'scotland': 'EduRank'},
+    'services': {'england': 'GBDec', 'scotland': 'GAccRank'},
+    'environment': {'england': 'EnvDec', 'scotland': 'HouseRank'}
+}
+
 
 # This file assumes all input shapefiles are already in WGS84 projection.
 # If we wish to use a new source dataset, to avoid needing to do any reprojection in this code, it is much more
 # efficient to reproject the source dataset first. For example, to reproject a UK shapefile to WGS84, run:
 # ogr2ogr -f "ESRI Shapefile" output-wgs84.shp input-ukproj.shp -s_srs EPSG:27700 -t_srs EPSG:4326
 
 @timeit
-def get_polygon_for_least_deprived_zones_england(minimum_deprivation_rank):
+def get_polygon_for_least_deprived_zones_england(rank_type, min_rank_value):
     # Metadata as per https://www.arcgis.com/home/item.html?id=5e1c399d787e48c0902e5fe4fc1ccfe3
     filtered_zones_polygons = []
     with fiona.open('datasets/uk/IMD_2019_WGS.shp') as allZones:
         # logging.debug("Total IMD data zones: " + str(len(allZones)))
 
         for singleZone in allZones:
-            if singleZone['properties']['IMDDec0'] >= minimum_deprivation_rank:
+            if singleZone['properties'][
+                rank_type_properties[rank_type]['england']
+            ] >= min_rank_value:
                 filtered_zones_polygons.append(shape(singleZone['geometry']))
 
     return MultiPolygon(filtered_zones_polygons)
 
 
 @timeit
-def get_polygon_for_least_deprived_zones_scotland(minimum_deprivation_rank):
+def get_polygon_for_least_deprived_zones_scotland(rank_type, min_rank_value):
     filtered_zones_polygons = []
     with fiona.open('datasets/uk/SG_SIMD_2016_WGS.shp') as allZones:
         # logging.debug("Total SIMD data zones: " + str(len(allZones)))
 
         for singleZone in allZones:
-            if singleZone['properties']['Decile'] >= minimum_deprivation_rank:
+            # All of the other comparison properties in the Scotland dataset are
+            # actually Ranks, not Deciles - the shapefile doesn't contain Decile values for all
+            # the other specific values, grrrr. So, this is a manual mapping of Decile -> Rank,
+            # as per the SIMD16-Rank-Decile-Mapping-00504608.xlsx dataset info spreadsheet
+            if rank_type != 'deprivation':
+                if min_rank_value == 2:
+                    min_rank_value = 698
+                if min_rank_value == 3:
+                    min_rank_value = 1396
+                if min_rank_value == 4:
+                    min_rank_value = 2093
+                if min_rank_value == 5:
+                    min_rank_value = 2791
+                if min_rank_value == 6:
+                    min_rank_value = 3489
+                if min_rank_value == 7:
+                    min_rank_value = 4186
+                if min_rank_value == 8:
+                    min_rank_value = 4884
+                if min_rank_value == 9:
+                    min_rank_value = 5581
+                if min_rank_value == 10:
+                    min_rank_value = 6279
+
+            if singleZone['properties'][
+                rank_type_properties[rank_type]['scotland']
+            ] >= min_rank_value:
                 filtered_zones_polygons.append(shape(singleZone['geometry']))
 
     return MultiPolygon(filtered_zones_polygons)
 
 
 @timeit
 @static_cache.cached()
-def get_polygon_for_least_deprived_zones_uk(minimum_deprivation_rank):
+def get_polygon_for_least_deprived_zones_uk(rank_type, min_rank_value):
     # Hah, guess you aren't a Scottish Independence voter ;)
     return MultiPolygon(itertools.chain(
-        get_polygon_for_least_deprived_zones_scotland(minimum_deprivation_rank),
-        get_polygon_for_least_deprived_zones_england(minimum_deprivation_rank)
+        get_polygon_for_least_deprived_zones_scotland(rank_type, min_rank_value),
+        get_polygon_for_least_deprived_zones_england(rank_type, min_rank_value)
     ))
 
 
@@ -75,16 +113,16 @@ def reproject_single_polygon(single_polygon, proj_partial):
 
 
 @timeit
-def get_world_min_deprivation_rank_wgs84_multipoly(minimum_deprivation_rank):
-    uk_multipoly_wgs84 = get_polygon_for_least_deprived_zones_uk(minimum_deprivation_rank)
+def get_world_min_deprivation_rank_wgs84_multipoly(rank_type, min_rank_value):
+    uk_multipoly_wgs84 = get_polygon_for_least_deprived_zones_uk(rank_type, min_rank_value)
 
     return uk_multipoly_wgs84
 
 
 @timeit
 @transient_cache.cached()
-def get_bounded_min_rank_multipoly(input_bounds, min_deprivation_rank):
-    min_rank_poly = get_world_min_deprivation_rank_wgs84_multipoly(min_deprivation_rank)
+def get_bounded_min_rank_multipoly(input_bounds, rank_type, min_rank_value):
+    min_rank_poly = get_world_min_deprivation_rank_wgs84_multipoly(rank_type, min_rank_value)
 
     input_multipoly_bounds = Polygon.from_bounds(*input_bounds).buffer(0.001)
 
@@ -93,9 +131,12 @@ def get_bounded_min_rank_multipoly(input_bounds, min_deprivation_rank):
 
 
 @timeit
-def intersect_multipoly_by_min_rank(input_multipoly, min_deprivation_rank):
-    min_rank_poly = get_bounded_min_rank_multipoly(input_multipoly.bounds, min_deprivation_rank)
+def intersect_multipoly_by_min_rank(input_multipoly, rank_type, min_rank_value):
+    min_rank_poly = get_bounded_min_rank_multipoly(input_multipoly.bounds, rank_type, min_rank_value)
 
-    input_multipoly = min_rank_poly.intersection(input_multipoly)
+    if isinstance(min_rank_poly, list):
+        logging.warning("min_rank_poly is list, not attempting to intersect it as presumably this is empty?")
+    else:
+        input_multipoly = min_rank_poly.intersection(input_multipoly)
 
     return input_multipoly

src/target_area.py

@@ -20,6 +20,12 @@ def get_target_area_polygons(
         max_train_time_mins: int,
         max_driving_time_mins: int,
         min_deprivation_rank: int,
+        min_income_rank: int,
+        min_crime_rank: int,
+        min_health_rank: int,
+        min_education_rank: int,
+        min_services_rank: int,
+        min_environment_rank: int,
         min_area_miles: float,
         max_radius_miles: float,
         simplify_factor: float,
@@ -87,18 +93,29 @@ def get_target_area_polygons(
         #         'polygon': combined_transport_poly
         #     }
 
-        if min_deprivation_rank > 0:
-            imd_multipoly = get_bounded_min_rank_multipoly(result_polygons.bounds, min_deprivation_rank)
-            imd_multipoly_joined = join_multi_to_single_poly(imd_multipoly)
-            return_object['deprivation'] = {
-                'label': 'Deprivation Rank >= ' + str(min_deprivation_rank),
-                'polygon': imd_multipoly_joined
-            }
+        deprivation_filter_values = {
+            'deprivation': {'label': 'Deprivation Rank', 'value': min_deprivation_rank},
+            'income': {'label': 'Income Rank', 'value': min_income_rank},
+            'crime': {'label': 'Crime Rank', 'value': min_crime_rank},
+            'health': {'label': 'Health Rank', 'value': min_health_rank},
+            'education': {'label': 'Education Rank', 'value': min_education_rank},
+            'services': {'label': 'Access to Services Rank', 'value': min_services_rank},
+            'environment': {'label': 'Living Environment Rank', 'value': min_environment_rank},
+        }
 
-            result_polygons = intersect_multipoly_by_min_rank(result_polygons, min_deprivation_rank)
+        for filter_name, filter_obj in deprivation_filter_values.items():
+            if filter_obj['value'] > 0:
+                imd_multipoly = get_bounded_min_rank_multipoly(result_polygons.bounds, filter_name, filter_obj['value'])
+                imd_multipoly_joined = join_multi_to_single_poly(imd_multipoly)
+                return_object[filter_name] = {
+                    'label': filter_obj['label'] + ' >= ' + str(filter_obj['value']),
+                    'polygon': imd_multipoly_joined
+                }
 
-            result_polygons_length = 1 if not hasattr(result_polygons, 'geoms') else len(result_polygons.geoms)
-            logging.info("Total result_polygons after min_deprivation_rank filter: " + str(result_polygons_length))
+                result_polygons = intersect_multipoly_by_min_rank(result_polygons, filter_name, filter_obj['value'])
+
+                result_polygons_length = 1 if not hasattr(result_polygons, 'geoms') else len(result_polygons.geoms)
+                logging.info("Total result_polygons after " + filter_name + " filter: " + str(result_polygons_length))
 
         if result_polygons_length > 0:
             if min_area_miles > 0:
@@ -172,6 +189,12 @@ def get_target_areas_polygons_json(targets_params: list):
         target_results = get_target_area_polygons(
             target_location_address=str(params['target']),
             min_deprivation_rank=int(params['deprivation']),
+            min_income_rank=int(params['income']),
+            min_crime_rank=int(params['crime']),
+            min_health_rank=int(params['health']),
+            min_education_rank=int(params['education']),
+            min_services_rank=int(params['services']),
+            min_environment_rank=int(params['environment']),
             max_walking_time_mins=int(params['walking']),
             max_cycling_time_mins=int(params['cycling']),
             max_bus_time_mins=int(params['bus']),

src/utils.py

@@ -2,7 +2,7 @@
 import os
 import sys
 import time
-import zipfile
+from pyunpack import Archive
 
 methods_timings_cumulative = {}
 
@@ -76,9 +76,9 @@ def preload_files(url_root, files_to_check):
             if not os.path.isfile(fetch_filepath):
                 raise Exception("Preload file download failed")
 
-            if fetch_filepath.endswith('.zip'):
-                logging.info("Preload file ends with .zip, unzipping")
-                with zipfile.ZipFile(fetch_filepath, "r") as zip_ref:
-                    zip_ref.extractall(single_check['dir'])
+            if fetch_filepath.endswith('.zip') or fetch_filepath.endswith('.7z'):
+                logging.info("Preload file ends with .zip or .7z, extracting")
+                Archive(fetch_filepath).extractall(single_check['dir'])
+
         else:
             logging.info("Preload file already exists: " + fetch_filepath)

static/main.js

@@ -12,7 +12,7 @@ function map_loaded(map) {
 
 $(function () {
     $("#addTargetButton").click(function (e) {
-        add_new_target_to_accordion();
+        add_new_target_to_accordion(true);
         return false;
     });
 
@@ -80,6 +80,12 @@ function show_saved_searches_modal() {
         train: "Train",
         driving: "Drive",
         deprivation: "Deprivation",
+        income: "Income",
+        crime: "Crime",
+        health: "Health",
+        education: "Education",
+        services: "Access to services",
+        environment: "Living Environment",
         radius: "Max. Radius",
         minarea: "Min. Area",
         simplify: "Simplify",
@@ -306,7 +312,7 @@ function load_saved_search(search_object) {
     $('#targetsAccordion .targetCard').remove();
 
     search_targets_array.forEach(function (target_search, target_index) {
-        let new_target_card = add_new_target_to_accordion();
+        let new_target_card = add_new_target_to_accordion(false);
 
         new_target_card.find(".targetAddressInput").val(target_search['target']).focus();
         if (target_search['walking']) new_target_card.find(".maxWalkingTimeInput").val(target_search['walking']);
@@ -316,6 +322,12 @@ function load_saved_search(search_object) {
         if (target_search['train']) new_target_card.find(".maxTrainTimeInput").val(target_search['train']);
         if (target_search['driving']) new_target_card.find(".maxDrivingTimeInput").val(target_search['driving']);
         if (target_search['deprivation']) new_target_card.find(".minIMDInput").val(target_search['deprivation']);
+        if (target_search['income']) new_target_card.find(".incomeRankInput").val(target_search['income']);
+        if (target_search['crime']) new_target_card.find(".crimeRankInput").val(target_search['crime']);
+        if (target_search['health']) new_target_card.find(".healthRankInput").val(target_search['health']);
+        if (target_search['education']) new_target_card.find(".educationRankInput").val(target_search['education']);
+        if (target_search['services']) new_target_card.find(".servicesRankInput").val(target_search['services']);
+        if (target_search['environment']) new_target_card.find(".environmentRankInput").val(target_search['environment']);
         if (target_search['radius'] > 0) new_target_card.find(".maxRadiusInput").val(target_search['radius']);
         if (target_search['minarea'] > 0) new_target_card.find(".minAreaRadiusInput").val(target_search['minarea']);
         if (target_search['simplify'] > 0) new_target_card.find(".simplifyFactorInput").val(target_search['simplify']);
@@ -377,7 +389,7 @@ function validate_and_submit_request() {
     );
 }
 
-function add_new_target_to_accordion() {
+function add_new_target_to_accordion(showTargetCard) {
     let targetsAccordion = $('#targetsAccordion');
     let newTargetCard = $('#targetCardTemplate').clone();
 
@@ -397,7 +409,7 @@ function add_new_target_to_accordion() {
 
     let newCollapseBody = newTargetCard.find('div.collapse');
     newCollapseBody.attr('id', "targetCollapse" + newTargetKey);
-    newCollapseBody.addClass('show');
+    if (showTargetCard) newCollapseBody.addClass('show');
 
     newTargetCard.find('input').keypress(function (e) {
         if (e.which === 13) {
@@ -421,6 +433,9 @@ function add_new_target_to_accordion() {
         newTargetCard.remove();
     });
 
+    newTargetCard.find('.adjustShapeToggleButton').click();
+    newTargetCard.find('.specificDeprivationsToggleButton').click();
+
     return newTargetCard;
 }
 
@@ -460,6 +475,12 @@ function build_targets_array() {
             train: single_card.find(".maxTrainTimeInput").val(),
             driving: single_card.find(".maxDrivingTimeInput").val(),
             deprivation: single_card.find(".minIMDInput").val(),
+            income: single_card.find(".incomeRankInput").val(),
+            crime: single_card.find(".crimeRankInput").val(),
+            health: single_card.find(".healthRankInput").val(),
+            education: single_card.find(".educationRankInput").val(),
+            services: single_card.find(".servicesRankInput").val(),
+            environment: single_card.find(".environmentRankInput").val(),
             radius: single_card.find(".maxRadiusInput").val(),
             minarea: single_card.find(".minAreaRadiusInput").val(),
             simplify: single_card.find(".simplifyFactorInput").val(),