Merge pull request #6448 from OCHA-DAP/dev

dev into prod

ckanext-hdx_package/ckanext/hdx_package/actions/get.py

@@ -1037,7 +1037,8 @@ def hdx_recommend_tags(context, data_dict):
     tag_recommender = TagRecommender(data_dict.get('title'), data_dict.get('organization'))
     recommended_tags = tag_recommender.find_recommended_tags()
     approved_tags = get_action('cached_approved_tags_list')(context, {})
-    filtered_tags = [tag for tag in recommended_tags if tag['name'] in approved_tags]
+    filtered_tags = [tag for tag in recommended_tags if
+                     tag['name'] in approved_tags and not tag['name'].startswith('crisis-')]
     return filtered_tags
 
 

ckanext-hdx_package/ckanext/hdx_package/helpers/custom_validator.py

@@ -738,21 +738,108 @@ def hdx_dataseries_title_validator(value, context):
     return value
 
 
-def hdx_tag_name_approved_validator(key, data, errors, context):
+def hdx_tag_name_approved_validator(key: FlattenKey, data: FlattenDataDict, errors: FlattenErrorDict,
+                                    context: Context) -> Any:
     user = context.get('user')
-    ignore_auth = context.get('ignore_auth')
-    allowed_to_add_crisis_tags = ignore_auth or (user and authz.is_sysadmin(user))
+    allowed_to_add_crisis_tags = user and authz.is_sysadmin(user)
 
     tag_name = data.get(key)
 
     approved_tags = get_action('cached_approved_tags_list')(context, {})
 
+    pkg_id = data.get(('id',))
+    if pkg_id:
+        old_tag_names = _extract_old_tag_names(context, pkg_id)
+    else:
+        old_tag_names = []
+
+    if key not in errors:
+        errors[key] = []
+
     if tag_name not in approved_tags:
         approved_tags_url = 'https://data.humdata.org/rdr/spreadsheets/approved-tags'
         errors[key].append("Tag name '{}' is not in the approved list of tags. Check the list at: {}".format(tag_name, approved_tags_url))
     # Only sysadmins are allowed to use tags starting with "crisis-"
-    if tag_name.startswith('crisis-') and not allowed_to_add_crisis_tags:
-        errors[key].append("Tag name '{}' can only be added by sysadmins".format(tag_name))
+    if tag_name.startswith('crisis-'):
+        if not allowed_to_add_crisis_tags and tag_name not in old_tag_names:
+            errors[key].append("Tag name '{}' can only be added by sysadmins".format(tag_name))
+
+
+def hdx_tag_string_approved_validator(key: FlattenKey, data: FlattenDataDict, errors: FlattenErrorDict,
+                                      context: Context) -> Any:
+    index = 0
+    while ('tags', index, 'name') in data:
+        key = ('tags', index, 'name')
+        hdx_tag_name_approved_validator(key, data, errors, context)
+        index += 1
+
+
+def hdx_keep_crisis_tag_string_if_not_sysadmin(key: FlattenKey, data: FlattenDataDict, errors: FlattenErrorDict,
+                                               context: Context) -> Any:
+    user = context.get('user')
+
+    can_remove_crisis_tags = user and authz.is_sysadmin(user)
+    has_new_tags = ('tags', 0, 'name') in data
+
+    if not can_remove_crisis_tags and not has_new_tags:
+        pkg_id = data.get(('id',))
+        if pkg_id:
+            new_tag_names_value = data.get(key)
+            if isinstance(new_tag_names_value, list):
+                new_tag_names = set(new_tag_names_value)
+            elif isinstance(new_tag_names_value, str):
+                new_tag_names = set(new_tag_names_value.split(','))
+            else:
+                new_tag_names = set()
+            old_tag_names = _extract_old_tag_names(context, pkg_id)
+
+            crisis_tags = [tag for tag in old_tag_names if tag.startswith('crisis-') and tag not in new_tag_names]
+
+            combined_tags = new_tag_names.union(crisis_tags)
+            data[key] = ','.join(combined_tags)
+
+
+def hdx_keep_crisis_tags_if_not_sysadmin(key: FlattenKey, data: FlattenDataDict, errors: FlattenErrorDict,
+                                         context: Context) -> Any:
+    user = context.get('user')
+
+    can_remove_crisis_tags = user and authz.is_sysadmin(user)
+
+    if not can_remove_crisis_tags:
+        pkg_id = data.get(('id',))
+        if pkg_id:
+            current_tags = _extract_tag_names(data)
+            old_tag_names = _extract_old_tag_names(context, pkg_id)
+
+            i = len(current_tags)
+            for old_tag in old_tag_names:
+                if old_tag.startswith('crisis-') and old_tag not in current_tags:
+                    data[('tags', i, 'name')] = old_tag
+                    current_tags.append(old_tag)
+                    i += 1
+
+
+def _extract_tag_names(data: FlattenDataDict) -> list:
+    tag_names = []
+
+    index = 0
+    while ('tags', index, 'name') in data:
+        tag_names.append(data.get(('tags', index, 'name'), ''))
+        index += 1
+
+    return tag_names
+
+
+def _extract_old_tag_names(context: Context, pkg_id: str) -> list:
+    tag_names = []
+
+    prev_package_dict = __get_previous_package_dict(context, pkg_id)
+    tags = prev_package_dict.get('tags', [])
+
+    for tag in tags:
+        tag_names.append(tag.get('display_name'))
+
+    return tag_names
 
 
 def hdx_update_last_modified_if_url_changed(key: FlattenKey, data: FlattenDataDict,

ckanext-hdx_package/ckanext/hdx_package/plugin.py

@@ -200,12 +200,19 @@ def _modify_package_schema(self, schema):
                 tk.get_validator('ignore_missing'),
                 tk.get_validator('hdx_dataseries_title_validator'),
                 tk.get_converter('convert_to_extras')
+            ],
+            'tag_string': [
+                tk.get_validator('hdx_keep_crisis_tag_string_if_not_sysadmin'),
+                tk.get_validator('ignore_missing'),
+                tk.get_validator('tag_string_convert'),
+                tk.get_validator('hdx_tag_string_approved_validator'),
             ]
         })
 
         schema['tags'].update(
             {
                 'name': [
+                    tk.get_validator('hdx_keep_crisis_tags_if_not_sysadmin'),
                     tk.get_validator('not_missing'),
                     tk.get_validator('not_empty'),
                     tk.get_validator('unicode_safe'),
@@ -537,6 +544,9 @@ def get_validators(self):
             'hdx_keep_if_fs_check_format': vd.hdx_keep_if_fs_check_format,
             'hdx_add_update_fs_check_info': vd.hdx_add_update_fs_check_info,
             'hdx_tag_name_approved_validator': vd.hdx_tag_name_approved_validator,
+            'hdx_tag_string_approved_validator': vd.hdx_tag_string_approved_validator,
+            'hdx_keep_crisis_tag_string_if_not_sysadmin': vd.hdx_keep_crisis_tag_string_if_not_sysadmin,
+            'hdx_keep_crisis_tags_if_not_sysadmin': vd.hdx_keep_crisis_tags_if_not_sysadmin,
             'hdx_update_last_modified_if_url_changed': vd.hdx_update_last_modified_if_url_changed,
             'hdx_disable_live_frequency_filestore_resources_only': vd.hdx_disable_live_frequency_filestore_resources_only,
             'hdx_in_hapi_flag_values': vd.hdx_value_in_list_wrapper(IN_HAPI_FLAG_VALUES, True),

ckanext-hdx_package/ckanext/hdx_package/tests/test_actions/test_package_update.py

@@ -423,6 +423,30 @@ def test_hdx_package_tags_validation(self):
             assert "Tag name '{}' can only be added by sysadmins".format(crisis_tag_name) in e.error_dict.get('tags')[
                 0], 'Only sysadmins are allowed to add tags starting with "crisis-"'
 
+        data_dict = self._modify_field(context_user, joeadmin, package['name'], 'tags',
+                                       [{'name': crisis_tag_name}, {'name': 'disease'}])
+        modified_package = data_dict.get('modified_package')
+
+        assert len(modified_package.get('tags')) == 2
+        assert crisis_tag_name in [tag['name'] for tag in modified_package.get(
+            'tags')], 'Crisis tags should be kept if specified by a user, as they were already added by a sysadmin'
+        assert 'disease' in [tag['name'] for tag in modified_package.get('tags')]
+
+        data_dict = self._modify_field(context_user, joeadmin, package['name'], 'tags', [{'name': 'boys'}])
+        modified_package = data_dict.get('modified_package')
+
+        assert len(modified_package.get('tags')) == 2
+        assert 'boys' in [tag['name'] for tag in modified_package.get('tags')], \
+            'Crisis tags should be kept even if not specified by a user, as they were already added by a sysadmin'
+
+        data_dict = self._modify_field(context, testsysadmin, package['name'], 'tags',
+                                       [{'name': 'boys'}, {'name': 'disease'}])
+        modified_package = data_dict.get('modified_package')
+
+        assert len(modified_package.get('tags')) == 2
+        assert 'boys' in [tag['name'] for tag in modified_package.get('tags')], \
+            'Crisis tags should be removed if not specified by a sysadmin'
+
 
     def _modify_field(self, context, user, package_id, key, value):
         modified_fields = {'id': package_id,

ckanext-hdx_package/ckanext/hdx_package/tests/test_controller/test_contribute_flow.py

@@ -241,7 +241,7 @@ def test_edit_lists(self):
         auth = {'Authorization': self.testsysadmin_token}
 
         post_params = self._get_dataset_post_param('testing-dataset-edit-lists')
-        post_params['tag_string'] = 'list_test_tag'
+        post_params['tag_string'] = 'boys'
 
         post_params["dataset_date"] = "[1960-01-01 TO 2012-12-31]"
         post_params['save'] = 'update-dataset-json'

ckanext-hdx_theme/ckanext/hdx_theme/helpers/ui_constants/landing_pages/hapi.py

@@ -3,7 +3,7 @@
     'HERO_SECTION_DESCRIPTION': '''HDX HAPI is in beta phase, and we are seeking feedback. To share your thoughts or join our slack channel, send an email to <a href="{0}" title="Contact us">[email protected]</a>.''',
 
     'DATA_COVERAGE_SECTION_TITLE': '''Data Coverage''',
-    'DATA_COVERAGE_SECTION_DESCRIPTION': '''Our initial coverage aligns with the data included in the <a href="{0}" target="_blank" data-module="hdx_click_stopper" data-module-link_type="hapi data coverage description" title="HDX Data Grids">HDX Data Grids</a>, which places the most important crisis data into six categories and 20 sub-categories. Our beta version of HDX HAPI covers 62 indicators across 25 locations that have a <a href="{1}" target="_blank" title="Humanitarian Response Plan" data-module="hdx_click_stopper" data-module-link_type="hapi data coverage description">Humanitarian Response Plan</a>. This data has been shared by nine different contributing organisations.''',
+    'DATA_COVERAGE_SECTION_DESCRIPTION': '''Our initial coverage aligns with the data included in the <a href="{0}" target="_blank" data-module="hdx_click_stopper" data-module-link_type="hapi data coverage description" title="HDX Data Grids">HDX Data Grids</a>, which places the most important crisis data into six categories and 20 sub-categories. Our beta version of HDX HAPI covers 51 indicators across 25 locations that have a <a href="{1}" target="_blank" title="Humanitarian Response Plan" data-module="hdx_click_stopper" data-module-link_type="hapi data coverage description">Humanitarian Response Plan</a>. This data has been shared by nine different contributing organisations.''',
     'DATA_COVERAGE_SECTION_PARAGRAPH': '''Refer to the <a href="{0}" target="_blank" data-module="hdx_click_stopper" data-module-link_type="hapi data coverage description" title="HAPI - The Humanitarian API">documentation</a> for the latest coverage. <a href="{1}" data-module="hdx_click_stopper" data-module-link_type="hapi data coverage description" title="Contact us">Contact us</a> to request additional indicators in future versions of HDX HAPI.''',
 
     'BE_INSPIRED_SECTION_TITLE': '''Be Inspired''',
@@ -118,13 +118,13 @@
 
 PARTNERS_CONSTANTS = [
     ('acled', 'ACLED'),
-    ('inform', 'INFORM'),
+    ('fsnwg', 'FSNWG'),
     ('fts', 'FTS'),
-    ('unfpa', 'UNFPA'),
+    ('inform', 'INFORM'),
+    ('iom', 'IOM'),
+    ('ocha', 'OCHA'),
     ('ophi', 'OPHI'),
-    ('fsnwg', 'FSNWG'),
+    ('unfpa', 'UNFPA'),
     ('unchr', 'UNHCR'),
-    ('ocha', 'OCHA'),
     ('wfp', 'WFP'),
-    ('iom', 'IOM'),
 ]

ckanext-hdx_theme/ckanext/hdx_theme/version.py

@@ -1 +1 @@
-hdx_version = 'v1.83.5'
+hdx_version = 'v1.83.6'