Refactor import settings.

htan_girder/__init__.py

@@ -1,4 +1,7 @@
+import json
+
 from girder import plugin
+from girder.exceptions import ValidationException
 from girder.models.assetstore import Assetstore
 from girder.models.folder import Folder
 from girder.utility import setting_utilities
@@ -7,28 +10,35 @@
 from .rest import HTANResource
 
 
-@setting_utilities.validator(PluginSettings.HTAN_ASSETSTORE)
-def validateHTANAssetstore(doc):
-    if not doc.get('value', None):
-        doc['value'] = None
-    else:
-        Assetstore().load(doc['value'], exc=True)
-
-
-@setting_utilities.validator(PluginSettings.HTAN_IMPORT_PATH)
-def validateHTANImportPath(doc):
-    if not doc.get('value', None):
-        doc['value'] = None
-    else:
-        doc['value'] = str(doc['value'])
-
-
-@setting_utilities.validator(PluginSettings.HTAN_IMPORT_FOLDER)
-def validateHTANImportFolder(doc):
-    if not doc.get('value', None):
-        doc['value'] = None
-    else:
-        Folder().load(doc['value'], force=True, exc=True)
+@setting_utilities.validator(PluginSettings.HTAN_IMPORT_LIST)
+def validateHTANImportList(doc):
+    val = doc.get('value', None)
+    try:
+        if isinstance(val, list):
+            doc['value'] = json.dumps(val)
+        elif not val or val.strip() == '':
+            doc['value'] = None
+        else:
+            parsed = json.loads(val)
+            if not isinstance(parsed, list):
+                raise ValueError
+            doc['value'] = val.strip()
+    except (ValueError, AttributeError):
+        raise ValidationException('%s must be a JSON list.' % doc['key'], 'value')
+    keys = {}
+    for entry in json.loads(doc['value']):
+        if not entry.get('key') or entry.get('key') in keys:
+            raise ValidationException('Each entry must have a unique name', 'value')
+        try:
+            Assetstore().load(entry.get('assetstoreId'), exc=True)
+        except Exception:
+            raise ValidationException(
+                'Invalid assetstore ID %s.' % entry.get('assetstoreId'), 'value')
+        try:
+            Folder().load(entry.get('destinationId'), exc=True, force=True)
+        except Exception:
+            raise ValidationException(
+                'Invalid import folder ID %s.' % entry.get('destinationId'), 'value')
 
 
 class GirderPlugin(plugin.GirderPlugin):

htan_girder/constants.py

@@ -3,6 +3,4 @@
 
 # Constants representing the setting keys for this plugin
 class PluginSettings(object):
-    HTAN_ASSETSTORE = 'htan.assetstore'
-    HTAN_IMPORT_PATH = 'htan.import_path'
-    HTAN_IMPORT_FOLDER = 'htan.import_folder'
+    HTAN_IMPORT_LIST = 'htan.import_list'

htan_girder/reimport_job.py

@@ -1,3 +1,4 @@
+import json
 from threading import RLock
 
 from girder import logger
@@ -7,26 +8,31 @@
 from girder.models.user import User
 from girder.utility.progress import ProgressContext
 
+from girder_jobs.constants import JobStatus
 from girder_jobs.models.job import Job
 
 from .constants import PluginSettings
 
 _reimportStatus = {
     'lock': RLock(),
     'running': None,
-    'rerun': False,
+    'rerun': set(),
 }
 
 
-def reimportData():
+def reimportData(key):
     with _reimportStatus['lock']:
         if not _reimportStatus['running']:
-            startReimportJob()
+            startReimportJob(key)
         else:
-            _reimportStatus['rerun'] = True
+            if key not in _reimportStatus['rerun']:
+                logger.info('Adding future HTAN reimport job for %s.', key)
+            else:
+                logger.info('Already have a future HTAN reimport job for %s.', key)
+            _reimportStatus['rerun'] |= {key}
 
 
-def startReimportJob():
+def startReimportJob(key):
     with _reimportStatus['lock']:
         job = Job().createLocalJob(
             module='htan_girder.reimport_job',
@@ -35,20 +41,33 @@ def startReimportJob():
             type='htan_reimport',
             public=True,
             asynchronous=True,
+            kwargs=dict(key=key),
         )
         _reimportStatus['running'] = True
-        _reimportStatus['rerun'] = False
+        _reimportStatus['rerun'] -= {key}
         logger.info('Scheduling HTAN reimport job.')
         Job().scheduleJob(job)
 
 
-def reimportJob(*args, **kwargs):
+def reimportJob(job):
+    key = job['kwargs']['key']
+    job = Job().updateJob(
+        job, log='Started HTAN reimport job for %s\n' % key,
+        status=JobStatus.RUNNING)
     try:
-        assetstoreId = Setting().get(PluginSettings.HTAN_ASSETSTORE)
-        importPath = Setting().get(PluginSettings.HTAN_IMPORT_PATH) or ''
-        importFolderId = Setting().get(PluginSettings.HTAN_IMPORT_FOLDER)
+        importList = Setting().get(PluginSettings.HTAN_IMPORT_LIST)
+        if not importList:
+            return
+        record = {}
+        for entry in json.loads(importList):
+            if entry.get('key') == key:
+                record = entry
+        assetstoreId = record.get('assetstoreId')
+        importFolderId = record.get('destinationId')
         if not assetstoreId or not importFolderId:
-            logger.info('HTAN reimport job not configured.  An assetstore and folder must be set.')
+            logger.info(
+                'HTAN reimport job for key %s not configured.  An assetstore '
+                'and folder must be set.', key)
             return
         assetstore = Assetstore().load(assetstoreId)
         folder = Folder().load(importFolderId, force=True)
@@ -58,15 +77,32 @@ def reimportJob(*args, **kwargs):
                 'invalid or nonexistant assetstore or folder.')
             return
         admin = User().findOne({'admin': True})
-        logger.info('Starting HTAN reimport job.')
+        logger.info('Starting HTAN reimport job for %s.', key)
         with ProgressContext(True, user=admin, title='Importing data') as ctx:
             Assetstore().importData(
                 assetstore, parent=folder, parentType='folder',
-                params=dict(importPath=importPath), progress=ctx, user=admin,
+                params=dict(
+                    importPath=record.get('importPath', ''),
+                    fileIncludeRegex=record.get('fileIncludeRegex'),
+                    fileExcludeRegex=record.get('fileExcludeRegex'),
+                ), progress=ctx, user=admin,
                 leafFoldersAsItems=False)
-        logger.info('Finished HTAN reimport job.')
+        if record.get('endFunction'):
+            logger.info(
+                'Should now run function %s, but doing so it not implemented',
+                record['endFunction'])
+        logger.info('Finished HTAN reimport job for %s.' % key)
+        job = Job().updateJob(
+            job, log='Finished HTAN reimport job for %s.' % key,
+            status=JobStatus.SUCCESS)
+    except Exception:
+        job = Job().updateJob(
+            job, log='Error running HTAN reimport job for %s\n' % key,
+            status=JobStatus.ERROR)
+        logger.exception('Error running HTAN reimport job for %s', key)
+        return
     finally:
         with _reimportStatus['lock']:
             _reimportStatus['running'] = False
-            if _reimportStatus['rerun']:
-                startReimportJob()
+            if len(_reimportStatus['rerun']):
+                startReimportJob(sorted(_reimportStatus['rerun'])[0])

htan_girder/rest.py

@@ -10,12 +10,13 @@ def __init__(self):
         super().__init__()
         self.resourceName = 'htan'
 
-        self.route('POST', ('reimport', ), self.reimportData)
+        self.route('POST', ('reimport', ':key'), self.reimportData)
 
     @autoDescribeRoute(
         Description('Reimport a folder to an assetstore based on settings.')
+        .param('key', 'The key to reimport.', paramType='path')
     )
     @access.public
-    def reimportData(self):
-        reimportData()
+    def reimportData(self, key):
+        reimportData(key)
         return 'acknowledged'

htan_girder/web_client/views/ConfigView.js

@@ -1,5 +1,4 @@
 import $ from 'jquery';
-import _ from 'underscore';
 import View from '@girder/core/views/View';
 
 import PluginConfigBreadcrumbWidget from '@girder/core/views/widgets/PluginConfigBreadcrumbWidget';
@@ -16,26 +15,14 @@ import './ConfigView.styl';
  */
 var ConfigView = View.extend({
     events: {
-        'click #g-htan-save': function (event) {
-            this.$('#g-htan-error-message').text('');
-            var settings = _.map(this.settingsKeys, (key) => {
-                const element = this.$('#g-' + key.replace(/[_.]/g, '-'));
-                var result = {
-                    key,
-                    value: element.val() || null
-                };
-                if (key === 'htan.import_folder') {
-                    result.value = result.value ? result.value.split(' ')[0] : '';
-                }
-                return result;
-            });
-            this._saveSettings(settings);
-        },
+        'click #g-htan-save': '_recordSettings',
         'click #g-htan-cancel': function (event) {
             router.navigate('plugins', { trigger: true });
             router.navigate('plugins', { trigger: true });
         },
-        'click .g-open-browser': '_openBrowser'
+        'click .g-open-browser': '_openBrowser',
+        'click .g-htan-add': '_addRow',
+        'click .g-htan-delete': '_removeRow'
     },
     initialize: function () {
         this.breadcrumb = new PluginConfigBreadcrumbWidget({
@@ -44,9 +31,7 @@ var ConfigView = View.extend({
         });
 
         this.settingsKeys = [
-            'htan.assetstore',
-            'htan.import_path',
-            'htan.import_folder'
+            'htan.import_list'
         ];
         $.when(
             restRequest({
@@ -89,15 +74,15 @@ var ConfigView = View.extend({
             }
         });
         this.listenTo(this._browserWidgetView, 'g:saved', function (val) {
-            this.$('#g-htan-import-folder').val(val.id);
+            this._browserTarget.val(val.id);
             restRequest({
                 url: `resource/${val.id}/path`,
                 method: 'GET',
                 data: { type: val.get('_modelType') }
             }).done((result) => {
                 // Only add the resource path if the value wasn't altered
-                if (this.$('#g-htan-import-folder').val() === val.id) {
-                    this.$('#g-htan-import-folder').val(`${val.id} (${result})`);
+                if (this._browserTarget.val() === val.id) {
+                    this._browserTarget.val(`${val.id} (${result})`);
                 }
             });
         });
@@ -134,8 +119,44 @@ var ConfigView = View.extend({
         });
     },
 
-    _openBrowser: function () {
+    _openBrowser: function (el) {
+        this._browserTarget = $(el.target).closest('td').find('input');
         this._browserWidgetView.setElement($('#g-dialog-container')).render();
+    },
+
+    _addRow: function () {
+        let newrow = this.$('tr.htan-empty-row').clone();
+        let parent = this.$('tr.htan-empty-row').parent();
+        this.$('tr.htan-empty-row').removeClass('htan-empty-row');
+        parent.append(newrow);
+    },
+    _removeRow: function (evt) {
+        $(evt.target).closest('tr').remove();
+    },
+    _recordSettings: function (event) {
+        this.$('#g-htan-error-message').text('');
+        let importList = [];
+        this.$('tr').each(function () {
+            let entry = {};
+            $(this).find('[htan_prop]').each(function () {
+                let input = $(this);
+                let key = input.attr('htan_prop');
+                let value = input.val().trim();
+                if (value) {
+                    if (key === 'destinationId') {
+                        value = value.split(' ')[0];
+                    }
+                    entry[key] = value;
+                }
+            });
+            if (Object.keys(entry).length) {
+                importList.push(entry);
+            }
+        });
+        this._saveSettings([{
+            key: 'htan.import_list',
+            value: JSON.stringify(importList)
+        }]);
     }
 });