From 69dacbc89f368567a473bdcfdf8c6dc8184d4e59 Mon Sep 17 00:00:00 2001 From: Ryan Birmingham Date: Mon, 1 Apr 2019 19:58:35 -0400 Subject: [PATCH] 3.2.1 disto release --- Deps/Bindaas/Dockerfile | 4 +- Deps/auth_service/Dockerfile | 2 +- Deps/auth_service/app.js | 47 ++++++++++-- Deps/auth_service/package-lock.json | 73 +++++++++++++++++++ Deps/auth_service/package.json | 1 + Deps/auth_service/readme.md | 10 +++ README.md | 9 +-- auth_without_pathdb.md | 33 +++++++++ caMicroscope.yml | 23 +++--- config/default_data.js | 26 +++++++ config/login.html | 75 +++++++++++++++++++ config/pathdb_pre.sh | 4 ++ config/pathdb_routes.json | 67 +++++++++++++++++ config/routes.json | 12 ++-- config/run_idx.sh | 4 +- config/test_seed.js | 8 ++- jwt_keys/.DS_Store | Bin 0 -> 6148 bytes jwt_keys/.gitignore | 4 ++ jwt_keys/make_keys.sh | 1 + keys/.gitignore | 2 + keys/make_keys.sh | 1 + migrate/convert_heatmaps.js | 3 +- pathDbCamic.yml | 61 ++++++++++++++++ utils/README.md | 22 ------ utils/annots-2-to-3.js | 60 ---------------- utils/debug-annot.json | 1 - utils/debug-slide.json | 19 ----- utils/debug-template.json | 108 ---------------------------- utils/slides-2-to-3.js | 20 ------ utils/templates-2-to-3.js | 32 --------- 30 files changed, 437 insertions(+), 295 deletions(-) create mode 100644 Deps/auth_service/readme.md create mode 100644 auth_without_pathdb.md create mode 100644 config/default_data.js create mode 100644 config/login.html create mode 100644 config/pathdb_pre.sh create mode 100644 config/pathdb_routes.json create mode 100644 jwt_keys/.DS_Store create mode 100644 jwt_keys/.gitignore create mode 100644 jwt_keys/make_keys.sh create mode 100644 keys/.gitignore create mode 100644 keys/make_keys.sh create mode 100644 pathDbCamic.yml delete mode 100644 utils/README.md delete mode 100644 utils/annots-2-to-3.js delete mode 100644 utils/debug-annot.json delete mode 100644 utils/debug-slide.json delete mode 100644 utils/debug-template.json delete mode 100644 utils/slides-2-to-3.js delete mode 100644 utils/templates-2-to-3.js diff --git a/Deps/Bindaas/Dockerfile b/Deps/Bindaas/Dockerfile index f261aaf..3405d36 100644 --- a/Deps/Bindaas/Dockerfile +++ b/Deps/Bindaas/Dockerfile @@ -14,8 +14,8 @@ WORKDIR /root/src RUN apk --no-cache add openjdk8-jre WORKDIR /root/bindaas/ -RUN wget https://github.com/sharmalab/bindaas/releases/download/v3.3.5/bindaas-dist-3.3.5.tar.gz -RUN tar -xvf bindaas-dist-3.3.5.tar.gz && rm bindaas-dist-3.3.5.tar.gz +RUN wget https://github.com/sharmalab/bindaas/releases/download/v3.3.8/bindaas-dist-3.3.8.tar.gz +RUN tar -xvf bindaas-dist-3.3.8.tar.gz && rm bindaas-dist-3.3.8.tar.gz COPY bindaas.config.json /root/bindaas/bin/ COPY run.sh /root/bindaas/bin/ diff --git a/Deps/auth_service/Dockerfile b/Deps/auth_service/Dockerfile index 1359d2f..bc86932 100644 --- a/Deps/auth_service/Dockerfile +++ b/Deps/auth_service/Dockerfile @@ -1,4 +1,4 @@ -FROM node:8 +FROM node:8-alpine RUN mkdir /root/src COPY . /root/src WORKDIR /root/src diff --git a/Deps/auth_service/app.js b/Deps/auth_service/app.js index 2ae42f6..bfb65de 100644 --- a/Deps/auth_service/app.js +++ b/Deps/auth_service/app.js @@ -1,9 +1,40 @@ const express = require('express') const rp = require('request-promise'); const app = express(); +const fs = require("fs"); var jwt = require('jsonwebtoken'); +var jwkToPem = require('jwk-to-pem'); var PORT = process.env.PORT || 8010 var BASE_USER_URL = "http://ca-data:9099/services/caMicroscope/Authorization/query/getAuth?name=" +var SECRET = process.env.SECRET +var EXPIRY = process.env.EXPIRY || "1h" + +try { + let prikey_path = "/keys/key" + if(fs.existsSync(prikey_path)){ + var PRIKEY = fs.readFileSync(prikey_path, 'utf8') + } +} catch (err){ + console.error(err) +} + +try { + let cert_path = "/keys/certificate" + if(fs.existsSync(cert_path)){ + var SECRET = fs.readFileSync(cert_path, 'utf8') + } +} catch (err){ + console.error(err) +} +// jwks +try { + let jwk_path = "/keys/jwk.json" + if(fs.existsSync(jwk_path)){ + var SECRET = jwkToPem(JSON.parse(fs.readFileSync(jwk_path, 'utf8'))) + } +} catch (err){ + console.error(err) +} const getToken = function(req) { if (req.headers.authorization && req.headers.authorization.split(' ')[0] === 'Bearer') { // Authorization: Bearer g1jipjgi1ifjioj @@ -19,25 +50,27 @@ const getToken = function(req) { } app.get("/check", async function(req,res){ - var token = jwt.decode(getToken(req)) + var token = jwt.verify(getToken(req), SECRET) if (!(token && (token.email || token.sub))){ // jwt doesn't say who you are, so bye res.sendStatus(401) } else { var name = token.email || token.sub - var attr = req.query.attr user_detail = rp({ uri: BASE_USER_URL + name, json: true }) user_detail.then(x=>{ console.log(x) - if (x.length >= 1){ - if (!attr || x[0].attrs.includes(attr)){ - res.sendStatus(200) - } else { - res.sendStatus(401) + if (x.length >= 1 && x[0].hasOwnProperty('name')){ + let attrs = x[0].attrs || [] + data = { + 'name':x[0].name, + 'attrs':attrs } + // sign using the mounted key + var token = jwt.sign(data, PRIKEY, {algorithm:"RS256", expiresIn: EXPIRY}) + res.send({'token':token}) } else { res.sendStatus(401) } diff --git a/Deps/auth_service/package-lock.json b/Deps/auth_service/package-lock.json index b178863..09b4082 100644 --- a/Deps/auth_service/package-lock.json +++ b/Deps/auth_service/package-lock.json @@ -37,6 +37,16 @@ "safer-buffer": "~2.1.0" } }, + "asn1.js": { + "version": "4.10.1", + "resolved": "https://registry.npmjs.org/asn1.js/-/asn1.js-4.10.1.tgz", + "integrity": "sha512-p32cOF5q0Zqs9uBiONKYLm6BClCoBCM5O9JfeUSlnQLBTxYdTK+pW+nXflm8UkKd2UYlEbYz5qEi0JuZR9ckSw==", + "requires": { + "bn.js": "^4.0.0", + "inherits": "^2.0.1", + "minimalistic-assert": "^1.0.0" + } + }, "assert-plus": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", @@ -70,6 +80,11 @@ "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.3.tgz", "integrity": "sha512-/qKPUQlaW1OyR51WeCPBvRnAlnZFUJkCSG5HzGnuIqhgyJtF+T94lFnn33eiazjRm2LAHVy2guNnaq48X9SJuw==" }, + "bn.js": { + "version": "4.11.8", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.8.tgz", + "integrity": "sha512-ItfYfPLkWHUjckQCk8xC+LwxgK8NYcXywGigJgSwOP8Y2iyWT4f2vsZnoOXTTbo+o5yXmIUJ4gn5538SO5S3gA==" + }, "body-parser": { "version": "1.18.3", "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.18.3.tgz", @@ -87,6 +102,11 @@ "type-is": "~1.6.16" } }, + "brorand": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/brorand/-/brorand-1.1.0.tgz", + "integrity": "sha1-EsJe/kCkXjwyPrhnWgoM5XsiNx8=" + }, "buffer-equal-constant-time": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", @@ -188,6 +208,20 @@ "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" }, + "elliptic": { + "version": "6.4.1", + "resolved": "https://registry.npmjs.org/elliptic/-/elliptic-6.4.1.tgz", + "integrity": "sha512-BsXLz5sqX8OHcsh7CqBMztyXARmGQ3LWPtGjJi6DiJHq5C/qvi9P3OqgswKSDftbu8+IoI/QDTAm2fFnQ9SZSQ==", + "requires": { + "bn.js": "^4.4.0", + "brorand": "^1.0.1", + "hash.js": "^1.0.0", + "hmac-drbg": "^1.0.0", + "inherits": "^2.0.1", + "minimalistic-assert": "^1.0.0", + "minimalistic-crypto-utils": "^1.0.0" + } + }, "encodeurl": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", @@ -326,6 +360,25 @@ "har-schema": "^2.0.0" } }, + "hash.js": { + "version": "1.1.7", + "resolved": "https://registry.npmjs.org/hash.js/-/hash.js-1.1.7.tgz", + "integrity": "sha512-taOaskGt4z4SOANNseOviYDvjEJinIkRgmp7LbKP2YTTmVxWBl87s/uzK9r+44BclBSp2X7K1hqeNfz9JbBeXA==", + "requires": { + "inherits": "^2.0.3", + "minimalistic-assert": "^1.0.1" + } + }, + "hmac-drbg": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/hmac-drbg/-/hmac-drbg-1.0.1.tgz", + "integrity": "sha1-0nRXAQJabHdabFRXk+1QL8DGSaE=", + "requires": { + "hash.js": "^1.0.3", + "minimalistic-assert": "^1.0.0", + "minimalistic-crypto-utils": "^1.0.1" + } + }, "http-errors": { "version": "1.6.3", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz", @@ -439,6 +492,16 @@ "safe-buffer": "^5.0.1" } }, + "jwk-to-pem": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/jwk-to-pem/-/jwk-to-pem-2.0.1.tgz", + "integrity": "sha512-KKu0WuDDjqw2FlRFp9/vk9TMO/KvgpZVKzdhhYcNyy5OwE8dw9lOK5OQTQHIJ7m+HioI/4P44sAtVuDrQ8KQfw==", + "requires": { + "asn1.js": "^4.5.2", + "elliptic": "^6.2.3", + "safe-buffer": "^5.0.1" + } + }, "jws": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/jws/-/jws-3.2.1.tgz", @@ -521,6 +584,16 @@ "mime-db": "~1.38.0" } }, + "minimalistic-assert": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/minimalistic-assert/-/minimalistic-assert-1.0.1.tgz", + "integrity": "sha512-UtJcAD4yEaGtjPezWuO9wC4nwUnVH/8/Im3yEHQP4b67cXlD/Qr9hdITCU1xDbSEXg2XKNaP8jsReV7vQd00/A==" + }, + "minimalistic-crypto-utils": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/minimalistic-crypto-utils/-/minimalistic-crypto-utils-1.0.1.tgz", + "integrity": "sha1-9sAMHAsIIkblxNmd+4x8CDsrWCo=" + }, "ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", diff --git a/Deps/auth_service/package.json b/Deps/auth_service/package.json index 7f3cd9a..c42bc31 100644 --- a/Deps/auth_service/package.json +++ b/Deps/auth_service/package.json @@ -12,6 +12,7 @@ "express": "^4.16.4", "express-promise": "^0.4.0", "jsonwebtoken": "^8.4.0", + "jwk-to-pem": "^2.0.1", "request": "^2.88.0", "request-promise": "^4.2.4" } diff --git a/Deps/auth_service/readme.md b/Deps/auth_service/readme.md new file mode 100644 index 0000000..111be3a --- /dev/null +++ b/Deps/auth_service/readme.md @@ -0,0 +1,10 @@ +# Auth Service for Elevate + + +Uses bindaas endpoint to determine users and their permissions. + +Set SECRET to the jwt secret, or use /keys/certificate with a certificate, or /keys/jwk.json with a jwk. + +Use EXPIRY to set a token expiration other than the default one hour. + +This tool needs key and key.pub mounted to /keys to sign JWTs to elevate. In this distribution, run jwt_keys/make_keys.sh to do so. diff --git a/README.md b/README.md index c23c4c8..dff007e 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,4 @@ # caMicroscope Distribution - -[![Build Status](https://travis-ci.org/camicroscope/Distro.svg?branch=master)](https://travis-ci.org/camicroscope/Distro) - caMicroscope 3.X distribution @@ -13,7 +10,7 @@ Use `docker-compose -f caMicroscope.yml build` to rebuild the services. Once everything is up, go to :4010/ to see the landing page. ## SSL -To enable ssl, mount the private key and certificate files to elevate in /root/src/ssl/privatekey.pem and /root/src/ssl/certificate.pem respectively. HTTPS mode will only be enabled if both of these files are present. +To enable ssl, mount the private key and certificate files to elevate in /root/src/ssl/privatekey.pem and /root/src/ssl/certificate.pem respectively. HTTPS mode will only be enabled if both of these files are present. ## Component Services mongo - vanilla mongo container @@ -39,5 +36,9 @@ Image Volume - This is, by default, the images directory in this directory. If t Packages - Packages are built in the viewer service using parcel, mount a different directory with packages.js to the package directory to overwrite or add functionality. +## PathDB + +To use pathdb, use pathDbCamic.yml instead of caMicroscope.yml, and replace routes.json with pathdb_routes.json. This deployment does not include the auth and loader as separate services, as this PathDB provides that functionality. + ## Support Feel free to add any support inquiry as a github issue to this repository. diff --git a/auth_without_pathdb.md b/auth_without_pathdb.md new file mode 100644 index 0000000..403da2e --- /dev/null +++ b/auth_without_pathdb.md @@ -0,0 +1,33 @@ +# QUIP Auth Without PathDB + +## About this Method and Guide + +PathDB is used as both authentication and authorization. In instances where PathDB is not desired, we use a combination of an external identity provider and an internal authorization service. The authorization service consumes Json Web Tokens (JWTs) from the identity provider, and then will issue JWTs which convey both authentication and authorization, which are consumed by the application. + +This guide aims to explain how to make a deployment use this method. + +## Getting an Identity Provider and Setting up Login + +There are many identity providers, but for testing and examples, we have been using auth0. + +When selecting, an identity provider, note that we expect it to provide a JWT, and to have a certificate/public key/secret which can be used to verify such JWTs. + +The example given in the Distro within config/login.html is set up for auth0; simply change the corresponding variables for your auth0 application if auth0 is used. If using another identity provider, then login.html, or equivalent, needs to, at least, set the JWT to a cookie called "token", and call the auth service's 'check' route, and save a successful result as the token. Follow the guide which your identity provider uses for further guidance. + +## Keys/Certificates + +Add the following files; by default, they are mounted: + +- --./jwt\_keys/certificate or ./jwt\_keys/jwk.json is the certificate/public key/secret or jwk (respectively) from the **identity provider**. (If both are included, the jwk takes precedence). +- --./jwt\_keys/key and ./jwt\_keys/key.pub are used as the signing and check keys for the **auth service** + - --These can (and should) be generated with ./kwt\_keys/make\_keys.sh + +## Deployment Configuration + +Turn off disable security under the elevate service to block routes. + +## Adding Users to Database + +Add users as in ./config/add\_mongo\_users.js. Attributes can be added to deny access to routes (e.g. allow only some users to post and delete) + +The name field is the email field (or failing that, sub field) in that priority from the identity provider. diff --git a/caMicroscope.yml b/caMicroscope.yml index 9cbc95e..7512e1f 100644 --- a/caMicroscope.yml +++ b/caMicroscope.yml @@ -2,15 +2,17 @@ version: '3' services: mongo: - image: mongo + image: mongo:3.4.19-jessie container_name: ca-mongo volumes: - ./db:/data/db logging: driver: none idxMongo: - image: mongo + image: mongo:3.4.19-jessie container_name: ca-idx + logging: + driver: none links: - mongo volumes: @@ -30,36 +32,37 @@ services: - ./config/bindaas_projects/:/root/bindaas/bin/projects/ - ./config/bindaas.config.json:/root/bindaas/bin/bindaas.config.json iip: - build: "https://github.com/camicroscope/iipImage.git#v3.1.0" + build: "https://github.com/camicroscope/iipImage.git#develop" container_name: ca-iip - logging: - driver: none volumes: - ./images/:/images/ viewer: - build: "https://github.com/camicroscope/caMicroscope.git#v3.1.0" + build: "https://github.com/camicroscope/caMicroscope.git#develop" container_name: ca-front + volumes: + - ./config/login.html:/var/www/html/login.html logging: driver: none loader: - build: "https://github.com/camicroscope/SlideLoader.git#v3.1.0" + build: "https://github.com/camicroscope/SlideLoader.git#develop" container_name: ca-load logging: driver: none volumes: - ./images/:/images/ elevate: - build: "https://github.com/camicroscope/Security.git#v3.1.0" + build: "https://github.com/camicroscope/Security.git#develop" container_name: ca-security ports: ["4010:4010"] volumes: - ./config/routes.json:/root/src/routes.json + - ./jwt_keys/:/keys/ environment: - DISABLE_SEC=true - logging: - driver: none auth: build: "./Deps/auth_service/" + volumes: + - ./jwt_keys/:/keys/ container_name: ca-auth logging: driver: none diff --git a/config/default_data.js b/config/default_data.js new file mode 100644 index 0000000..abc320f --- /dev/null +++ b/config/default_data.js @@ -0,0 +1,26 @@ +let defaultTemplate = { + "_id": "0", + "type": "object", + "id": "annotation-form", + "name": "AnnotSchema", + "description": "", + "links": [], + "additionalProperties": false, + "properties": { + "name": { + "id": "a0", + "title": "Identity Name", + "type": "string", + "required": true, + "description": "note name" + },"notes": { + "id": "a1", + "title": "Notes: ", + "type": "string", + "format":"textarea", + "maxLength": 128 + } + } +} + +db.template.insert(defaultTemplate) diff --git a/config/login.html b/config/login.html new file mode 100644 index 0000000..9a10206 --- /dev/null +++ b/config/login.html @@ -0,0 +1,75 @@ + +

Redirecting...

+ + + diff --git a/config/pathdb_pre.sh b/config/pathdb_pre.sh new file mode 100644 index 0000000..9b8ee8f --- /dev/null +++ b/config/pathdb_pre.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +cp /etc/httpd/conf/quip.key /keys/key +cp /etc/httpd/conf/quip.crt /keys/key.pub +sh /root/run.sh diff --git a/config/pathdb_routes.json b/config/pathdb_routes.json new file mode 100644 index 0000000..0916fb6 --- /dev/null +++ b/config/pathdb_routes.json @@ -0,0 +1,67 @@ +{ + "root":"http://quip-pathdb:8777/", + "_root_public": "true", + "services":{ + "data":{ + "_base": "http://ca-data:9099/services/caMicroscope", + "Mark": { + "get":"/Mark/query/get", + "find":"/Mark/query/find", + "multi":"/Mark/query/multiFind", + "findBound":"/Mark/query/findBound", + "types":"/Mark/query/findTypes", + "post":{"path": "/Mark/submit/json", "attr": "write"}, + "delete":{"path": "/Mark/delete/deleteMark", "attr": "write"} + }, + "Heatmap": { + "get":"/Heatmap/query/get", + "find":"/Heatmap/query/find", + "types":"/Heatmap/query/findTypes", + "post":{"path": "/Heatmap/submit/json", "attr": "write"}, + "delete":{"path": "/Heatmap/delete/deleteHeatmap", "attr": "write"} + }, + "Overlay": { + "get":"/Overlay/query/get", + "find":"/Overlay/query/find", + "post":{"path": "/Overlay/submit/json", "attr": "write"}, + "delete":{"path": "/Overlay/delete/deleteOverlay", "attr": "write"} + }, + "Slide": { + "get":"/Slide/query/get", + "find":"/Slide/query/find", + "post":{"path": "/Slide/submit/json", "attr": "write"}, + "delete":{"path": "/Slide/delete/deleteSlide", "attr": "write"}, + "update":{"path": "/Slide/update/updateSlide", "attr": "write"} + }, + "Template": { + "get":"/Template/query/get", + "find":"/Template/query/find", + "post":{"path": "/Template/submit/json", "attr": "write"}, + "delete":{"path": "/Template/delete/deleteTemplate", "attr": "write"} + } + }, + "img": { + "_public": "true", + "_base":"http://ca-iip/fcgi-bin/iipsrv.fcgi", + "IIP": { + "raw":"" + }, + "Slide":{ + "_resolver":{ + "destination":"?DeepZoom={OUT}", + "url": "http://ca-pathdb/node/{IN}?_format=json", + "field":"field_iip_path[0].value", + "before":["_files", ".dzi"] + } + } + }, + "pathdb":{ + "_public": "true", + "_base": "http://quip-pathdb/", + "slide":{ + "info":"/node/", + "unmod":"" + } + } + } +} diff --git a/config/routes.json b/config/routes.json index 65cb53d..adb9d6f 100644 --- a/config/routes.json +++ b/config/routes.json @@ -2,13 +2,10 @@ "root":"http://ca-front:80/", "_root_public": "true", "auth":{ - "elevate_url":"http://ca-auth:8010/check", - "elevate_ok":{"mode": "status"}, - "attr_suffix":"?attr=" + "permissions_field":"attrs" }, "services":{ "data":{ - "_public": "true", "_base": "http://ca-data:9099/services/caMicroscope", "Mark": { "get":"/Mark/query/get", @@ -71,6 +68,13 @@ "info":"/data/one", "thumb":"/data/thumbnail" } + }, + "auth": { + "_public": "true", + "_base": "http://ca-auth:8010", + "Token":{ + "check":"/check" + } } } } diff --git a/config/run_idx.sh b/config/run_idx.sh index de1976f..314f02a 100755 --- a/config/run_idx.sh +++ b/config/run_idx.sh @@ -1,9 +1,11 @@ until mongo --host ca-mongo --eval "print(\"waited for connection\")" do - sleep 1 + sleep 2 done mongo --host ca-mongo camic /config/mongo_idx.js echo "indexes created" mongo --host ca-mongo camic /config/add_mongo_users.js echo "users created" +mongo --host ca-mongo camic /config/default_data.js +echo "defaults added" diff --git a/config/test_seed.js b/config/test_seed.js index 53706f8..effeb19 100644 --- a/config/test_seed.js +++ b/config/test_seed.js @@ -212,10 +212,12 @@ const annotation_schema = { "title": "Check if tumor present (For BL3 only)", "type": "boolean" }, - "additional_notes": { + "notes": { "id": "a10", - "title": "Additional notes: ", - "type": "radio" + "title": "Notes: ", + "type": "string", + "format":"textarea", + "maxLength": 128 } } }; diff --git a/jwt_keys/.DS_Store b/jwt_keys/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..2e90103ed5626ef4ed020aa85368372cdb700b18 GIT binary patch literal 6148 zcmeHKJxc>Y5Pf3;24ay;%Pp-0Ynvs+!rC98o(h7I1VReC{X70jee(gX29kkf;EMsh9}0(J4Q!5fbx>IgK%8>ggtq4rwdCodwthRO*4n~x%b}uiO$`cdE5$@B#eC3v{ return{ name:d, - range:[0,1] + range:[0,1], + value:[0.5,1] }}); return `{ "provenance":{ diff --git a/pathDbCamic.yml b/pathDbCamic.yml new file mode 100644 index 0000000..71a50c9 --- /dev/null +++ b/pathDbCamic.yml @@ -0,0 +1,61 @@ +version: '3' + +services: + mongo: + image: mongo + container_name: ca-mongo + volumes: + - ./db:/data/db + logging: + driver: none + idxMongo: + image: mongo + container_name: ca-idx + links: + - mongo + volumes: + - ./config:/config + command: + - /config/run_idx.sh + depends_on: + - mongo + bindaas: + build: "./Deps/Bindaas/" + depends_on: + - "mongo" + container_name: ca-data + logging: + driver: none + ports: ["8080:8080", "9099:9099"] + volumes: + - ./config/bindaas_projects/:/root/bindaas/bin/projects/ + - ./config/bindaas.config.json:/root/bindaas/bin/bindaas.config.json + iip: + build: "https://github.com/camicroscope/iipImage.git#release" + container_name: ca-iip + volumes: + - ./images/:/data/images/ + viewer: + build: "https://github.com/camicroscope/caMicroscope.git#develop" + container_name: ca-front + logging: + driver: none + elevate: + build: "https://github.com/camicroscope/Security.git#develop" + container_name: ca-security + ports: ["4010:4010"] + volumes: + - ./config/routes.json:/root/src/routes.json + environment: + - DISABLE_SEC=true + pathdb: + build: "https://github.com/SBU-BMI/PathDB.git#master" + container_name: quip-pathdb + ports: ["8887:80"] + volumes: + - ./data:/data/ + - ./images/:/data/pathdb/files/wsi + - ./jwt_keys/:/keys/ + - ./config:/config + command: + - /config/pathdb_pre.sh diff --git a/utils/README.md b/utils/README.md deleted file mode 100644 index 6e6bbab..0000000 --- a/utils/README.md +++ /dev/null @@ -1,22 +0,0 @@ -# Migrating Data from QUIP 2.0 to QUIP 3.0 - -## Annotations - -Since the annotation type has changed to allow for more dynamic options, the annotation type has undergone some changes. -Run the tool with `node annots-to-to-3.js annots.json > out.json`, where annots.json is a list of QUIP 2.0 style annotations (see debug-annot.json for a short example). -Use the result with `mongoimport --db camic --collection mark --file out.json --jsonArray` in the ca-mongo container. - -**Please note that this should be run on a single image's annotations at a time, otherwise it will combine them incorrectly.** - - -## Slides - -Most slide data is preserved, we've just added some fields to make things simpler for viewer interactions, and made 'name' the canonical form of the slide, replacing the often improperly used 'case_id'. -Run the tool with `node slides-to-to-3.js slides.json > out.json`, where slides.json is a list of QUIP 2.0 style slides (see debug-slide.json for a short example). -Use the result with `mongoimport --db camic --collection slide --file out.json --jsonArray` in the ca-mongo container. - -## Templates - -The template format is entirely different as a result of a switch from js-form to pure-form. -Run the tool with `node templates-to-to-3.js templates.json > out.json`, where templates.json is a list of QUIP 2.0 style templates (see debug-template.json for a short example). -Use the result with `mongoimport --db camic --collection template --file out.json --jsonArray` in the ca-mongo container. diff --git a/utils/annots-2-to-3.js b/utils/annots-2-to-3.js deleted file mode 100644 index e691ee2..0000000 --- a/utils/annots-2-to-3.js +++ /dev/null @@ -1,60 +0,0 @@ -// run with node -// take in a json file of annotations -// convert annots to new form -// spit out result to stdout - - -var fs = require("fs"); -let filename = process.argv[2] || "annots.json" -var annots = JSON.parse(fs.readFileSync(filename)); -names = new Set() -res = {} -var available_colors = ['#a6cee3', '#1f78b4', '#b2df8a', '#33a02c', '#fb9a99', '#e31a1c', '#fdbf6f', '#ff7f00', '#cab2d6', '#6a3d9a', '#ffff99', '#b15928']; -for (x in annots){ - annot = annots[x] - name = annot.provenance.analysis.execution_id - names.add(name) - // pick a color, default style - color = available_colors[x%available_colors.length] - annot.provenance.image.slide = annot.provenance.image.case_id - annot.properties = {annotations:{name: annot.provenance.analysis.execution_id}} - annot.provenance.analysis.execution_id = "_a" + x - props = {footprint: annot.footprint, style: {color:color, "lineJoin": "round", "lineCap": "round", lineWidth:"30"}} - if (name in res){ - // recalculate bounds - s = annot.geometry.coordinates[0].concat(res[name].geometries.features[0].bound) - xs = s.map(x=>x[0]) - ys = s.map(x=>x[1]) - // get bounds - x0 = Math.min.apply(null, xs) - x1 = Math.max.apply(null, xs) - y0 = Math.min.apply(null, ys) - y1 = Math.max.apply(null, ys) - res[name].geometries.features[0].bound = {type:"Polygon", coordinates: coords} - // add polygon - props.style.color = res[name].geometries.features[0].properties.style.color - res[name].geometries.features.push({type:"Feature", properties:props, geometry: annot.geometry}) - } else { - // convert geometry - props.style.color = available_colors[x%available_colors.length] - annot.geometries = {type:"FeatureCollection", features:[{type:"Feature", properties:props, geometry: annot.geometry}]} - s = annot.geometry.coordinates[0] - xs = s.map(x=>x[0]) - ys = s.map(x=>x[1]) - // get bounds - x0 = Math.min.apply(null, xs) - x1 = Math.max.apply(null, xs) - y0 = Math.min.apply(null, ys) - y1 = Math.max.apply(null, ys) - coords = [[[x0,y0], [x0,y1], [x1,y1], [x1,y0], [x0,y0]]] - annot.geometries.features[0].bound = {type:"Polygon", coordinates: coords} - // remove old geometry - delete annot['geometry'] - delete annot['_id'] - res[name] = annot - } -} -var values = Object.keys(res).map(function(key){ - return res[key]; -}); -console.log(JSON.stringify(values)) diff --git a/utils/debug-annot.json b/utils/debug-annot.json deleted file mode 100644 index ddd89c7..0000000 --- a/utils/debug-annot.json +++ /dev/null @@ -1 +0,0 @@ -[{ "_id" : { "$oid" : "5b42e92056d9cf49a45d04e9"} , "geometry" : { "type" : "Polygon" , "coordinates" : [ [ [ 0.56640625 , 0.12932342290878296] , [ 0.5680338740348816 , 0.12932342290878296] , [ 0.5680338740348816 , 0.13140928745269775] , [ 0.56640625 , 0.13140928745269775] , [ 0.56640625 , 0.12932342290878296]]]} , "footprint" : 4000000.0 , "provenance" : { "image" : { "case_id" : "TCGA-2F-A9KO-01Z-00-DX1"} , "analysis" : { "execution_id" : "cluster-1" , "source" : "computer"}}},{ "_id" : { "$oid" : "5b42e92056d9cf49a45d04ea"} , "geometry" : { "type" : "Polygon" , "coordinates" : [ [ [ 0.5680338740348816 , 0.12932342290878296] , [ 0.5696614384651184 , 0.12932342290878296] , [ 0.5696614384651184 , 0.13140928745269775] , [ 0.5680338740348816 , 0.13140928745269775] , [ 0.5680338740348816 , 0.12932342290878296]]]} , "footprint" : 4000000.0 , "provenance" : { "image" : { "case_id" : "TCGA-2F-A9KO-01Z-00-DX1"} , "analysis" : { "execution_id" : "cluster-1" , "source" : "computer"}}},{ "_id" : { "$oid" : "5b42e92056d9cf49a45d04f1"} , "geometry" : { "type" : "Polygon" , "coordinates" : [ [ [ 0.5680338740348816 , 0.13140928745269775] , [ 0.5696614384651184 , 0.13140928745269775] , [ 0.5696614384651184 , 0.13349515199661255] , [ 0.5680338740348816 , 0.13349515199661255] , [ 0.5680338740348816 , 0.13140928745269775]]]} , "footprint" : 4000000.0 , "provenance" : { "image" : { "case_id" : "TCGA-2F-A9KO-01Z-00-DX1"} , "analysis" : { "execution_id" : "cluster-1" , "source" : "computer"}}},{ "_id" : { "$oid" : "5b42e92056d9cf49a45d04eb"} , "geometry" : { "type" : "Polygon" , "coordinates" : [ [ [ 0.5696614384651184 , 0.12932342290878296] , [ 0.5712890625 , 0.12932342290878296] , [ 0.5712890625 , 0.13140928745269775] , [ 0.5696614384651184 , 0.13140928745269775] , [ 0.5696614384651184 , 0.12932342290878296]]]} , "footprint" : 4000000.0 , "provenance" : { "image" : { "case_id" : "TCGA-2F-A9KO-01Z-00-DX1"} , "analysis" : { "execution_id" : "cluster-1" , "source" : "computer"}}}] diff --git a/utils/debug-slide.json b/utils/debug-slide.json deleted file mode 100644 index f9c8cd7..0000000 --- a/utils/debug-slide.json +++ /dev/null @@ -1,19 +0,0 @@ -[{ - "objective" : 40, - "md5sum" : "f557f03dfd1a5fe02860e6282c977ccd", - "vendor" : "aperio", - "level_count" : 1, - "timestamp" : 1530211282.84724, - "file-location" : "/data/images/VTRPDAC_Test_PC1709_BL3_XX.svs", - "mpp-x" : "0.251", - "filename" : "/data/images/VTRPDAC_Test_PC1709_BL3_XX.svs", - "width" : null, - "case_id" : "VTRPDAC_Test_PC1709_BL3_XX", - "entry_date" : "2018-06-28", - "person_id" : "PC1709", - "height" : null, - "registry_id" : "R4", - "mpp-y" : "0.251", - "block_id" : "BL3", - "study_id" : "VTRPDAC_Test" -}] diff --git a/utils/debug-template.json b/utils/debug-template.json deleted file mode 100644 index 43a99ca..0000000 --- a/utils/debug-template.json +++ /dev/null @@ -1,108 +0,0 @@ -[{ - "digital_slide_quality" : { - "title" : "Check if histology is able to be evaluated", - "type" : "boolean" - }, - "histology" : { - "title" : "Histology: (For BL1 and BL2 only)", - "type" : "string", - "enum" : [ - "-", - "PDAC", - "PNET", - "other" - ] - }, - "hist_other_type" : { - "title" : "Other Histology: (For BL1 and BL2 only)", - "type" : "string", - "enum" : [ - "-", - "N/A", - "Colloid carcinoma (mucinous noncystic carcinoma)", - "Signet-ring cell carcinoma", - "Adenosquamous carcinoma", - "Intraductal papillary-mucinous neoplasm with an associated invasive carcinoma", - "Intraductal tubulopapillary neoplasm with an associated invasive carcinoma", - "Mucinous cystic neoplasm with an associated invasive carcinoma", - "Large cell neuroendocrine carcinoma", - "Small cell neuroendocrine carcinoma", - "Neuroendocrine carcinoma (poorly differentiated)", - "Undifferentiated (anaplastic) carcinoma", - "Undifferentiated carcinoma with osteoclast-like giant cells", - "Acinar cell carcinoma", - "Acinar cell cystadenocarcinoma", - "Serous cystadenocarcinoma", - "Mixed acinar-ductal carcinoma", - "Mixed ductal-neuroendocrine carcinoma", - "Mixed acinar-neuroendocrine carcinoma", - "Mixed acinar-neuroendocrine-ductal carcinoma", - "Solid-pseudopapillary neoplasm", - "Hepatoid carcinoma", - "Medullary carcinoma" - ] - }, - "cellularity_10" : { - "title" : "Cellularity percentage", - "type" : "string", - "enum" : [ - "-", - "0-10%", - "11-20%", - "21-30%", - "31-40%", - "41-50%", - "51-60%", - "61-70%", - "71-80%", - "81-90%", - "91-100%" - ] - }, - "tumor_cellularity" : { - "title" : "Tumor Cellularity: (For BL1 and BL2 only)", - "type" : "string", - "enum" : [ - "-", - "<20%", - ">=20%" - ] - }, - "tumor_necrosis" : { - "title" : "Tumor Necrosis: (For BL1 and BL2 only)", - "type" : "string", - "enum" : [ - "-", - "<20%", - ">=20%" - ] - }, - "adequacy" : { - "title" : "Adequacy: (For BL1 and BL2 only)", - "type" : "string", - "enum" : [ - "-", - "Adequate", - "Inadequate" - ] - }, - "normal_tissue_type" : { - "title" : "Normal Tissue Type: (For BL3 only)", - "type" : "string", - "enum" : [ - "-", - "Duodenum", - "Lymph Node", - "Spleen", - "Other" - ] - }, - "tumor_present" : { - "title" : "Check if tumor present (For BL3 only)", - "type" : "boolean" - }, - "additional_notes" : { - "title" : "Additional notes: ", - "type" : "textarea" - } -}] diff --git a/utils/slides-2-to-3.js b/utils/slides-2-to-3.js deleted file mode 100644 index 64c5852..0000000 --- a/utils/slides-2-to-3.js +++ /dev/null @@ -1,20 +0,0 @@ -// run with node -// take in a json file of slides/images metadata -// convert slides to new form -// spit out result to stdout - - -var fs = require("fs"); -let filename = process.argv[2] || "slides.json" -var slides = JSON.parse(fs.readFileSync(filename)); -res = [] -for (x in slides){ - slide = slides[x] - delete slide['_id'] - slide.location = slide['file-location'] || slide.filename - slide.name = slide.case_id - slide.mpp = slide['mpp-x'] || slide['mpp-y'] || slide.mpp - res.push(slide) -} - -console.log(JSON.stringify(res)) diff --git a/utils/templates-2-to-3.js b/utils/templates-2-to-3.js deleted file mode 100644 index ba109c5..0000000 --- a/utils/templates-2-to-3.js +++ /dev/null @@ -1,32 +0,0 @@ -// run with node -// take in a json file of templates -// convert templates to new form -// spit out result to stdout - - -var fs = require("fs"); -let filename = process.argv[2] || "templates.json" -var templates = JSON.parse(fs.readFileSync(filename)); -res = [] -for (x in templates){ - template = templates[x] - new_t = {} - new_t.type = "object" - new_t.id = "migrated-" + x - new_t.name = "migrated-" + x - new_t.links = [] - new_t.additionalProperties = false - props = {} - for (y in Object.keys(template)){ - yi = parseInt(y,10)+1 - k = Object.keys(template)[y] - props['arg'+yi] = {id:yi, title: template[k].title, type:template[k].type} - if ('enum' in template[k]){ - props['arg'+yi].enum = template[k].enum - } - } - new_t.properties = props - res.push(new_t) -} - -console.log(JSON.stringify(res))