From 0d7ed00ea6cd2c026d49df65260ea09320aad322 Mon Sep 17 00:00:00 2001 From: David Steinberg Date: Mon, 4 Apr 2022 11:39:02 -0700 Subject: [PATCH 01/47] Turn off a test that was incorrectly specified --- scripts/tests/test-webservice-image-digest.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/scripts/tests/test-webservice-image-digest.py b/scripts/tests/test-webservice-image-digest.py index b621a49..f2ddae8 100644 --- a/scripts/tests/test-webservice-image-digest.py +++ b/scripts/tests/test-webservice-image-digest.py @@ -21,11 +21,12 @@ class TestDigest(unittest.TestCase): - def test_branch(self): - cmd = "{} {}".format(base_command, branch) - ret = subprocess.check_output(cmd, shell=True, universal_newlines=True).rstrip() - self.assertEqual(ret, "sha256:52cf6b09e89a238bfd1d98dd01139442d67fcaaa377c179f315dd06555f7bcae") - pass +# This test is turned off because there is no stable branch to test against +# def test_branch(self): +# cmd = "{} {}".format(base_command, branch) +# ret = subprocess.check_output(cmd, shell=True, universal_newlines=True).rstrip() +# self.assertEqual(ret, "sha256:52cf6b09e89a238bfd1d98dd01139442d67fcaaa377c179f315dd06555f7bcae") +# pass def test_simple_tag(self): cmd = "{} {}".format(base_command, simple_tag) From bcbd75023aebeca36598fce12f143770f21324f3 Mon Sep 17 00:00:00 2001 From: Charles Overbeck Date: Tue, 5 Apr 2022 09:22:41 -0700 Subject: [PATCH 02/47] Disallow robots unless it's prod (#220) https://ucsc-cgl.atlassian.net/browse/SEAB-4094 --- templates/robots.txt.template | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/templates/robots.txt.template b/templates/robots.txt.template index de94c39..c3f76cd 100644 --- a/templates/robots.txt.template +++ b/templates/robots.txt.template @@ -1,4 +1,8 @@ User-Agent: * +{{#PRODUCTION}} Allow: / - Sitemap: http{{#HTTPS}}s{{/HTTPS}}://{{ DOMAIN_NAME }}/sitemap.txt; +{{/PRODUCTION}} +{{^PRODUCTION}} +Disallow: / +{{/PRODUCTION}} From 24c9ff1ebe4dcaa868be29412826d4031b31f222 Mon Sep 17 00:00:00 2001 From: Kathy Tran Date: Wed, 20 Apr 2022 12:41:19 -0400 Subject: [PATCH 03/47] Add 1.13.0 migration --- templates/init_migration.sh.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/init_migration.sh.template b/templates/init_migration.sh.template index a5c2329..a23a8d7 100755 --- a/templates/init_migration.sh.template +++ b/templates/init_migration.sh.template @@ -12,4 +12,4 @@ java -Ddw.database.user=postgres -Ddw.database.password="{{{ POSTGRES_DBPASSWORD # this particular migration needs to run as postgres because only postgres can surrender ownership java -Ddw.database.user=postgres -Ddw.database.password="{{{ POSTGRES_DBPASSWORD }}}" -jar dockstore-webservice-*.jar db migrate web.yml --include 1.7.0.relinquish # future migrations will start here and should be run as dockstore -java -Ddw.database.user=dockstore -Ddw.database.password="{{{ DOCKSTORE_DBPASSWORD }}}" -jar dockstore-webservice-*.jar db migrate web.yml --include 1.8.0,1.9.0,1.10.0,1.11.0,1.12.0 | tee --append /dockstore_logs/webservice.out +java -Ddw.database.user=dockstore -Ddw.database.password="{{{ DOCKSTORE_DBPASSWORD }}}" -jar dockstore-webservice-*.jar db migrate web.yml --include 1.8.0,1.9.0,1.10.0,1.11.0,1.12.0,1.13.0 | tee --append /dockstore_logs/webservice.out From eb33c4204f1fccc2b9dcb4357a8764e9232442ce Mon Sep 17 00:00:00 2001 From: Kathy Tran Date: Thu, 21 Apr 2022 10:36:30 -0400 Subject: [PATCH 04/47] Enable xtrace for install_bootstrap --- install_bootstrap | 1 + 1 file changed, 1 insertion(+) diff --git a/install_bootstrap b/install_bootstrap index 86d4d1d..a90cdde 100755 --- a/install_bootstrap +++ b/install_bootstrap @@ -6,6 +6,7 @@ set -o errexit set -o pipefail set -o nounset +set -o xtrace # When changing these versions, you will probably need to delete all files in the working directory From 11b1b5dcfa92ebc96250ee0d50c50201fe5860f7 Mon Sep 17 00:00:00 2001 From: Kathy Tran Date: Thu, 21 Apr 2022 13:50:25 -0400 Subject: [PATCH 05/47] Revert "Enable xtrace for install_bootstrap" This reverts commit eb33c4204f1fccc2b9dcb4357a8764e9232442ce. --- install_bootstrap | 1 - 1 file changed, 1 deletion(-) diff --git a/install_bootstrap b/install_bootstrap index a90cdde..86d4d1d 100755 --- a/install_bootstrap +++ b/install_bootstrap @@ -6,7 +6,6 @@ set -o errexit set -o pipefail set -o nounset -set -o xtrace # When changing these versions, you will probably need to delete all files in the working directory From aa0cbdcc5a79badc96414de794e02d4ee15780e4 Mon Sep 17 00:00:00 2001 From: David Steinberg Date: Tue, 26 Apr 2022 09:53:14 -0700 Subject: [PATCH 06/47] Add security policy --- SECURITY.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 SECURITY.md diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..a813608 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,21 @@ +# Security Policy + +## Supported Versions + +The Dockstore system routinely receives security updates to the most recently +released tagged minor version. All previous versions are unsupported. + +## Reporting a Vulnerability + +Users are able to open helpdesk tickets on [Discourse](https://discuss.dockstore.org/). Users can create helpdesk tickets in case of privacy complaints, security vulnerabilities, or any other urgent matter related to Dockstore. Helpdesk tickets will be addressed by Dockstore administrators. + +The following steps can be taken to create a helpdesk ticket (also shown [here](https://discuss.dockstore.org/t/opening-helpdesk-tickets/1506)). + +1. Navigate to [Discourse](https://discuss.dockstore.org/) and login. +2. Select your profile icon, located in the top right corner of the screen. +3. Select the `mail` icon, located in the dropdown. +4. Send a message to the `dockstore_admins` group. + +Note + +> If you are unable to see a New Message button on the mail page, you may be considered a new user and have insufficient privileges. Entering 5 topics and viewing 30 posts over a minimum of 10 minutes will raise your privileges. You will be notified of any privilege changes to your account via the mailbox. From 83e347b59523673ecb0f6040f54ff7bdbf147456 Mon Sep 17 00:00:00 2001 From: David Steinberg Date: Tue, 26 Apr 2022 11:09:15 -0700 Subject: [PATCH 07/47] Add the ability to specify the full directory --- .gitallowed | 1 + scripts/tests/test-webservice-image-digest.py | 7 +++++ scripts/webservice-image-digest.py | 26 ++++++++++++++----- 3 files changed, 28 insertions(+), 6 deletions(-) diff --git a/.gitallowed b/.gitallowed index 6d0f543..4b944c6 100644 --- a/.gitallowed +++ b/.gitallowed @@ -1,3 +1,4 @@ #initial .gitallowed, in the future this is where git-secrets false-positives should be added scripts/tests/test-webservice-image-digest.py:.*sha256 +scripts/webservice-image-digest.py:.*sha256 diff --git a/scripts/tests/test-webservice-image-digest.py b/scripts/tests/test-webservice-image-digest.py index f2ddae8..091aaf3 100644 --- a/scripts/tests/test-webservice-image-digest.py +++ b/scripts/tests/test-webservice-image-digest.py @@ -16,6 +16,7 @@ base_command = "python {}".format(script_location) branch = "develop" +full_directory = "develop-b667562" simple_tag = "digest_test" annotated_tag = "1.12.0-beta.1" @@ -28,6 +29,12 @@ class TestDigest(unittest.TestCase): # self.assertEqual(ret, "sha256:52cf6b09e89a238bfd1d98dd01139442d67fcaaa377c179f315dd06555f7bcae") # pass + def test_full_directory(self): + cmd = "{} {}".format(base_command, full_directory) + ret = subprocess.check_output(cmd, shell=True, universal_newlines=True).rstrip() + self.assertEqual(ret, "sha256:08c67131daf6109fadb19d994d753ede7ae28e41c675322e2980327597bcb665") + pass + def test_simple_tag(self): cmd = "{} {}".format(base_command, simple_tag) ret = subprocess.check_output(cmd, shell=True, universal_newlines=True).rstrip() diff --git a/scripts/webservice-image-digest.py b/scripts/webservice-image-digest.py index 3acf7c3..07fc435 100755 --- a/scripts/webservice-image-digest.py +++ b/scripts/webservice-image-digest.py @@ -6,6 +6,14 @@ images by digest as opposed to by tag and will also guarantee the image pulled from Quay has not been changed since being built. +Assets in S3 follow the directory structure: `./branch-shortcommit/image-digest.txt` + +Provide a git tag, branch, or branch-shorthash + +The output is formatted to be easily used to select a specific image digest via docker + +sha256:08c67131daf6109fadb19d994d753ede7ae28e41c675322e2980327597bcb665 + """ import argparse @@ -14,7 +22,7 @@ parser = argparse.ArgumentParser( description='Gather an image digest for the Dockstore Webservice from S3 as created by CircleCI') parser.add_argument('tag', type=str, - help='The git tag (or branch)') + help='The git tag, branch, or branch-hash of a Webservice commit') args = parser.parse_args() @@ -46,12 +54,12 @@ def get_commit_from_github(tag_or_branch): print("No commit for that tag or branch found!") exit(1) -def get_digest_from_s3(tag, commit): +def get_digest_from_s3(directory): # downloads the image-digest.txt from a directory in S3 base_url = "https://gui.dockstore.org" - response = requests.get("{}/{}-{}/image-digest.txt".format(base_url, tag, commit[0:7])) + response = requests.get("{}/{}/image-digest.txt".format(base_url, directory)) if (response.status_code != 200): - print("Expected a file at {}".format("{}/{}-{}/image-digest.txt".format(base_url, tag, commit[0:7]))) + print("Expected a file at {}".format("{}/{}/image-digest.txt".format(base_url, directory))) print("The image-digest.txt was not found in S3, did the build succeed?") exit(1) # There is a newline at the end of the file we rstrip @@ -59,7 +67,13 @@ def get_digest_from_s3(tag, commit): if __name__ == "__main__": # slashes are replaced with _ in docker image tags - commit = get_commit_from_github(args.tag) - circle_digest = get_digest_from_s3(args.tag, commit) + # check to see if input includes a dash followed by 7 chars + parsed = args.tag.split('-') + if len(parsed) == 2 and len(parsed[1]) == 7: + directory = args.tag + else: + commit = get_commit_from_github(args.tag) + directory = "{}-{}".format(args.tag, commit[0:7]) + circle_digest = get_digest_from_s3(directory) print("sha256:{}".format(circle_digest)) exit(0) From 660350403fe419bc2ae6ec56ff2f6bc712a9f516 Mon Sep 17 00:00:00 2001 From: David Steinberg Date: Thu, 28 Apr 2022 11:26:39 -0700 Subject: [PATCH 08/47] Check for hex in branch-shorthash combination --- scripts/webservice-image-digest.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/webservice-image-digest.py b/scripts/webservice-image-digest.py index 07fc435..1d7dbe6 100755 --- a/scripts/webservice-image-digest.py +++ b/scripts/webservice-image-digest.py @@ -18,6 +18,7 @@ import argparse import requests +import string parser = argparse.ArgumentParser( description='Gather an image digest for the Dockstore Webservice from S3 as created by CircleCI') @@ -69,7 +70,7 @@ def get_digest_from_s3(directory): # slashes are replaced with _ in docker image tags # check to see if input includes a dash followed by 7 chars parsed = args.tag.split('-') - if len(parsed) == 2 and len(parsed[1]) == 7: + if len(parsed) == 2 and len(parsed[1]) == 7 and all(c in string.hexdigits for c in parsed[1]): directory = args.tag else: commit = get_commit_from_github(args.tag) From 274e958bda2b7717bb1fb131e6ce5d2fadc9a3ab Mon Sep 17 00:00:00 2001 From: Kathy Tran Date: Wed, 11 May 2022 16:04:46 -0400 Subject: [PATCH 09/47] Add IS_FARGATE_DEPLOY variable --- install_bootstrap | 49 ++++++++++++++++------ templates/default.nginx_http.conf.template | 10 +++++ 2 files changed, 46 insertions(+), 13 deletions(-) diff --git a/install_bootstrap b/install_bootstrap index 86d4d1d..02dbf3a 100755 --- a/install_bootstrap +++ b/install_bootstrap @@ -20,15 +20,36 @@ MSG function template() { mkdir -p config + + NGINX_CONF_DIRECTORY="config" + NGINX_HTML2_DIRECTORY="config" + WEBSERVICE_DIRECTORY="config" + + if [ $IS_FARGATE_DEPLOY == "true" ]; then + mkdir -p config/webservice + mkdir -p config/nginx-conf + mkdir -p config/nginx-html2 + # Place the config files into specific directories so they can be mounted to container paths as bind mounts + NGINX_CONF_DIRECTORY="${NGINX_CONF_DIRECTORY}/nginx-conf" + NGINX_HTML2_DIRECTORY="${NGINX_HTML2_DIRECTORY}/nginx-html2" + WEBSERVICE_DIRECTORY="${WEBSERVICE_DIRECTORY}/webservice" + + wget -qO ${NGINX_HTML2_DIRECTORY}/index.html https://gui.dockstore.org/${UI2_HASH}/index.html + wget -qO ${NGINX_HTML2_DIRECTORY}/manifest.json https://gui.dockstore.org/${UI2_HASH}/manifest.json + fi + mustache dockstore_launcher_config/compose.config templates/Dockerfile_ui2.template > config/Dockerfile_ui2 - mustache dockstore_launcher_config/compose.config templates/robots.txt.template > config/robots.txt - mustache dockstore_launcher_config/compose.config templates/web.yml.template > config/web.yml - mustache dockstore_launcher_config/compose.config templates/default.nginx_http.conf.template > config/default.nginx_http.conf - mustache dockstore_launcher_config/compose.config templates/default.nginx_http.shared.conf.template > config/default.nginx_http.shared.conf - mustache dockstore_launcher_config/compose.config templates/default.nginx_http.security.conf.template > config/default.nginx_http.security.conf - - mustache dockstore_launcher_config/compose.config templates/init_webservice.sh.template > config/init_webservice.sh - mustache dockstore_launcher_config/compose.config templates/init_migration.sh.template > config/init_migration.sh + mustache dockstore_launcher_config/compose.config templates/robots.txt.template > ${NGINX_HTML2_DIRECTORY}/robots.txt + mustache dockstore_launcher_config/compose.config templates/default.nginx_http.conf.template > ${NGINX_CONF_DIRECTORY}/default.nginx_http.conf + mustache dockstore_launcher_config/compose.config templates/default.nginx_http.shared.conf.template > ${NGINX_CONF_DIRECTORY}/default.nginx_http.shared.conf + mustache dockstore_launcher_config/compose.config templates/default.nginx_http.security.conf.template > ${NGINX_CONF_DIRECTORY}/default.nginx_http.security.conf + + mustache dockstore_launcher_config/compose.config templates/web.yml.template > ${WEBSERVICE_DIRECTORY}/web.yml + mustache dockstore_launcher_config/compose.config templates/init_webservice.sh.template > ${WEBSERVICE_DIRECTORY}/init_webservice.sh + mustache dockstore_launcher_config/compose.config templates/init_migration.sh.template > ${WEBSERVICE_DIRECTORY}/init_migration.sh + chmod a+rx ${WEBSERVICE_DIRECTORY}/init_webservice.sh + chmod a+rx ${WEBSERVICE_DIRECTORY}/init_migration.sh + mustache dockstore_launcher_config/compose.config templates/elasticsearch.yml > config/elasticsearch.yml mustache dockstore_launcher_config/compose.config templates/metricbeat.yml > config/metricbeat.yml mustache dockstore_launcher_config/compose.config templates/essnapshot_backup.sh > scripts/essnapshot_backup.sh @@ -66,11 +87,13 @@ template download_galaxy "$*" -# We need to set the environment variable for the image digest -source .env -DOCKSTORE_IMAGE_DIGEST=$(scripts/webservice-image-digest.py $DOCKSTORE_VERSION) -echo "DOCKSTORE_IMAGE_DIGEST=$DOCKSTORE_IMAGE_DIGEST" >> .env +if [ $IS_FARGATE_DEPLOY == "false" ]; then + # We need to set the environment variable for the image digest + source .env + DOCKSTORE_IMAGE_DIGEST=$(scripts/webservice-image-digest.py $DOCKSTORE_VERSION) + echo "DOCKSTORE_IMAGE_DIGEST=$DOCKSTORE_IMAGE_DIGEST" >> .env -docker-compose build + docker-compose build +fi echo "Exiting now." diff --git a/templates/default.nginx_http.conf.template b/templates/default.nginx_http.conf.template index 8e62d2a..a46d140 100644 --- a/templates/default.nginx_http.conf.template +++ b/templates/default.nginx_http.conf.template @@ -26,7 +26,12 @@ access_log off; resolver 127.0.0.11 valid=10s; server { + {{#IS_FARGATE_DEPLOY}} + set $webservice "localhost" + {{/IS_FARGATE_DEPLOY}} + {{^IS_FARGATE_DEPLOY}} set $webservice "webservice"; + {{/IS_FARGATE_DEPLOY}} server_name .{{ DOMAIN_NAME }}; include /etc/nginx/conf.d/default.nginx_http.shared.conf; include /etc/nginx/conf.d/default.nginx_http.security.conf; @@ -102,7 +107,12 @@ server { # Server block for access via IP instead of domain server { + {{#IS_FARGATE_DEPLOY}} + set $webservice "localhost" + {{/IS_FARGATE_DEPLOY}} + {{^IS_FARGATE_DEPLOY}} set $webservice "webservice"; + {{/IS_FARGATE_DEPLOY}} access_log /var/log/nginx/access.log custom; listen 4200 default_server; From 4bebcb987eea9b043d634f30b2dae6cd0b5b64ab Mon Sep 17 00:00:00 2001 From: Kathy Tran Date: Wed, 11 May 2022 16:09:06 -0400 Subject: [PATCH 10/47] Add IS_FARGATE_DEPLOY to compose.config --- dockstore_launcher_config/compose.config | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dockstore_launcher_config/compose.config b/dockstore_launcher_config/compose.config index fe485bd..906ef38 100644 --- a/dockstore_launcher_config/compose.config +++ b/dockstore_launcher_config/compose.config @@ -51,5 +51,6 @@ "CWL_PARSING_LAMBDA_VERSION":"n/a", "WDL_PARSING_LAMBDA_VERSION":"n/a", "NEXTFLOW_PARSING_LAMBDA_VERSION":"n/a", -"CHECK_URL_LAMBDA_VERSION":"n/a" +"CHECK_URL_LAMBDA_VERSION":"n/a", +"IS_FARGATE_DEPLOY":false } From ba98239cdcfeed42b706963c4cd4068aace86071 Mon Sep 17 00:00:00 2001 From: Kathy Tran Date: Wed, 11 May 2022 16:23:47 -0400 Subject: [PATCH 11/47] Specify absolute path for webservice jar --- templates/init_migration.sh.template | 8 ++++---- templates/init_webservice.sh.template | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/templates/init_migration.sh.template b/templates/init_migration.sh.template index a23a8d7..afb5824 100755 --- a/templates/init_migration.sh.template +++ b/templates/init_migration.sh.template @@ -4,12 +4,12 @@ cd "$(dirname "$0")" {{#DATABASE_GENERATED}} -java -Ddw.database.user=postgres -Ddw.database.password="{{{ POSTGRES_DBPASSWORD }}}" -jar dockstore-webservice-*.jar db migrate web.yml --include 1.3.0.generated,1.3.1.consistency,1.4.0,1.5.0,1.6.0,1.7.0 | tee --append /dockstore_logs/webservice.out +java -Ddw.database.user=postgres -Ddw.database.password="{{{ POSTGRES_DBPASSWORD }}}" -jar /home/dockstore-webservice-*.jar db migrate web.yml --include 1.3.0.generated,1.3.1.consistency,1.4.0,1.5.0,1.6.0,1.7.0 | tee --append /dockstore_logs/webservice.out {{/DATABASE_GENERATED}} {{^DATABASE_GENERATED}} -java -Ddw.database.user=postgres -Ddw.database.password="{{{ POSTGRES_DBPASSWORD }}}" -jar dockstore-webservice-*.jar db migrate web.yml --include 1.3.1.consistency,1.4.0,1.5.0,1.6.0,1.7.0 | tee --append /dockstore_logs/webservice.out +java -Ddw.database.user=postgres -Ddw.database.password="{{{ POSTGRES_DBPASSWORD }}}" -jar /home/dockstore-webservice-*.jar db migrate web.yml --include 1.3.1.consistency,1.4.0,1.5.0,1.6.0,1.7.0 | tee --append /dockstore_logs/webservice.out {{/DATABASE_GENERATED}} # this particular migration needs to run as postgres because only postgres can surrender ownership -java -Ddw.database.user=postgres -Ddw.database.password="{{{ POSTGRES_DBPASSWORD }}}" -jar dockstore-webservice-*.jar db migrate web.yml --include 1.7.0.relinquish +java -Ddw.database.user=postgres -Ddw.database.password="{{{ POSTGRES_DBPASSWORD }}}" -jar /home/dockstore-webservice-*.jar db migrate web.yml --include 1.7.0.relinquish # future migrations will start here and should be run as dockstore -java -Ddw.database.user=dockstore -Ddw.database.password="{{{ DOCKSTORE_DBPASSWORD }}}" -jar dockstore-webservice-*.jar db migrate web.yml --include 1.8.0,1.9.0,1.10.0,1.11.0,1.12.0,1.13.0 | tee --append /dockstore_logs/webservice.out +java -Ddw.database.user=dockstore -Ddw.database.password="{{{ DOCKSTORE_DBPASSWORD }}}" -jar /home/dockstore-webservice-*.jar db migrate web.yml --include 1.8.0,1.9.0,1.10.0,1.11.0,1.12.0,1.13.0 | tee --append /dockstore_logs/webservice.out diff --git a/templates/init_webservice.sh.template b/templates/init_webservice.sh.template index 49e690e..6ea6adc 100755 --- a/templates/init_webservice.sh.template +++ b/templates/init_webservice.sh.template @@ -2,6 +2,6 @@ cd "$(dirname "$0")" -java -XX:MaxRAMPercentage=50.0 -XX:+ExitOnOutOfMemoryError -jar dockstore-webservice-*.jar server web.yml | tee --append /dockstore_logs/webservice.out +java -XX:MaxRAMPercentage=50.0 -XX:+ExitOnOutOfMemoryError -jar /home/dockstore-webservice-*.jar server web.yml | tee --append /dockstore_logs/webservice.out From f34ce658e1cdc8d6a8e8fd4c80adb7cd7e385fec Mon Sep 17 00:00:00 2001 From: Kathy Tran Date: Wed, 11 May 2022 16:46:52 -0400 Subject: [PATCH 12/47] Add missing semi-colon --- templates/default.nginx_http.conf.template | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/default.nginx_http.conf.template b/templates/default.nginx_http.conf.template index a46d140..87290c1 100644 --- a/templates/default.nginx_http.conf.template +++ b/templates/default.nginx_http.conf.template @@ -27,7 +27,7 @@ resolver 127.0.0.11 valid=10s; server { {{#IS_FARGATE_DEPLOY}} - set $webservice "localhost" + set $webservice "localhost"; {{/IS_FARGATE_DEPLOY}} {{^IS_FARGATE_DEPLOY}} set $webservice "webservice"; @@ -108,7 +108,7 @@ server { # Server block for access via IP instead of domain server { {{#IS_FARGATE_DEPLOY}} - set $webservice "localhost" + set $webservice "localhost"; {{/IS_FARGATE_DEPLOY}} {{^IS_FARGATE_DEPLOY}} set $webservice "webservice"; From a2bf281b90765f1d5755c9b7e08660436d4c0926 Mon Sep 17 00:00:00 2001 From: Kathy Tran Date: Wed, 11 May 2022 16:53:36 -0400 Subject: [PATCH 13/47] Try without quotes --- templates/default.nginx_http.conf.template | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/default.nginx_http.conf.template b/templates/default.nginx_http.conf.template index 87290c1..155c461 100644 --- a/templates/default.nginx_http.conf.template +++ b/templates/default.nginx_http.conf.template @@ -27,7 +27,7 @@ resolver 127.0.0.11 valid=10s; server { {{#IS_FARGATE_DEPLOY}} - set $webservice "localhost"; + set $webservice localhost; {{/IS_FARGATE_DEPLOY}} {{^IS_FARGATE_DEPLOY}} set $webservice "webservice"; @@ -108,7 +108,7 @@ server { # Server block for access via IP instead of domain server { {{#IS_FARGATE_DEPLOY}} - set $webservice "localhost"; + set $webservice localhost; {{/IS_FARGATE_DEPLOY}} {{^IS_FARGATE_DEPLOY}} set $webservice "webservice"; From a2d145acc3159a046feb98b90cd3b5a3c326f23d Mon Sep 17 00:00:00 2001 From: Kathy Tran Date: Thu, 12 May 2022 09:24:07 -0400 Subject: [PATCH 14/47] Try turning off ipv6 for resolver --- templates/default.nginx_http.conf.template | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/templates/default.nginx_http.conf.template b/templates/default.nginx_http.conf.template index 155c461..5ef30b6 100644 --- a/templates/default.nginx_http.conf.template +++ b/templates/default.nginx_http.conf.template @@ -23,11 +23,11 @@ access_log off; # nginx caches ips on startup, so doesn't survive webservice restarts # https://www.nginx.com/blog/dns-service-discovery-nginx-plus/ # https://stackoverflow.com/questions/46660436/nginx-does-not-automatically-pick-up-dns-changes-in-swarm/46664433#46664433 -resolver 127.0.0.11 valid=10s; +resolver 127.0.0.11 valid=10s ipv6=off; server { {{#IS_FARGATE_DEPLOY}} - set $webservice localhost; + set $webservice "localhost"; {{/IS_FARGATE_DEPLOY}} {{^IS_FARGATE_DEPLOY}} set $webservice "webservice"; @@ -108,7 +108,7 @@ server { # Server block for access via IP instead of domain server { {{#IS_FARGATE_DEPLOY}} - set $webservice localhost; + set $webservice "localhost"; {{/IS_FARGATE_DEPLOY}} {{^IS_FARGATE_DEPLOY}} set $webservice "webservice"; From d7e9a1a286cf753e30000b2e3f2db221c7dcf8b9 Mon Sep 17 00:00:00 2001 From: Kathy Tran Date: Thu, 12 May 2022 10:00:31 -0400 Subject: [PATCH 15/47] Try 127.0.0.1 instead of localhost --- templates/default.nginx_http.conf.template | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/templates/default.nginx_http.conf.template b/templates/default.nginx_http.conf.template index 5ef30b6..8b3c2ae 100644 --- a/templates/default.nginx_http.conf.template +++ b/templates/default.nginx_http.conf.template @@ -23,11 +23,11 @@ access_log off; # nginx caches ips on startup, so doesn't survive webservice restarts # https://www.nginx.com/blog/dns-service-discovery-nginx-plus/ # https://stackoverflow.com/questions/46660436/nginx-does-not-automatically-pick-up-dns-changes-in-swarm/46664433#46664433 -resolver 127.0.0.11 valid=10s ipv6=off; +resolver 127.0.0.11 valid=10s; server { {{#IS_FARGATE_DEPLOY}} - set $webservice "localhost"; + set $webservice "127.0.0.1"; {{/IS_FARGATE_DEPLOY}} {{^IS_FARGATE_DEPLOY}} set $webservice "webservice"; @@ -108,7 +108,7 @@ server { # Server block for access via IP instead of domain server { {{#IS_FARGATE_DEPLOY}} - set $webservice "localhost"; + set $webservice "127.0.0.1"; {{/IS_FARGATE_DEPLOY}} {{^IS_FARGATE_DEPLOY}} set $webservice "webservice"; From 069831e229b99012c7a63f61ab441dd6dcb09732 Mon Sep 17 00:00:00 2001 From: Kathy Tran Date: Tue, 17 May 2022 09:35:31 -0400 Subject: [PATCH 16/47] Make GITHUB_APP_PRIVATE_KEY_FILE configurable --- docker-compose.yml | 2 +- templates/web.yml.template | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 913fc14..2d05159 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -24,7 +24,7 @@ services: - log_volume:/dockstore_logs - ./config/web.yml:/home/web.yml - ./config/init_webservice.sh:/home/init_webservice.sh - - ${GITHUB_APP_PRIVATE_KEY_FILE}:/home/dockstore_github_app_private_key.pem + - ${GITHUB_APP_PRIVATE_KEY_FILE}:${GITHUB_APP_PRIVATE_KEY_FILE} - ../language-plugins/:/root/.dockstore/language-plugins command: ["bash", "/home/init_webservice.sh"] ports: diff --git a/templates/web.yml.template b/templates/web.yml.template index 906b28c..3a143ea 100644 --- a/templates/web.yml.template +++ b/templates/web.yml.template @@ -29,7 +29,7 @@ zenodoUrl: {{ ZENODO_URL }} orcidClientID: {{ ORCID_CLIENT_ID }} orcidClientSecret: {{ ORCID_CLIENT_SECRET }} -gitHubAppPrivateKeyFile: /home/dockstore_github_app_private_key.pem +gitHubAppPrivateKeyFile: {{ GITHUB_APP_PRIVATE_KEY_FILE }} gitHubAppId: {{ GITHUB_APP_ID }} toolTesterBucket: {{ TOOLTESTER_BUCKET_NAME }} From b1f35945631db2c0cc4c21572f043397f04fa020 Mon Sep 17 00:00:00 2001 From: Kathy Tran Date: Wed, 18 May 2022 12:57:10 -0400 Subject: [PATCH 17/47] Don't make gitHubAppPrivateKeyFile configurable. Instead, change the path --- docker-compose.yml | 2 +- templates/web.yml.template | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 2d05159..30f1e73 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -24,7 +24,7 @@ services: - log_volume:/dockstore_logs - ./config/web.yml:/home/web.yml - ./config/init_webservice.sh:/home/init_webservice.sh - - ${GITHUB_APP_PRIVATE_KEY_FILE}:${GITHUB_APP_PRIVATE_KEY_FILE} + - ${GITHUB_APP_PRIVATE_KEY_FILE}:/home/github-key/dockstore_github_app_private_key.pem - ../language-plugins/:/root/.dockstore/language-plugins command: ["bash", "/home/init_webservice.sh"] ports: diff --git a/templates/web.yml.template b/templates/web.yml.template index 3a143ea..c3b8cbf 100644 --- a/templates/web.yml.template +++ b/templates/web.yml.template @@ -29,7 +29,7 @@ zenodoUrl: {{ ZENODO_URL }} orcidClientID: {{ ORCID_CLIENT_ID }} orcidClientSecret: {{ ORCID_CLIENT_SECRET }} -gitHubAppPrivateKeyFile: {{ GITHUB_APP_PRIVATE_KEY_FILE }} +gitHubAppPrivateKeyFile: /home/github-key/dockstore_github_app_private_key.pem gitHubAppId: {{ GITHUB_APP_ID }} toolTesterBucket: {{ TOOLTESTER_BUCKET_NAME }} From 7526400c1df8249eeb57b46ce9d775844ba34c96 Mon Sep 17 00:00:00 2001 From: Kathy Tran Date: Wed, 18 May 2022 14:38:01 -0400 Subject: [PATCH 18/47] Use dashses instead of underscores for gitHubAppPrivateKeyFile --- docker-compose.yml | 2 +- templates/web.yml.template | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 30f1e73..ba87615 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -24,7 +24,7 @@ services: - log_volume:/dockstore_logs - ./config/web.yml:/home/web.yml - ./config/init_webservice.sh:/home/init_webservice.sh - - ${GITHUB_APP_PRIVATE_KEY_FILE}:/home/github-key/dockstore_github_app_private_key.pem + - ${GITHUB_APP_PRIVATE_KEY_FILE}:/home/github-key/dockstore-github-app-private-key.pem - ../language-plugins/:/root/.dockstore/language-plugins command: ["bash", "/home/init_webservice.sh"] ports: diff --git a/templates/web.yml.template b/templates/web.yml.template index c3b8cbf..c8d6418 100644 --- a/templates/web.yml.template +++ b/templates/web.yml.template @@ -29,7 +29,7 @@ zenodoUrl: {{ ZENODO_URL }} orcidClientID: {{ ORCID_CLIENT_ID }} orcidClientSecret: {{ ORCID_CLIENT_SECRET }} -gitHubAppPrivateKeyFile: /home/github-key/dockstore_github_app_private_key.pem +gitHubAppPrivateKeyFile: /home/github-key/dockstore-github-app-private-key.pem gitHubAppId: {{ GITHUB_APP_ID }} toolTesterBucket: {{ TOOLTESTER_BUCKET_NAME }} From 3614fdc30ee1c1801ae82428b38e77f76a0fae04 Mon Sep 17 00:00:00 2001 From: Kathy Tran Date: Wed, 18 May 2022 15:11:21 -0400 Subject: [PATCH 19/47] Remove app from github private key name --- docker-compose.yml | 2 +- templates/web.yml.template | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index ba87615..7e13f80 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -24,7 +24,7 @@ services: - log_volume:/dockstore_logs - ./config/web.yml:/home/web.yml - ./config/init_webservice.sh:/home/init_webservice.sh - - ${GITHUB_APP_PRIVATE_KEY_FILE}:/home/github-key/dockstore-github-app-private-key.pem + - ${GITHUB_APP_PRIVATE_KEY_FILE}:/home/github-key/dockstore-github-private-key.pem - ../language-plugins/:/root/.dockstore/language-plugins command: ["bash", "/home/init_webservice.sh"] ports: diff --git a/templates/web.yml.template b/templates/web.yml.template index c8d6418..5d02883 100644 --- a/templates/web.yml.template +++ b/templates/web.yml.template @@ -29,7 +29,7 @@ zenodoUrl: {{ ZENODO_URL }} orcidClientID: {{ ORCID_CLIENT_ID }} orcidClientSecret: {{ ORCID_CLIENT_SECRET }} -gitHubAppPrivateKeyFile: /home/github-key/dockstore-github-app-private-key.pem +gitHubAppPrivateKeyFile: /home/github-key/dockstore-github-private-key.pem gitHubAppId: {{ GITHUB_APP_ID }} toolTesterBucket: {{ TOOLTESTER_BUCKET_NAME }} From 587358c473dc0683dd9c8c9c8cd497dfc7c2c5e6 Mon Sep 17 00:00:00 2001 From: Kathy Tran Date: Tue, 24 May 2022 14:18:33 -0400 Subject: [PATCH 20/47] Use /dockstore dir instead of /home for github key --- docker-compose.yml | 2 +- templates/web.yml.template | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 7e13f80..1d4776d 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -24,7 +24,7 @@ services: - log_volume:/dockstore_logs - ./config/web.yml:/home/web.yml - ./config/init_webservice.sh:/home/init_webservice.sh - - ${GITHUB_APP_PRIVATE_KEY_FILE}:/home/github-key/dockstore-github-private-key.pem + - ${GITHUB_APP_PRIVATE_KEY_FILE}:/dockstore/github-key/dockstore-github-private-key.pem - ../language-plugins/:/root/.dockstore/language-plugins command: ["bash", "/home/init_webservice.sh"] ports: diff --git a/templates/web.yml.template b/templates/web.yml.template index 5d02883..db603a0 100644 --- a/templates/web.yml.template +++ b/templates/web.yml.template @@ -29,7 +29,7 @@ zenodoUrl: {{ ZENODO_URL }} orcidClientID: {{ ORCID_CLIENT_ID }} orcidClientSecret: {{ ORCID_CLIENT_SECRET }} -gitHubAppPrivateKeyFile: /home/github-key/dockstore-github-private-key.pem +gitHubAppPrivateKeyFile: /dockstore/github-key/dockstore-github-private-key.pem gitHubAppId: {{ GITHUB_APP_ID }} toolTesterBucket: {{ TOOLTESTER_BUCKET_NAME }} From de904e4ba537dd392ec16b3137cfc09676493a4d Mon Sep 17 00:00:00 2001 From: David Steinberg Date: Fri, 24 Jun 2022 13:28:06 -0700 Subject: [PATCH 21/47] Add more trusted CSP domains --- templates/default.nginx_http.security.conf.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/default.nginx_http.security.conf.template b/templates/default.nginx_http.security.conf.template index c1d280d..e692a95 100644 --- a/templates/default.nginx_http.security.conf.template +++ b/templates/default.nginx_http.security.conf.template @@ -18,7 +18,7 @@ add_header X-XSS-Protection "1; mode=block" always; add_header Referrer-Policy "strict-origin-when-cross-origin" always; # Explicitly list domains allowed to serve content for this site -add_header Content-Security-Policy-Report-Only "report-uri https://api.dockstore-security.org/csp-report; default-src 'self'; object-src 'none'; base-uri 'self'; manifest-src 'self'; media-src 'self'; worker-src 'none'; script-src 'report-sample' 'self' 'unsafe-hashes' 'unsafe-inline' 'unsafe-eval' discuss.dockstore.org gui.dockstore.org *.twitter.com *.twimg.com www.google-analytics.com www.googletagmanager.com; style-src 'report-sample' 'self' 'unsafe-inline' cdnjs.cloudflare.com fonts.googleapis.com *.twitter.com *.twimg.com gui.dockstore.org; connect-src 'self' s3.amazonaws.com api.github.com view.commonwl.org www.google-analytics.com gui.dockstore.org; font-src 'self' fonts.gstatic.com gui.dockstore.org; frame-src 'self' discuss.dockstore.org platform.twitter.com; img-src data: 'self' avatars0.githubusercontent.com avatars1.githubusercontent.com avatars2.githubusercontent.com avatars3.githubusercontent.com camo.githubusercontent.com gui.dockstore.org i.imgur.com api.travis-ci.com img.shields.io quay.io via.placeholder.com *.wp.com *.googleusercontent.com www.googletagmanager.com www.google-analytics.com www.gravatar.com *.twitter.com *.twimg.com;" always; +add_header Content-Security-Policy-Report-Only "report-uri https://api.dockstore-security.org/csp-report; default-src 'self'; object-src 'none'; base-uri 'self'; manifest-src 'self' dockstore.org; media-src 'self'; worker-src 'none'; script-src 'report-sample' 'self' 'unsafe-hashes' 'unsafe-inline' 'unsafe-eval' discuss.dockstore.org gui.dockstore.org *.twitter.com *.twimg.com www.google-analytics.com www.googletagmanager.com; style-src 'report-sample' 'self' 'unsafe-inline' cdnjs.cloudflare.com fonts.googleapis.com *.twitter.com *.twimg.com gui.dockstore.org; connect-src 'self' s3.amazonaws.com api.github.com view.commonwl.org www.google-analytics.com content.dockstore.org gui.dockstore.org; font-src 'self' fonts.gstatic.com gui.dockstore.org; frame-src 'self' discuss.dockstore.org platform.twitter.com youtube.com; img-src data: 'self' user-images.githubusercontent.com avatars.githubusercontent.com avatars0.githubusercontent.com avatars1.githubusercontent.com avatars2.githubusercontent.com avatars3.githubusercontent.com camo.githubusercontent.com circleci.com gui.dockstore.org gstatic.com i.imgur.com api.travis-ci.com travis-ci.com img.shields.io quay.io via.placeholder.com *.wp.com *.googleusercontent.com www.googletagmanager.com www.google-analytics.com www.gravatar.com *.twitter.com i.ytimg.com *.twimg.com zenodo.org;" always; # Hide server header proxy_hide_header Server; From dc4ecee069ba4db0218ae968a2d8cbc4eec721f8 Mon Sep 17 00:00:00 2001 From: Kathy Tran Date: Tue, 28 Jun 2022 10:50:08 -0400 Subject: [PATCH 22/47] Remove unused keys --- dockstore_launcher_config/compose.config | 3 --- 1 file changed, 3 deletions(-) diff --git a/dockstore_launcher_config/compose.config b/dockstore_launcher_config/compose.config index 906ef38..a598e08 100644 --- a/dockstore_launcher_config/compose.config +++ b/dockstore_launcher_config/compose.config @@ -12,7 +12,6 @@ "DOMAIN_NAME":"foobar", "HTTPS":false, "COMPOSE_SETUP_VERSION":"foo", -"DEPLOY_COMMIT_ID":"foo", "DISCOURSE_URL":"foobar", "DISCOURSE_KEY":"foobar", "DISCOURSE_CATEGORY_ID":"10", @@ -46,8 +45,6 @@ "SLACK_URL":"replaeceme", "GALAXY_PLUGIN_VERSION":"0.0.6", "DATABASE_DOMAIN":"postgres", -"DBUSER":"dockstore", -"DBPASSWORD":"dockstore", "CWL_PARSING_LAMBDA_VERSION":"n/a", "WDL_PARSING_LAMBDA_VERSION":"n/a", "NEXTFLOW_PARSING_LAMBDA_VERSION":"n/a", From f7883d9dd0b36423133191929a1d57657e2771eb Mon Sep 17 00:00:00 2001 From: Kathy Tran Date: Tue, 28 Jun 2022 10:51:59 -0400 Subject: [PATCH 23/47] Fix checkUrlLambdaVersion typo in web.yml --- templates/web.yml.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/web.yml.template b/templates/web.yml.template index db603a0..3e3ef2b 100644 --- a/templates/web.yml.template +++ b/templates/web.yml.template @@ -198,5 +198,5 @@ uiConfig: wdlParsingLambdaVersion: {{ WDL_PARSING_LAMBDA_VERSION }} nextflowParsingLambdaVersion: {{ NEXTFLOW_PARSING_LAMBDA_VERSION }} galaxyParsingPluginVersion: {{ GALAXY_PLUGIN_VERSION }} - checkUrlLambdaVersion: {{ CHECK_URL_LAMDBA_VERSION }} + checkUrlLambdaVersion: {{ CHECK_URL_LAMBDA_VERSION }} From 40263e4e66210742e1a8f3a7f70e3eaf02cac43e Mon Sep 17 00:00:00 2001 From: Kathy Tran Date: Tue, 28 Jun 2022 11:04:02 -0400 Subject: [PATCH 24/47] Add missing keys --- dockstore_launcher_config/compose.config | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/dockstore_launcher_config/compose.config b/dockstore_launcher_config/compose.config index a598e08..8af742f 100644 --- a/dockstore_launcher_config/compose.config +++ b/dockstore_launcher_config/compose.config @@ -39,6 +39,7 @@ "ORCID_CLIENT_ID":"replaceme", "ORCID_CLIENT_SECRET":"replaceme", "ORCID_URL":"replaceme", +"ORCID_SCOPE":"replaceme", "ZENODO_CLIENT_ID":"replaceme", "ZENODO_CLIENT_SECRET":"replaceme", "ZENODO_URL":"replaceme", @@ -49,5 +50,15 @@ "WDL_PARSING_LAMBDA_VERSION":"n/a", "NEXTFLOW_PARSING_LAMBDA_VERSION":"n/a", "CHECK_URL_LAMBDA_VERSION":"n/a", -"IS_FARGATE_DEPLOY":false +"IS_FARGATE_DEPLOY":false, +"DEPLOY_VERSION":"replaceme", +"DOCKSTORE_DBPASSWORD":"replaceme", +"ELASTICSEARCH_DOMAIN":"replaceme", +"ELASTICSEARCH_PROTOCOL":"http", +"ELASTICSEARCH_PORT":"80", +"ELASTICSEARCH_USER":"replaceme", +"ELASTICSEARCH_PASSWORD":"replaceme", +"ELASTICSEARCH_MAX_CONCURRENT_SESSIONS":"123", +"FEATURED_NEWS_URL":"replaceme", +"POSTGRES_DBPASSWORD":"replaceme" } From 680d2fe71e78294af60215b63386a7aab25c5d24 Mon Sep 17 00:00:00 2001 From: Kathy Tran Date: Tue, 28 Jun 2022 11:17:08 -0400 Subject: [PATCH 25/47] Sort alphabetically --- dockstore_launcher_config/compose.config | 84 ++++++++++++------------ 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/dockstore_launcher_config/compose.config b/dockstore_launcher_config/compose.config index 8af742f..7874b6c 100644 --- a/dockstore_launcher_config/compose.config +++ b/dockstore_launcher_config/compose.config @@ -1,64 +1,64 @@ { -"PUBLIC_LAUNCHER_IP_ADDRESS":"", -"PRODUCTION":false, -"DOCKSTORE_VERSION":"develop", -"UI2_HASH":"develop", -"GITHUB_CLIENT2_ID":"foobared", -"GITHUB_CLIENT2_SECRET":"foobared", -"QUAY_CLIENT_ID":"foobared", -"QUAY_CLIENT_SECRET":"foobared", +"AUTHORIZER_TYPE":"replaceme", +"BD_CATALYST_SEVEN_BRIDGES_IMPORT_URL":"replaceme", +"BD_CATALYST_TERRA_IMPORT_URL":"replaceme", "BITBUCKET_CLIENT_ID":"foobared", "BITBUCKET_CLIENT_SECRET":"foobared", -"DOMAIN_NAME":"foobar", -"HTTPS":false, +"CHECK_URL_EXISTS_ENDPOINT":"replaceme", +"CHECK_URL_LAMBDA_VERSION":"n/a", "COMPOSE_SETUP_VERSION":"foo", -"DISCOURSE_URL":"foobar", -"DISCOURSE_KEY":"foobar", +"CWL_PARSING_LAMBDA_VERSION":"n/a", +"DATABASE_DOMAIN":"postgres", +"DATABASE_GENERATED":false, +"DEPLOY_VERSION":"replaceme", "DISCOURSE_CATEGORY_ID":"10", +"DISCOURSE_KEY":"foobar", +"DISCOURSE_URL":"foobar", +"DOCKSTORE_DBPASSWORD":"replaceme", +"DOCKSTORE_VERSION":"develop", "DOCUMENTATION_URL":"foobar", +"DOMAIN_NAME":"foobar", +"ELASTICSEARCH_DOMAIN":"replaceme", +"ELASTICSEARCH_MAX_CONCURRENT_SESSIONS":"123", +"ELASTICSEARCH_PASSWORD":"replaceme", +"ELASTICSEARCH_PORT":"80", +"ELASTICSEARCH_PROTOCOL":"http", +"ELASTICSEARCH_USER":"replaceme", +"EXTERNAL_GOOGLE_CLIENT_PREFIX1":"replaceme", "FEATURED_CONTENT_URL":"foobar", -"GITHUB_APP_PRIVATE_KEY_FILE": "/replaceme", +"FEATURED_NEWS_URL":"replaceme", +"GALAXY_PLUGIN_VERSION":"0.0.6", "GITHUB_APP_ID": "1234", "GITHUB_APP_NAME": "foobar", -"TOOLTESTER_BUCKET_NAME": "replaceme", -"CHECK_URL_EXISTS_ENDPOINT":"replaceme", -"TAG_MANAGER_ID":"foobar", +"GITHUB_APP_PRIVATE_KEY_FILE": "/replaceme", +"GITHUB_CLIENT2_ID":"foobared", +"GITHUB_CLIENT2_SECRET":"foobared", "GITLAB_CLIENT_ID":"foobar", "GITLAB_CLIENT_SECRET":"foobar", "GOOGLE_CLIENT_ID":"potato", "GOOGLE_CLIENT_SECRET":"potato", +"HTTPS":false, +"IS_FARGATE_DEPLOY":false, "LOGSTASH":false, "LOGSTASH_HOST":"replaceme", -"TERRA_IMPORT_URL":"replaceme", -"BD_CATALYST_SEVEN_BRIDGES_IMPORT_URL":"replaceme", -"BD_CATALYST_TERRA_IMPORT_URL":"replaceme", -"AUTHORIZER_TYPE":"replaceme", -"EXTERNAL_GOOGLE_CLIENT_PREFIX1":"replaceme", -"SAM_PATH":"replaceme", -"DATABASE_GENERATED":false, +"NEXTFLOW_PARSING_LAMBDA_VERSION":"n/a", "ORCID_CLIENT_ID":"replaceme", "ORCID_CLIENT_SECRET":"replaceme", -"ORCID_URL":"replaceme", "ORCID_SCOPE":"replaceme", +"ORCID_URL":"replaceme", +"POSTGRES_DBPASSWORD":"replaceme" +"PRODUCTION":false, +"PUBLIC_LAUNCHER_IP_ADDRESS":"", +"QUAY_CLIENT_ID":"foobared", +"QUAY_CLIENT_SECRET":"foobared", +"SAM_PATH":"replaceme", +"SLACK_URL":"replaeceme", +"TAG_MANAGER_ID":"foobar", +"TERRA_IMPORT_URL":"replaceme", +"TOOLTESTER_BUCKET_NAME": "replaceme", +"UI2_HASH":"develop", +"WDL_PARSING_LAMBDA_VERSION":"n/a", "ZENODO_CLIENT_ID":"replaceme", "ZENODO_CLIENT_SECRET":"replaceme", "ZENODO_URL":"replaceme", -"SLACK_URL":"replaeceme", -"GALAXY_PLUGIN_VERSION":"0.0.6", -"DATABASE_DOMAIN":"postgres", -"CWL_PARSING_LAMBDA_VERSION":"n/a", -"WDL_PARSING_LAMBDA_VERSION":"n/a", -"NEXTFLOW_PARSING_LAMBDA_VERSION":"n/a", -"CHECK_URL_LAMBDA_VERSION":"n/a", -"IS_FARGATE_DEPLOY":false, -"DEPLOY_VERSION":"replaceme", -"DOCKSTORE_DBPASSWORD":"replaceme", -"ELASTICSEARCH_DOMAIN":"replaceme", -"ELASTICSEARCH_PROTOCOL":"http", -"ELASTICSEARCH_PORT":"80", -"ELASTICSEARCH_USER":"replaceme", -"ELASTICSEARCH_PASSWORD":"replaceme", -"ELASTICSEARCH_MAX_CONCURRENT_SESSIONS":"123", -"FEATURED_NEWS_URL":"replaceme", -"POSTGRES_DBPASSWORD":"replaceme" } From 8ab74dc1e66035a9b226d1af70b3cc3961f9af43 Mon Sep 17 00:00:00 2001 From: Kathy Tran Date: Tue, 28 Jun 2022 11:20:26 -0400 Subject: [PATCH 26/47] Fix commas --- dockstore_launcher_config/compose.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dockstore_launcher_config/compose.config b/dockstore_launcher_config/compose.config index 7874b6c..0b6b018 100644 --- a/dockstore_launcher_config/compose.config +++ b/dockstore_launcher_config/compose.config @@ -46,7 +46,7 @@ "ORCID_CLIENT_SECRET":"replaceme", "ORCID_SCOPE":"replaceme", "ORCID_URL":"replaceme", -"POSTGRES_DBPASSWORD":"replaceme" +"POSTGRES_DBPASSWORD":"replaceme", "PRODUCTION":false, "PUBLIC_LAUNCHER_IP_ADDRESS":"", "QUAY_CLIENT_ID":"foobared", @@ -60,5 +60,5 @@ "WDL_PARSING_LAMBDA_VERSION":"n/a", "ZENODO_CLIENT_ID":"replaceme", "ZENODO_CLIENT_SECRET":"replaceme", -"ZENODO_URL":"replaceme", +"ZENODO_URL":"replaceme" } From 20dc5a118fb7e1eab378ad81cc1b896d65cac2a8 Mon Sep 17 00:00:00 2001 From: Kathy Tran Date: Wed, 20 Jul 2022 10:16:04 -0400 Subject: [PATCH 27/47] Invalidate cache every 10 seconds --- templates/web.yml.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/web.yml.template b/templates/web.yml.template index 30f411e..557c62a 100644 --- a/templates/web.yml.template +++ b/templates/web.yml.template @@ -63,7 +63,7 @@ externalConfig: scheme: http{{#HTTPS}}s{{/HTTPS}} port: -authenticationCachePolicy: maximumSize=10000, expireAfterAccess=10m +authenticationCachePolicy: maximumSize=10000, expireAfterAccess=10s httpClient: timeout: 5500ms From 46fef8e031498fac335034c94b4813ed766d7b02 Mon Sep 17 00:00:00 2001 From: Charles Overbeck Date: Tue, 26 Jul 2022 09:24:03 -0700 Subject: [PATCH 28/47] PR template (#231) Inspired by: https://ucsc-cgl.atlassian.net/browse/SEAB-4603 --- .github/PULL_REQUEST_TEMPLATE.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 .github/PULL_REQUEST_TEMPLATE.md diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..2450d87 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,9 @@ +**Description** +A description of the PR, should include a decent explanation as to why this change was needed and a decent explanation as to what this change does + +**Issue** +A link to a github issue or SEAB- ticket (using that as a prefix) + +Please make sure that you've checked the following before submitting your pull request. Thanks! + +- [ ] Ensure that the PR targets the correct branch. Check the milestone or fix version of the ticket. From 1a4ea0a9b7f35e4a3483c72a2e7a1daedacda66d Mon Sep 17 00:00:00 2001 From: Denis Yuen Date: Thu, 15 Sep 2022 10:31:24 -0400 Subject: [PATCH 29/47] Update PULL_REQUEST_TEMPLATE.md (#232) --- .github/PULL_REQUEST_TEMPLATE.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 2450d87..5dd1fe5 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,6 +1,11 @@ **Description** A description of the PR, should include a decent explanation as to why this change was needed and a decent explanation as to what this change does + +**Review Instructions** +Describe if this ticket needs review and if so, how one may go about it in qa and/or staging environments. +For example, a ticket based on Security Hub, Snyk, or Dependabot may not need review since those services will generate new warnings if the issue has not been resolved properly. On the other hand, an infrastructure ticket that results in visible changes to the end-user will definitely require review. Many tickets will likely be between these two extremes, so some judgement may be required. + **Issue** A link to a github issue or SEAB- ticket (using that as a prefix) From aa3c833e67f4b3533056400b66f86166716679f1 Mon Sep 17 00:00:00 2001 From: David Steinberg Date: Mon, 26 Sep 2022 14:49:38 -0700 Subject: [PATCH 30/47] Start cron with the container --- templates/init_webservice.sh.template | 2 ++ 1 file changed, 2 insertions(+) diff --git a/templates/init_webservice.sh.template b/templates/init_webservice.sh.template index 6ea6adc..1f117c8 100755 --- a/templates/init_webservice.sh.template +++ b/templates/init_webservice.sh.template @@ -2,6 +2,8 @@ cd "$(dirname "$0")" +service cron start + java -XX:MaxRAMPercentage=50.0 -XX:+ExitOnOutOfMemoryError -jar /home/dockstore-webservice-*.jar server web.yml | tee --append /dockstore_logs/webservice.out From 39ce831929e7776b399ea17ccc096f6e69d0c6da Mon Sep 17 00:00:00 2001 From: Denis Yuen Date: Thu, 27 Oct 2022 16:42:20 -0400 Subject: [PATCH 31/47] fix up bad merge --- install_bootstrap | 1 - scripts/webservice-image-digest.py | 1 - 2 files changed, 2 deletions(-) diff --git a/install_bootstrap b/install_bootstrap index 8f11df5..f103bc0 100755 --- a/install_bootstrap +++ b/install_bootstrap @@ -73,7 +73,6 @@ fi template -if [ $IS_FARGATE_DEPLOY == "false" ]; then if [ $IS_FARGATE_DEPLOY == "false" ]; then # We need to set the environment variable for the image digest source .env diff --git a/scripts/webservice-image-digest.py b/scripts/webservice-image-digest.py index e7c6e20..1d7dbe6 100755 --- a/scripts/webservice-image-digest.py +++ b/scripts/webservice-image-digest.py @@ -59,7 +59,6 @@ def get_digest_from_s3(directory): # downloads the image-digest.txt from a directory in S3 base_url = "https://gui.dockstore.org" response = requests.get("{}/{}/image-digest.txt".format(base_url, directory)) - response = requests.get(digest_url) if (response.status_code != 200): print("Expected a file at {}".format("{}/{}/image-digest.txt".format(base_url, directory))) print("The image-digest.txt was not found in S3, did the build succeed?") From ff22ced9fb10c58e53f019fee1a977c7b98dfd41 Mon Sep 17 00:00:00 2001 From: Kathy Tran Date: Fri, 4 Nov 2022 11:28:44 -0400 Subject: [PATCH 32/47] Remove ec2 Dockstore stuff --- docker-compose.yml | 76 ---------------------- dockstore_launcher_config/compose.config | 1 - install_bootstrap | 36 +++------- templates/Dockerfile_ui2.template | 17 ----- templates/default.nginx_http.conf.template | 10 --- templates/env.template | 5 -- 6 files changed, 10 insertions(+), 135 deletions(-) delete mode 100755 docker-compose.yml delete mode 100644 templates/Dockerfile_ui2.template delete mode 100644 templates/env.template diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100755 index 2a18ae8..0000000 --- a/docker-compose.yml +++ /dev/null @@ -1,76 +0,0 @@ -version: '2' - -services: - - migration: - image: quay.io/dockstore/dockstore-webservice@${DOCKSTORE_IMAGE_DIGEST} - volumes: - - ./config/web.yml:/home/web.yml - - ./config/init_migration.sh:/home/init_migration.sh - command: ["bash", "/home/init_migration.sh"] - logging: - driver: "awslogs" - options: - awslogs-group: ${LOG_GROUP_NAME} - awslogs-stream: "migration" - - webservice: - image: quay.io/dockstore/dockstore-webservice@${DOCKSTORE_IMAGE_DIGEST} - restart: always - depends_on: - - migration - volumes: - - ./config/web.yml:/home/web.yml - - ./config/init_webservice.sh:/home/init_webservice.sh - - ${GITHUB_APP_PRIVATE_KEY_FILE}:/dockstore/github-key/dockstore-github-private-key.pem - command: ["bash", "/home/init_webservice.sh"] - ports: - - "8081:8081" - logging: - driver: "awslogs" - options: - awslogs-group: ${LOG_GROUP_NAME} - awslogs-stream: "webservice" - nginx_dockstore: - build: - context: . - dockerfile: config/Dockerfile_ui2 - restart: always - depends_on: - - webservice - ports: - - "80:4200" - volumes: - - ./config/default.nginx_http.conf:/etc/nginx/conf.d/default.conf:ro - - ./config/default.nginx_http.shared.conf:/etc/nginx/conf.d/default.nginx_http.shared.conf:ro - - ./config/default.nginx_http.security.conf:/etc/nginx/conf.d/default.nginx_http.security.conf:ro - logging: - driver: "awslogs" - options: - awslogs-group: ${LOG_GROUP_NAME} - awslogs-stream: "nginx_dockstore" - -# re-enable if we go down the route of these metrics going into ELK -# metricbeat: -# image: docker.elastic.co/beats/metricbeat:6.4.0 -# user: root -# restart: always -# depends_on: -# - webservice -# volumes: -# - /proc:/hostfs/proc:ro -# - /sys/fs/cgroup:/hostfs/sys/fs/cgroup:ro -# - /:/hostfs:ro -# - ./config/metricbeat.yml:/usr/share/metricbeat/metricbeat.yml -# command: -# - "--strict.perms=false" -# - "-system.hostfs=/hostfs" -# logging: -# driver: "awslogs" -# options: -# awslogs-group: ${LOG_GROUP_NAME} -# awslogs-stream: "metricbeat" - -volumes: - esdata1: - driver: local diff --git a/dockstore_launcher_config/compose.config b/dockstore_launcher_config/compose.config index b06a079..2c46786 100644 --- a/dockstore_launcher_config/compose.config +++ b/dockstore_launcher_config/compose.config @@ -39,7 +39,6 @@ "GOOGLE_CLIENT_ID":"potato", "GOOGLE_CLIENT_SECRET":"potato", "HTTPS":false, -"IS_FARGATE_DEPLOY":false, "LOGSTASH":false, "LOGSTASH_HOST":"replaceme", "NEXTFLOW_PARSING_LAMBDA_VERSION":"n/a", diff --git a/install_bootstrap b/install_bootstrap index f103bc0..83fb28a 100755 --- a/install_bootstrap +++ b/install_bootstrap @@ -19,26 +19,19 @@ MSG function template() { - mkdir -p config + # Place the config files into specific directories so they can be mounted to container paths as bind mounts + NGINX_CONF_DIRECTORY="config/nginx-conf" + NGINX_HTML2_DIRECTORY="config/nginx-html2" + WEBSERVICE_DIRECTORY="config/webservice" - NGINX_CONF_DIRECTORY="config" - NGINX_HTML2_DIRECTORY="config" - WEBSERVICE_DIRECTORY="config" + mkdir -p ${NGINX_CONF_DIRECTORY} + mkdir -p ${NGINX_CONF_DIRECTORY} + mkdir -p ${WEBSERVICE_DIRECTORY} - if [ $IS_FARGATE_DEPLOY == "true" ]; then - mkdir -p config/webservice - mkdir -p config/nginx-conf - mkdir -p config/nginx-html2 - # Place the config files into specific directories so they can be mounted to container paths as bind mounts - NGINX_CONF_DIRECTORY="${NGINX_CONF_DIRECTORY}/nginx-conf" - NGINX_HTML2_DIRECTORY="${NGINX_HTML2_DIRECTORY}/nginx-html2" - WEBSERVICE_DIRECTORY="${WEBSERVICE_DIRECTORY}/webservice" + # Download index.html for UI we will be serving + wget -qO ${NGINX_HTML2_DIRECTORY}/index.html https://gui.dockstore.org/${UI2_HASH}/index.html + wget -qO ${NGINX_HTML2_DIRECTORY}/manifest.json https://gui.dockstore.org/${UI2_HASH}/manifest.json - wget -qO ${NGINX_HTML2_DIRECTORY}/index.html https://gui.dockstore.org/${UI2_HASH}/index.html - wget -qO ${NGINX_HTML2_DIRECTORY}/manifest.json https://gui.dockstore.org/${UI2_HASH}/manifest.json - fi - - mustache dockstore_launcher_config/compose.config templates/Dockerfile_ui2.template > config/Dockerfile_ui2 mustache dockstore_launcher_config/compose.config templates/robots.txt.template > ${NGINX_HTML2_DIRECTORY}/robots.txt mustache dockstore_launcher_config/compose.config templates/default.nginx_http.conf.template > ${NGINX_CONF_DIRECTORY}/default.nginx_http.conf mustache dockstore_launcher_config/compose.config templates/default.nginx_http.shared.conf.template > ${NGINX_CONF_DIRECTORY}/default.nginx_http.shared.conf @@ -73,13 +66,4 @@ fi template -if [ $IS_FARGATE_DEPLOY == "false" ]; then - # We need to set the environment variable for the image digest - source .env - DOCKSTORE_IMAGE_DIGEST=$(scripts/webservice-image-digest.py $DOCKSTORE_VERSION) - echo "DOCKSTORE_IMAGE_DIGEST=$DOCKSTORE_IMAGE_DIGEST" >> .env - - docker-compose build -fi - echo "Exiting now." diff --git a/templates/Dockerfile_ui2.template b/templates/Dockerfile_ui2.template deleted file mode 100644 index 518b20f..0000000 --- a/templates/Dockerfile_ui2.template +++ /dev/null @@ -1,17 +0,0 @@ -FROM ubuntu:18.04 as builder - -RUN apt update \ - && apt install -yq \ - wget \ - && apt clean - -# Download index.html for UI we will be serving -RUN wget -qO index.html https://gui.dockstore.org/{{ UI2_HASH }}/index.html -RUN wget -qO manifest.json https://gui.dockstore.org/{{ UI2_HASH }}/manifest.json - -FROM nginx:1.13.1 - -RUN mkdir /usr/share/nginx/html2 -COPY --from=builder /index.html /usr/share/nginx/html2 -COPY --from=builder /manifest.json /usr/share/nginx/html2 -COPY config/robots.txt /usr/share/nginx/html2 diff --git a/templates/default.nginx_http.conf.template b/templates/default.nginx_http.conf.template index 8b3c2ae..18fe5b0 100644 --- a/templates/default.nginx_http.conf.template +++ b/templates/default.nginx_http.conf.template @@ -26,12 +26,7 @@ access_log off; resolver 127.0.0.11 valid=10s; server { - {{#IS_FARGATE_DEPLOY}} set $webservice "127.0.0.1"; - {{/IS_FARGATE_DEPLOY}} - {{^IS_FARGATE_DEPLOY}} - set $webservice "webservice"; - {{/IS_FARGATE_DEPLOY}} server_name .{{ DOMAIN_NAME }}; include /etc/nginx/conf.d/default.nginx_http.shared.conf; include /etc/nginx/conf.d/default.nginx_http.security.conf; @@ -107,12 +102,7 @@ server { # Server block for access via IP instead of domain server { - {{#IS_FARGATE_DEPLOY}} set $webservice "127.0.0.1"; - {{/IS_FARGATE_DEPLOY}} - {{^IS_FARGATE_DEPLOY}} - set $webservice "webservice"; - {{/IS_FARGATE_DEPLOY}} access_log /var/log/nginx/access.log custom; listen 4200 default_server; diff --git a/templates/env.template b/templates/env.template deleted file mode 100644 index 2f3541a..0000000 --- a/templates/env.template +++ /dev/null @@ -1,5 +0,0 @@ -LOG_GROUP_NAME={{ DOMAIN_NAME }} -DOCKSTORE_VERSION={{ DOCKSTORE_VERSION }} -GITHUB_APP_PRIVATE_KEY_FILE={{ GITHUB_APP_PRIVATE_KEY_FILE }} -GALAXY_PLUGIN_VERSION={{ GALAXY_PLUGIN_VERSION }} -DATABASE_DOMAIN={{ DATABASE_DOMAIN }} From d8ebfbddb3262d6912d7b0dd8ba3ca93822e2ca8 Mon Sep 17 00:00:00 2001 From: Kathy Tran Date: Fri, 4 Nov 2022 12:48:50 -0400 Subject: [PATCH 33/47] Remove .env file from install_bootstrap --- install_bootstrap | 2 -- 1 file changed, 2 deletions(-) diff --git a/install_bootstrap b/install_bootstrap index 83fb28a..9b8b2ff 100755 --- a/install_bootstrap +++ b/install_bootstrap @@ -52,8 +52,6 @@ function template() mustache dockstore_launcher_config/compose.config templates/jvm.options.logstash.template > config/jvm.options.logstash mustache dockstore_launcher_config/compose.config templates/postgres_backup.sh.template > scripts/postgres_backup.sh chmod a+rx scripts/postgres_backup.sh - # .env file - mustache dockstore_launcher_config/compose.config templates/env.template > .env mkdir -p config/rules for f in $(ls templates/rules/); do mustache dockstore_launcher_config/compose.config templates/rules/$f > config/rules/$f; done From f87027113dbbb815235845fd00f50c1650af2f78 Mon Sep 17 00:00:00 2001 From: Kathy Tran Date: Fri, 4 Nov 2022 12:53:43 -0400 Subject: [PATCH 34/47] Update nginx config directories in GH Action --- .github/workflows/docker-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index a599c74..cd6d430 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -38,7 +38,7 @@ jobs: - name: Run script to generate templates run: bash install_bootstrap --script - name: Run Docker image to validate nginx config - run: docker run -v $PWD/config/default.nginx_http.conf:/etc/nginx/conf.d/default.conf:ro -v $PWD/config/default.nginx_http.shared.conf:/etc/nginx/conf.d/default.nginx_http.shared.conf:ro -v $PWD/config/default.nginx_http.security.conf:/etc/nginx/conf.d/default.nginx_http.security.conf:ro nginx:1.13.1 nginx -t -c /etc/nginx/nginx.conf + run: docker run -v $PWD/config/nginx-conf/default.nginx_http.conf:/etc/nginx/conf.d/default.conf:ro -v $PWD/config/nginx-conf/default.nginx_http.shared.conf:/etc/nginx/conf.d/default.nginx_http.shared.conf:ro -v $PWD/config/nginx-conf/default.nginx_http.security.conf:/etc/nginx/conf.d/default.nginx_http.security.conf:ro nginx:1.13.1 nginx -t -c /etc/nginx/nginx.conf - name: Test image digest script run: | cd scripts/tests From 77cddfccf99721b1d16c9b66c222db58080d993c Mon Sep 17 00:00:00 2001 From: Kathy Tran Date: Fri, 4 Nov 2022 14:10:54 -0400 Subject: [PATCH 35/47] Fix typo --- install_bootstrap | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install_bootstrap b/install_bootstrap index 9b8b2ff..f361fe0 100755 --- a/install_bootstrap +++ b/install_bootstrap @@ -25,7 +25,7 @@ function template() WEBSERVICE_DIRECTORY="config/webservice" mkdir -p ${NGINX_CONF_DIRECTORY} - mkdir -p ${NGINX_CONF_DIRECTORY} + mkdir -p ${NGINX_HTML2_DIRECTORY} mkdir -p ${WEBSERVICE_DIRECTORY} # Download index.html for UI we will be serving From ef9191ed61cc89cddb672d43977e89a5c2059aed Mon Sep 17 00:00:00 2001 From: Kathy Tran Date: Fri, 4 Nov 2022 14:19:08 -0400 Subject: [PATCH 36/47] Remove duplicate keys from compose.config --- dockstore_launcher_config/compose.config | 6 ------ 1 file changed, 6 deletions(-) diff --git a/dockstore_launcher_config/compose.config b/dockstore_launcher_config/compose.config index 2c46786..12704cc 100644 --- a/dockstore_launcher_config/compose.config +++ b/dockstore_launcher_config/compose.config @@ -42,12 +42,6 @@ "LOGSTASH":false, "LOGSTASH_HOST":"replaceme", "NEXTFLOW_PARSING_LAMBDA_VERSION":"n/a", -"BD_CATALYST_SEVEN_BRIDGES_IMPORT_URL":"replaceme", -"BD_CATALYST_TERRA_IMPORT_URL":"replaceme", -"AUTHORIZER_TYPE":"replaceme", -"EXTERNAL_GOOGLE_CLIENT_PREFIX1":"replaceme", -"SAM_PATH":"replaceme", -"DATABASE_GENERATED":false, "ORCID_CLIENT_ID":"replaceme", "ORCID_CLIENT_SECRET":"replaceme", "ORCID_SCOPE":"replaceme", From 06e8ac2f0918bff0160ab353dc7c1b25d0436717 Mon Sep 17 00:00:00 2001 From: Kathy Tran Date: Mon, 7 Nov 2022 11:35:59 -0500 Subject: [PATCH 37/47] Remove unused webservice-digest script --- scripts/tests/test-webservice-image-digest.py | 51 ------------ scripts/webservice-image-digest.py | 80 ------------------- 2 files changed, 131 deletions(-) delete mode 100644 scripts/tests/test-webservice-image-digest.py delete mode 100755 scripts/webservice-image-digest.py diff --git a/scripts/tests/test-webservice-image-digest.py b/scripts/tests/test-webservice-image-digest.py deleted file mode 100644 index 091aaf3..0000000 --- a/scripts/tests/test-webservice-image-digest.py +++ /dev/null @@ -1,51 +0,0 @@ -""" - -Test the script webservice-image-digest by calling this from -the tests directory - -cd compose_setup/scripts/tests -python test-webservice-image-digest.py - -""" - -import unittest - -import subprocess - -script_location = "../webservice-image-digest.py" - -base_command = "python {}".format(script_location) -branch = "develop" -full_directory = "develop-b667562" -simple_tag = "digest_test" -annotated_tag = "1.12.0-beta.1" - -class TestDigest(unittest.TestCase): - -# This test is turned off because there is no stable branch to test against -# def test_branch(self): -# cmd = "{} {}".format(base_command, branch) -# ret = subprocess.check_output(cmd, shell=True, universal_newlines=True).rstrip() -# self.assertEqual(ret, "sha256:52cf6b09e89a238bfd1d98dd01139442d67fcaaa377c179f315dd06555f7bcae") -# pass - - def test_full_directory(self): - cmd = "{} {}".format(base_command, full_directory) - ret = subprocess.check_output(cmd, shell=True, universal_newlines=True).rstrip() - self.assertEqual(ret, "sha256:08c67131daf6109fadb19d994d753ede7ae28e41c675322e2980327597bcb665") - pass - - def test_simple_tag(self): - cmd = "{} {}".format(base_command, simple_tag) - ret = subprocess.check_output(cmd, shell=True, universal_newlines=True).rstrip() - self.assertEqual(ret, "sha256:f21d00e9f01d54eb891c128fb88b76554cb0b47c775929dc05e39a03954e7b0b") - pass - - def test_annotated_tag(self): - cmd = "{} {}".format(base_command, annotated_tag) - ret = subprocess.check_output(cmd, shell=True, universal_newlines=True).rstrip() - self.assertEqual(ret, "sha256:e6dcfdc9ea351b57cde556ff3c68f96b838e8e30cdb4ee693a29b6ef16f3a4be") - pass - -if __name__ == '__main__': - unittest.main() diff --git a/scripts/webservice-image-digest.py b/scripts/webservice-image-digest.py deleted file mode 100755 index 1d7dbe6..0000000 --- a/scripts/webservice-image-digest.py +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env python3 -""" - -This script will gather a sha256 digest created by CircleCI and uploaded to S3 -then print it to the terminal. This is meant to simplify the process of using -images by digest as opposed to by tag and will also guarantee the image pulled -from Quay has not been changed since being built. - -Assets in S3 follow the directory structure: `./branch-shortcommit/image-digest.txt` - -Provide a git tag, branch, or branch-shorthash - -The output is formatted to be easily used to select a specific image digest via docker - -sha256:08c67131daf6109fadb19d994d753ede7ae28e41c675322e2980327597bcb665 - -""" - -import argparse -import requests -import string - -parser = argparse.ArgumentParser( - description='Gather an image digest for the Dockstore Webservice from S3 as created by CircleCI') -parser.add_argument('tag', type=str, - help='The git tag, branch, or branch-hash of a Webservice commit') - -args = parser.parse_args() - -DOCKER_TAG_BASE = "quay.io/dockstore/dockstore-webservice" - -def get_commit_from_tag_url(tag_url): - # takes a tag url and gets the commit hash it is pointed at - response = requests.get(tag_url) - return response.json()['object']['sha'] - -def get_commit_from_github(tag_or_branch): - # takes a tag or branch and returns the latest commit for a branch or commit for a tag - # try tag - base_url = "https://api.github.com/repos/dockstore/dockstore" - tag_url = "{}/{}/{}".format(base_url, "git/ref/tags", tag_or_branch) - response = requests.get(tag_url) - if (response.status_code == 200): - # simple tag - if (response.json()['object']['type'] == "commit"): - return response.json()['object']['sha'] - else: - # annotated tag - return get_commit_from_tag_url(response.json()['object']['url']) - # try branch - branch_url = "{}/{}={}".format(base_url, "commits?sha", tag_or_branch) - response = requests.get(branch_url) - if (response.status_code == 200): - return response.json()[0]['sha'] - print("No commit for that tag or branch found!") - exit(1) - -def get_digest_from_s3(directory): - # downloads the image-digest.txt from a directory in S3 - base_url = "https://gui.dockstore.org" - response = requests.get("{}/{}/image-digest.txt".format(base_url, directory)) - if (response.status_code != 200): - print("Expected a file at {}".format("{}/{}/image-digest.txt".format(base_url, directory))) - print("The image-digest.txt was not found in S3, did the build succeed?") - exit(1) - # There is a newline at the end of the file we rstrip - return response.text.rstrip() - -if __name__ == "__main__": - # slashes are replaced with _ in docker image tags - # check to see if input includes a dash followed by 7 chars - parsed = args.tag.split('-') - if len(parsed) == 2 and len(parsed[1]) == 7 and all(c in string.hexdigits for c in parsed[1]): - directory = args.tag - else: - commit = get_commit_from_github(args.tag) - directory = "{}-{}".format(args.tag, commit[0:7]) - circle_digest = get_digest_from_s3(directory) - print("sha256:{}".format(circle_digest)) - exit(0) From fdd2961678cdd8643e8d2a97b1a495cdc189364b Mon Sep 17 00:00:00 2001 From: Kathy Tran Date: Mon, 7 Nov 2022 11:37:56 -0500 Subject: [PATCH 38/47] Remove image digest test in GH action --- .github/workflows/docker-image.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index cd6d430..0b36fdd 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -39,7 +39,3 @@ jobs: run: bash install_bootstrap --script - name: Run Docker image to validate nginx config run: docker run -v $PWD/config/nginx-conf/default.nginx_http.conf:/etc/nginx/conf.d/default.conf:ro -v $PWD/config/nginx-conf/default.nginx_http.shared.conf:/etc/nginx/conf.d/default.nginx_http.shared.conf:ro -v $PWD/config/nginx-conf/default.nginx_http.security.conf:/etc/nginx/conf.d/default.nginx_http.security.conf:ro nginx:1.13.1 nginx -t -c /etc/nginx/nginx.conf - - name: Test image digest script - run: | - cd scripts/tests - python test-webservice-image-digest.py From 4a07cc23471d075bdd2d94df8670896ef926c182 Mon Sep 17 00:00:00 2001 From: Kathy Tran Date: Wed, 9 Nov 2022 14:52:04 -0500 Subject: [PATCH 39/47] Remove galaxy plugin version because it's not used --- dockstore_launcher_config/compose.config | 1 - 1 file changed, 1 deletion(-) diff --git a/dockstore_launcher_config/compose.config b/dockstore_launcher_config/compose.config index 12704cc..365a3a6 100644 --- a/dockstore_launcher_config/compose.config +++ b/dockstore_launcher_config/compose.config @@ -28,7 +28,6 @@ "EXTERNAL_GOOGLE_CLIENT_PREFIX1":"replaceme", "FEATURED_CONTENT_URL":"foobar", "FEATURED_NEWS_URL":"replaceme", -"GALAXY_PLUGIN_VERSION":"0.0.6", "GITHUB_APP_ID": "1234", "GITHUB_APP_NAME": "foobar", "GITHUB_APP_PRIVATE_KEY_FILE": "/replaceme", From bf958b9ab0a9dc3fa68d3ff4985ca7e291ac91f7 Mon Sep 17 00:00:00 2001 From: David Steinberg Date: Thu, 10 Nov 2022 09:58:09 -0800 Subject: [PATCH 40/47] Add zenodo sandbox to CSP --- templates/default.nginx_http.security.conf.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/default.nginx_http.security.conf.template b/templates/default.nginx_http.security.conf.template index c740558..5639985 100644 --- a/templates/default.nginx_http.security.conf.template +++ b/templates/default.nginx_http.security.conf.template @@ -18,7 +18,7 @@ add_header X-XSS-Protection "1; mode=block" always; add_header Referrer-Policy "strict-origin-when-cross-origin" always; # Explicitly list domains allowed to serve content for this site -add_header Content-Security-Policy-Report-Only "report-uri https://api.dockstore-security.org/csp-report; default-src 'self'; object-src 'none'; base-uri 'self'; manifest-src 'self' dockstore.org; media-src 'self'; worker-src 'none'; script-src 'report-sample' 'self' 'unsafe-hashes' 'unsafe-inline' 'unsafe-eval' discuss.dockstore.org gui.dockstore.org *.twitter.com *.twimg.com www.google-analytics.com www.googletagmanager.com; style-src 'report-sample' 'self' 'unsafe-inline' cdnjs.cloudflare.com fonts.googleapis.com *.twitter.com *.twimg.com gui.dockstore.org; connect-src 'self' s3.amazonaws.com api.github.com view.commonwl.org www.google-analytics.com content.dockstore.org gui.dockstore.org; font-src 'self' fonts.gstatic.com gui.dockstore.org; frame-src 'self' discuss.dockstore.org platform.twitter.com youtube.com; img-src data: 'self' user-images.githubusercontent.com avatars.githubusercontent.com avatars0.githubusercontent.com avatars1.githubusercontent.com avatars2.githubusercontent.com avatars3.githubusercontent.com camo.githubusercontent.com circleci.com gui.dockstore.org gstatic.com i.imgur.com api.travis-ci.com travis-ci.com img.shields.io quay.io via.placeholder.com *.wp.com *.googleusercontent.com www.googletagmanager.com www.google-analytics.com www.gravatar.com *.twitter.com i.ytimg.com *.twimg.com zenodo.org;" always; +add_header Content-Security-Policy-Report-Only "report-uri https://api.dockstore-security.org/csp-report; default-src 'self'; object-src 'none'; base-uri 'self'; manifest-src 'self' dockstore.org; media-src 'self'; worker-src 'none'; script-src 'report-sample' 'self' 'unsafe-hashes' 'unsafe-inline' 'unsafe-eval' discuss.dockstore.org gui.dockstore.org *.twitter.com *.twimg.com www.google-analytics.com www.googletagmanager.com; style-src 'report-sample' 'self' 'unsafe-inline' cdnjs.cloudflare.com fonts.googleapis.com *.twitter.com *.twimg.com gui.dockstore.org; connect-src 'self' s3.amazonaws.com api.github.com view.commonwl.org www.google-analytics.com content.dockstore.org gui.dockstore.org; font-src 'self' fonts.gstatic.com gui.dockstore.org; frame-src 'self' discuss.dockstore.org platform.twitter.com youtube.com; img-src data: 'self' user-images.githubusercontent.com avatars.githubusercontent.com avatars0.githubusercontent.com avatars1.githubusercontent.com avatars2.githubusercontent.com avatars3.githubusercontent.com camo.githubusercontent.com circleci.com gui.dockstore.org gstatic.com i.imgur.com api.travis-ci.com travis-ci.com img.shields.io quay.io via.placeholder.com *.wp.com *.googleusercontent.com www.googletagmanager.com www.google-analytics.com www.gravatar.com *.twitter.com i.ytimg.com *.twimg.com sandbox.zenodo.org zenodo.org;" always; # Hide server header proxy_hide_header Server; From 1c6f7ed6a32a7f6acf2ae07c10efbcdeb137ed6a Mon Sep 17 00:00:00 2001 From: Kathy Tran Date: Mon, 14 Nov 2022 08:53:34 -0500 Subject: [PATCH 41/47] Remove PUBLIC_IP_LAUNCHER_ADDRESS and HTTPS --- dockstore_launcher_config/compose.config | 2 -- templates/default.nginx_http.conf.template | 4 ++-- templates/robots.txt.template | 2 +- templates/web.yml.template | 8 ++++---- 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/dockstore_launcher_config/compose.config b/dockstore_launcher_config/compose.config index 365a3a6..782b39f 100644 --- a/dockstore_launcher_config/compose.config +++ b/dockstore_launcher_config/compose.config @@ -37,7 +37,6 @@ "GITLAB_CLIENT_SECRET":"foobar", "GOOGLE_CLIENT_ID":"potato", "GOOGLE_CLIENT_SECRET":"potato", -"HTTPS":false, "LOGSTASH":false, "LOGSTASH_HOST":"replaceme", "NEXTFLOW_PARSING_LAMBDA_VERSION":"n/a", @@ -47,7 +46,6 @@ "ORCID_URL":"replaceme", "POSTGRES_DBPASSWORD":"replaceme", "PRODUCTION":false, -"PUBLIC_LAUNCHER_IP_ADDRESS":"", "QUAY_CLIENT_ID":"foobared", "QUAY_CLIENT_SECRET":"foobared", "SAM_PATH":"replaceme", diff --git a/templates/default.nginx_http.conf.template b/templates/default.nginx_http.conf.template index 18fe5b0..fa952eb 100644 --- a/templates/default.nginx_http.conf.template +++ b/templates/default.nginx_http.conf.template @@ -54,13 +54,13 @@ server { # used to redirect swagger.json as retrieved by swagger ui without changes location = /swagger.json { - rewrite ^/(.*)$ http{{#HTTPS}}s{{/HTTPS}}://{{ DOMAIN_NAME}}/api/swagger.json permanent; + rewrite ^/(.*)$ https://{{ DOMAIN_NAME}}/api/swagger.json permanent; } # used to redirect swagger.json as retrieved by swagger ui without changes location = /openapi.yaml { #proxy_pass http://webservice:8080/openapi.yaml; - rewrite ^/(.*)$ http{{#HTTPS}}s{{/HTTPS}}://{{ DOMAIN_NAME}}/api/openapi.yaml permanent; + rewrite ^/(.*)$ https://{{ DOMAIN_NAME}}/api/openapi.yaml permanent; } # used when port 8443 was unreachable from google diff --git a/templates/robots.txt.template b/templates/robots.txt.template index c3f76cd..152a8a2 100644 --- a/templates/robots.txt.template +++ b/templates/robots.txt.template @@ -1,7 +1,7 @@ User-Agent: * {{#PRODUCTION}} Allow: / -Sitemap: http{{#HTTPS}}s{{/HTTPS}}://{{ DOMAIN_NAME }}/sitemap.txt; +Sitemap: https://{{ DOMAIN_NAME }}/sitemap.txt; {{/PRODUCTION}} {{^PRODUCTION}} Disallow: / diff --git a/templates/web.yml.template b/templates/web.yml.template index 557c62a..53dbdb3 100644 --- a/templates/web.yml.template +++ b/templates/web.yml.template @@ -11,11 +11,11 @@ bitbucketClientSecret: {{ BITBUCKET_CLIENT_SECRET }} gitlabClientID: {{ GITLAB_CLIENT_ID }} gitlabClientSecret: {{ GITLAB_CLIENT_SECRET }} -gitlabRedirectURI: http{{#HTTPS}}s{{/HTTPS}}://{{ DOMAIN_NAME }}:443/auth/gitlab.com +gitlabRedirectURI: https://{{ DOMAIN_NAME }}:443/auth/gitlab.com googleClientID: {{ GOOGLE_CLIENT_ID }} googleClientSecret: {{ GOOGLE_CLIENT_SECRET }} -googleRedirectURI: http{{#HTTPS}}s{{/HTTPS}}://{{ DOMAIN_NAME }}:443 +googleRedirectURI: https://{{ DOMAIN_NAME }}:443 discourseKey: {{ DISCOURSE_KEY }} discourseUrl: {{ DISCOURSE_URL }} @@ -23,7 +23,7 @@ discourseCategoryId: {{ DISCOURSE_CATEGORY_ID }} zenodoClientID: {{ ZENODO_CLIENT_ID }} zenodoClientSecret: {{ ZENODO_CLIENT_SECRET }} -zenodoRedirectURI: http{{#HTTPS}}s{{/HTTPS}}://{{ DOMAIN_NAME }}/auth/zenodo.org +zenodoRedirectURI: https://{{ DOMAIN_NAME }}/auth/zenodo.org zenodoUrl: {{ ZENODO_URL }} orcidClientID: {{ ORCID_CLIENT_ID }} @@ -60,7 +60,7 @@ samconfiguration: externalConfig: basePath: /api/ hostname: {{ DOMAIN_NAME }} - scheme: http{{#HTTPS}}s{{/HTTPS}} + scheme: https port: authenticationCachePolicy: maximumSize=10000, expireAfterAccess=10s From e28cd1e5f387fa1142e6859dc14a98a01be6621d Mon Sep 17 00:00:00 2001 From: Kathy Tran Date: Mon, 14 Nov 2022 09:31:41 -0500 Subject: [PATCH 42/47] Update README --- README.md | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 9732d24..0e19428 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,8 @@ # compose\_setup -This project documents how to setup Dockstore staging and production using composed Docker containers. +This project contains configuration template files needed to run Dockstore using AWS Fargate. View the [dockstore-deploy repository](https://github.com/dockstore/dockstore-deploy) +for more information about how Dockstore is setup using AWS Fargate. Log issues and see general documentation at [dockstore](https://github.com/ga4gh/dockstore/issues) and [docs.dockstore.org](https://docs.dockstore.org/) respectively -Port 80 is exposed over http. This port should not be exposed to the public. A separately [configured load -balancer](https://github.com/dockstore/dockstore-deploy) is responsible for SSL termination and forwarding traffic to this instance. Previously this repo handled the SSL termination with nginx and LetsEncrypt. - If you are looking for how to run Dockstore locally as a developer, you are probably in the wrong place and should take a look at https://github.com/dockstore/dockstore/blob/develop/docker-compose.yml ## Prerequisities @@ -17,24 +15,16 @@ If you are looking for how to run Dockstore locally as a developer, you are prob ## Usage -1. Call the install\_bootstrap script. This templates the contents of `templates` using mustache to the `config` directory while recording your answers for future use. Note that this will also -rebuild your docker images without affecting existing running containers +1. Call the install\_bootstrap script. This templates the contents of `templates` using mustache to the `config` directory while recording your answers for future use. 2. Some additional information on the answers requested in the script 1. Each integration requires a client id and a secret, it is worth saying that you should not check these in 2. The discourse URL is needed to link Dockstore to a discussion forum 3. the Google verification code and tag manager ID are used if you want to properly track visitors to Dockstore and what pages they browse to -3. The bootstrap script can also rebuild your Docker images. Keep in mind the following handy commands: - 1. `install_bootstrap --script` will template and build everything using your previous answers (useful for quick iteration) - 2. `docker-compose down` will bring all containers down safely - 3. `nohup docker-compose up --force-recreate --remove-orphans >/dev/null 2>&1 &` will re-create all containers known to docker-compose and delete those volumes that no longer are associated with running containers - 4. `docker system prune` for cleaning out old containers and images - 5. To watch the logs `docker-compose logs --follow` while debugging - -4. After following the instructions in the bootstrap script and starting up the site with `docker-compose`, you can browse to the Dockstore site hosted at port 443 by default. `https://` if you specified https or `http://:443` if you did not. +3. After following the instructions in the bootstrap script and starting up the site with AWS Fargate, you can browse to the Dockstore site hosted at port 443 by default using `https://`. -The current setup relies upon an externally hosted database (currently AWS RDS) and externally hosted search (currently AWS Elasticsearch). +The current setup relies upon an externally hosted container orchestration service (current AWS ECS with Fargate), externally hosted database (currently AWS RDS) and externally hosted search (currently AWS Elasticsearch). ### Loading Up a Database ### @@ -72,3 +62,10 @@ npm run install-git secrets This should install git secrets into your local repository and perform a scan. If secrets are found, the run will error and output the potential secret to stdout. If you believe the scan is a false-positive, add the line glob to .gitallowed. + +## Handy docker-compose commands: + 1. `install_bootstrap --script` will template and build everything using your previous answers (useful for quick iteration) + 2. `docker-compose down` will bring all containers down safely + 3. `nohup docker-compose up --force-recreate --remove-orphans >/dev/null 2>&1 &` will re-create all containers known to docker-compose and delete those volumes that no longer are associated with running containers + 4. `docker system prune` for cleaning out old containers and images + 5. To watch the logs `docker-compose logs --follow` while debugging \ No newline at end of file From 759bad0ea67f8df1c6ffedcb885609a300811b54 Mon Sep 17 00:00:00 2001 From: Kathy Tran Date: Tue, 15 Nov 2022 14:28:48 -0500 Subject: [PATCH 43/47] Add 1.14.0 migrations --- templates/init_migration.sh.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/init_migration.sh.template b/templates/init_migration.sh.template index 4c3444f..f671fe6 100755 --- a/templates/init_migration.sh.template +++ b/templates/init_migration.sh.template @@ -12,4 +12,4 @@ java -Ddw.database.user=postgres -Ddw.database.password="{{{ POSTGRES_DBPASSWORD # this particular migration needs to run as postgres because only postgres can surrender ownership java -Ddw.database.user=postgres -Ddw.database.password="{{{ POSTGRES_DBPASSWORD }}}" -jar /home/dockstore-webservice-*.jar db migrate web.yml --include 1.7.0.relinquish # future migrations will start here and should be run as dockstore -java -Ddw.database.user=dockstore -Ddw.database.password="{{{ DOCKSTORE_DBPASSWORD }}}" -jar /home/dockstore-webservice-*.jar db migrate web.yml --include 1.8.0,1.9.0,1.10.0,1.11.0,1.12.0,1.13.0 +java -Ddw.database.user=dockstore -Ddw.database.password="{{{ DOCKSTORE_DBPASSWORD }}}" -jar /home/dockstore-webservice-*.jar db migrate web.yml --include 1.8.0,1.9.0,1.10.0,1.11.0,1.12.0,1.13.0,1.14.0 From 434cb2ed53e287374c23f253b31cbcbb401539ec Mon Sep 17 00:00:00 2001 From: Kathy Tran Date: Wed, 30 Nov 2022 15:24:36 -0500 Subject: [PATCH 44/47] Migrate to ruby/setup-ruby bc actions/setup-ruby is deprecated --- .github/workflows/docker-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 0b36fdd..3e3454b 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -29,7 +29,7 @@ jobs: python -m pip install --upgrade pip pip install requests - name: Set up Ruby 2.6 - uses: actions/setup-ruby@v1 + uses: ruby/setup-ruby@v1 with: ruby-version: 2.6 - name: Build environment From f414126f928f586f40cb1fa9600150b5f75b4d7c Mon Sep 17 00:00:00 2001 From: David Steinberg Date: Thu, 22 Dec 2022 12:51:53 -0800 Subject: [PATCH 45/47] Remove unsafe hashes and eval from script-src --- templates/default.nginx_http.security.conf.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/default.nginx_http.security.conf.template b/templates/default.nginx_http.security.conf.template index 5639985..105b663 100644 --- a/templates/default.nginx_http.security.conf.template +++ b/templates/default.nginx_http.security.conf.template @@ -18,7 +18,7 @@ add_header X-XSS-Protection "1; mode=block" always; add_header Referrer-Policy "strict-origin-when-cross-origin" always; # Explicitly list domains allowed to serve content for this site -add_header Content-Security-Policy-Report-Only "report-uri https://api.dockstore-security.org/csp-report; default-src 'self'; object-src 'none'; base-uri 'self'; manifest-src 'self' dockstore.org; media-src 'self'; worker-src 'none'; script-src 'report-sample' 'self' 'unsafe-hashes' 'unsafe-inline' 'unsafe-eval' discuss.dockstore.org gui.dockstore.org *.twitter.com *.twimg.com www.google-analytics.com www.googletagmanager.com; style-src 'report-sample' 'self' 'unsafe-inline' cdnjs.cloudflare.com fonts.googleapis.com *.twitter.com *.twimg.com gui.dockstore.org; connect-src 'self' s3.amazonaws.com api.github.com view.commonwl.org www.google-analytics.com content.dockstore.org gui.dockstore.org; font-src 'self' fonts.gstatic.com gui.dockstore.org; frame-src 'self' discuss.dockstore.org platform.twitter.com youtube.com; img-src data: 'self' user-images.githubusercontent.com avatars.githubusercontent.com avatars0.githubusercontent.com avatars1.githubusercontent.com avatars2.githubusercontent.com avatars3.githubusercontent.com camo.githubusercontent.com circleci.com gui.dockstore.org gstatic.com i.imgur.com api.travis-ci.com travis-ci.com img.shields.io quay.io via.placeholder.com *.wp.com *.googleusercontent.com www.googletagmanager.com www.google-analytics.com www.gravatar.com *.twitter.com i.ytimg.com *.twimg.com sandbox.zenodo.org zenodo.org;" always; +add_header Content-Security-Policy-Report-Only "report-uri https://api.dockstore-security.org/csp-report; default-src 'self'; object-src 'none'; base-uri 'self'; manifest-src 'self' dockstore.org; media-src 'self'; worker-src 'none'; script-src 'report-sample' discuss.dockstore.org gui.dockstore.org *.twitter.com *.twimg.com www.google-analytics.com www.googletagmanager.com; style-src 'report-sample' 'self' 'unsafe-inline' cdnjs.cloudflare.com fonts.googleapis.com *.twitter.com *.twimg.com gui.dockstore.org; connect-src 'self' s3.amazonaws.com api.github.com view.commonwl.org www.google-analytics.com content.dockstore.org gui.dockstore.org; font-src 'self' fonts.gstatic.com gui.dockstore.org; frame-src 'self' discuss.dockstore.org platform.twitter.com youtube.com; img-src data: 'self' user-images.githubusercontent.com avatars.githubusercontent.com avatars0.githubusercontent.com avatars1.githubusercontent.com avatars2.githubusercontent.com avatars3.githubusercontent.com camo.githubusercontent.com circleci.com gui.dockstore.org gstatic.com i.imgur.com api.travis-ci.com travis-ci.com img.shields.io quay.io via.placeholder.com *.wp.com *.googleusercontent.com www.googletagmanager.com www.google-analytics.com www.gravatar.com *.twitter.com i.ytimg.com *.twimg.com sandbox.zenodo.org zenodo.org;" always; # Hide server header proxy_hide_header Server; From 07e3551a948f9d8e284a00af98a6fad5db2042e9 Mon Sep 17 00:00:00 2001 From: Kathy Tran Date: Wed, 8 Feb 2023 12:07:43 -0500 Subject: [PATCH 46/47] Add metrics bucket name config --- dockstore_launcher_config/compose.config | 1 + templates/web.yml.template | 2 ++ 2 files changed, 3 insertions(+) diff --git a/dockstore_launcher_config/compose.config b/dockstore_launcher_config/compose.config index 782b39f..4ee1382 100644 --- a/dockstore_launcher_config/compose.config +++ b/dockstore_launcher_config/compose.config @@ -39,6 +39,7 @@ "GOOGLE_CLIENT_SECRET":"potato", "LOGSTASH":false, "LOGSTASH_HOST":"replaceme", +"METRICS_BUCKET_NAME":"replaceme", "NEXTFLOW_PARSING_LAMBDA_VERSION":"n/a", "ORCID_CLIENT_ID":"replaceme", "ORCID_CLIENT_SECRET":"replaceme", diff --git a/templates/web.yml.template b/templates/web.yml.template index 53dbdb3..9c41c6e 100644 --- a/templates/web.yml.template +++ b/templates/web.yml.template @@ -55,6 +55,8 @@ externalGoogleClientIdPrefixes: samconfiguration: basepath: {{ SAM_PATH }} +metricsConfig: + s3BucketName: {{ METRICS_BUCKET_NAME }} # the following values describe where the webservice is being run (and on what port and using what scheme) to configure swagger externalConfig: From 6f68328b8099a52bc01352f8785079b41a1272e2 Mon Sep 17 00:00:00 2001 From: Denis Yuen Date: Thu, 16 Mar 2023 09:56:02 -0400 Subject: [PATCH 47/47] updates (#245) --- DEV-README.md | 2 +- README.md | 16 ++++++++-------- scripts/restart.sh | 4 ++-- templates/postgres_backup.sh.template | 2 +- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/DEV-README.md b/DEV-README.md index 43e8b03..6df2551 100644 --- a/DEV-README.md +++ b/DEV-README.md @@ -11,7 +11,7 @@ There are 3 different sets of metric logs being sent to logstash's elasticsearch ## Apache HTTP Logs -Currently, install\_bootstrap and docker-compose handles all necessary configuration +Currently, install\_bootstrap and docker compose handles all necessary configuration