vimc-7176 copy guidance reports into contrib portal

config/basic/montagu.yml

@@ -113,11 +113,12 @@ task_queue:
       password: password
     smtp:
       from: [email protected]
-# If fake_smtp_server config is provided, the task_queue will use this as its smtp server
-# Note this will override other config provided in the task_queue section above
+# Optional. If fake_smtp_server config is provided, the task_queue will use this as its smtp server
+# Note this will override other config provided in the task_queue section above.
 fake_smtp_server:
   repo: reachfive
   name: fake-smtp-server
   tag: latest
 
 orderly_web_api_url: https://localhost/reports/api/v2
+orderly_volume_name: montagu_orderly_volume

config/ci/montagu.yml

@@ -46,13 +46,18 @@ volumes:
 api:
   name: montagu-api
   tag: master
+  # Optional. To send real email notifications, a password and flowurl must be provided here.
+  email:
+    password: "changeme"
+    flow_url: "fakeurl"
   admin:
     name: montagu-cli
     tag: master
 db:
   name: montagu-db
   tag: master
   root_user: vimc
+  # Optional. If root_password is not provided, the tool will randomly generate a secure root password.
   root_password: changeme
   migrate:
     name: montagu-migrate
@@ -70,6 +75,12 @@ db:
     readonly:
       password: "readonlypassword"
       permissions: readonly
+    barman:
+      password: "barmanpassword"
+      option: superuser
+    streaming_barman:
+      password: "streamingpassword"
+      option: replication
   protected_tables:
     - gavi_support_level
     - activity_type
@@ -131,9 +142,12 @@ task_queue:
       password: password
     smtp:
       from: [email protected]
+# Optional. If fake_smtp_server config is provided, the task_queue will use this as its smtp server
+# Note this will override other config provided in the task_queue section above.
 fake_smtp_server:
   repo: reachfive
   name: fake-smtp-server
   tag: latest
 
 orderly_web_api_url: https://localhost/reports/api/v2
+orderly_volume_name: montagu_orderly_volume

config/guidance/montagu.yml

@@ -0,0 +1,138 @@
+## Prefix for container names; we'll use {container_prefix}-(container_name)
+container_prefix: montagu
+
+## Set this flag to true to prevent use of --volumes in the cli to remove
+## volumes on stop
+protect_data: false
+
+## Docker org for images
+repo: vimc
+
+## The name of the docker network that containers will be attached to.
+## If you want to proxy Packit to the host, you will need to
+## arrange a proxy on this network
+network: montagu-network
+
+# Domain where this instance of Montagu will be deployed. E.g. science.montagu.dide.ic.uk
+hostname: localhost
+
+## Names of the docker volumes to use
+volumes:
+  db: db_volume
+  burden_estimates: burden_estimate_files
+  emails: emails
+  templates: template_volume
+  guidance: guidance_volume
+  static: static_volume
+  static_logs: static_logs
+  mq: mq
+
+api:
+  name: montagu-api
+  tag: master
+  email:
+    password: "changeme"
+    flow_url: "fakeurl"
+  admin:
+    name: montagu-cli
+    tag: master
+db:
+  name: montagu-db
+  tag: master
+  root_user: vimc
+  root_password: "changeme"
+  migrate:
+    name: montagu-migrate
+    tag: master
+  users:
+    api:
+      password: "apipassword"
+      permissions: all
+    import:
+      password: "importpassword"
+      permissions: all
+    orderly:
+      password: "orderlypassword"
+      permissions: all
+    readonly:
+      password: "readonlypassword"
+      permissions: readonly
+    barman:
+      password: "barmanpassword"
+      option: superuser
+    streaming_barman:
+      password: "streamingpassword"
+      option: replication
+  protected_tables:
+    - gavi_support_level
+    - activity_type
+    - burden_outcome
+    - gender
+    - responsibility_set_status
+    - impact_outcome
+    - gavi_support_level
+    - support_type
+    - touchstone_status
+    - permission
+    - role
+    - role_permission
+proxy:
+  name: montagu-reverse-proxy
+  tag: vimc-7152
+  port_http: 80
+  port_https: 443
+  metrics:
+    repo: nginx
+    name: nginx-prometheus-exporter
+    tag: 0.4.1
+contrib:
+  name: montagu-contrib-portal
+  tag: master
+  # Optional: name and version of orderly guidance reports to be copied into the contrib
+  # portal container.
+  guidance_reports:
+    - example/20230803-125403-dd77aa93/
+admin:
+  name: montagu-admin-portal
+  tag: master
+static:
+  name: montagu-static
+  tag: master
+mq:
+  repo: docker.io
+  name: redis
+  tag: latest
+  port: 6379
+flower:
+  repo: mher
+  name: flower
+  tag: 0.9.5
+  port: 5555
+task_queue:
+  name: task-queue-worker
+  tag: master
+  youtrack_token: "faketoken"
+  servers:
+    montagu:
+      user: [email protected]
+      password: password
+    orderlyweb:
+      url: http://orderly-web-web:8888
+    youtrack:
+      token: faketoken
+    smtp:
+      host: smtp.cc.ic.ac.uk
+      port: 587
+      user: montagu
+      password: p@ssword
+      from: [email protected]
+  tasks:
+    diagnostic_reports:
+      use_additional_recipients: true
+      poll_seconds: 5
+      reports:
+    archive_folder_contents:
+      min_file_age_seconds: 3600
+
+orderly_web_api_url: https://localhost/reports/api/v2
+orderly_volume_name: montagu_orderly_volume

config/ssl/diagnostic-reports.yml

@@ -0,0 +1,9 @@
+testGroup:
+  testDisease:
+    - report_name: diagnostic
+      assignee: a.hill
+      success_email:
+        recipients:
+          - [email protected]
+          - [email protected]
+        subject: "VIMC diagnostic report: {touchstone} - {group} - {disease}"

config/complete/montagu.yml → config/ssl/montagu.yml

@@ -143,3 +143,4 @@ task_queue:
       min_file_age_seconds: 3600
 
 orderly_web_api_url: https://localhost/reports/api/v2
+orderly_volume_name: montagu_orderly_volume

src/montagu_deploy/config.py

@@ -28,6 +28,7 @@ def __init__(self, path, extra=None, options=None):
         self.repo = config.config_string(dat, ["repo"])
         self.hostname = config.config_string(dat, ["hostname"])
         self.orderly_web_api_url = config.config_string(dat, ["orderly_web_api_url"])
+        self.orderly_volume_name = config.config_string(dat, ["orderly_volume_name"])
 
         # API
         self.api_ref = self.build_ref(dat, "api")
@@ -72,6 +73,7 @@ def __init__(self, path, extra=None, options=None):
         # Portals
         self.admin_ref = self.build_ref(dat, "admin")
         self.contrib_ref = self.build_ref(dat, "contrib")
+        self.guidance_reports = config.config_list(dat, ["contrib", "guidance_reports"], is_optional=True)
 
         self.static_ref = self.build_ref(dat, "static")
 

src/montagu_deploy/montagu_constellation.py

@@ -2,6 +2,7 @@
 
 import constellation
 import docker
+import docker.client
 import yaml
 from constellation import docker_util
 from psycopg2 import connect
@@ -61,7 +62,21 @@ def contrib_container(cfg):
         constellation.ConstellationMount("templates", "/usr/share/nginx/html/templates"),
         constellation.ConstellationMount("guidance", "/usr/share/nginx/html/guidance"),
     ]
-    return constellation.ConstellationContainer(name, cfg.contrib_ref, mounts=mounts)
+    return constellation.ConstellationContainer(name, cfg.contrib_ref, mounts=mounts, configure=contrib_configure)
+
+
+def contrib_configure(_, cfg):
+    if cfg.guidance_reports is not None:
+        print("[contrib] Configuring contrib portal")
+        for p in cfg.guidance_reports:
+            if len(p) > 0:
+                path_to_artefacts = join("archive", p, "*.html")
+                add_reports_to_contrib_portal(cfg, path_to_artefacts)
+
+
+def add_reports_to_contrib_portal(cfg, path_to_reports):
+    print("[contrib] Copying guidance reports from orderly to contrib portal")
+    copy_between_volumes(cfg.orderly_volume_name, cfg.volumes["guidance"], path_to_reports)
 
 
 def static_container(cfg):
@@ -260,6 +275,17 @@ def proxy_configure(container, cfg):
         docker_util.string_into_container(cfg.dhparam, container, join(ssl_path, "dhparam.pem"))
 
 
+def copy_between_volumes(source_volume, destination_volume, path_to_copy, destination_path="."):
+    client = docker.client.from_env()
+    cmd = fr"cd /to ; mkdir -p {destination_path} && find /from/{path_to_copy} -exec cp -a {{}} {destination_path} \;"
+    client.containers.run(
+        "alpine",
+        ["ash", "-c", cmd],
+        remove=True,
+        mounts=[docker.types.Mount("/from", source_volume), docker.types.Mount("/to", destination_volume)],
+    )
+
+
 def start_proxy_metrics(cfg):
     name = "{}-{}".format(cfg.container_prefix, cfg.containers["metrics"])
     proxy_name = cfg.containers["proxy"]

tests/guidance/orderly-web.yml

@@ -0,0 +1,83 @@
+## Prefix for container names; we'll use {container_prefix}-(orderly,web)
+container_prefix: orderly-web
+
+## The name of the docker network that containers will be attached to.
+## If you want to proxy OrderlyWeb to the host, you will need to
+## arrange a proxy on this network, or use dev_mode in the web section
+## below.
+network: montagu-network
+
+## Names of the docker volumes to use:
+##
+## orderly: stores the orderly archive
+## proxy_logs: stores logs from the reverse proxy (only used if proxy is given)
+##
+## (More volumes are anticipated as the tool develops)
+volumes:
+  orderly: montagu_orderly_volume
+  proxy_logs: orderly_web_proxy_logs
+  redis: orderly_web_redis_data
+
+## Redis configuration
+redis:
+  image:
+    name: redis
+    tag: "5.0"
+  volume: orderly_web_redis_data
+
+## Orderly configuration
+orderly:
+  ## Image to use for orderly.  This should build off of
+  ## vimc/orderly.server but can be extended to use whatever packages
+  ## you need.  The components repo, image and tag will be assembled as
+  ## <repo>/<name>:<tag> for the full docker image reference.
+  image:
+    repo: vimc
+    name: orderly.server
+    tag: master
+    worker_name: orderly.server
+  initial:
+    source: clone
+    url: https://github.com/reside-ic/orderly-example
+
+## Api and Website configuration
+web:
+  ## Image to use for the web components.  As for the orderly
+  ## configuration these will be assembled as <repo>/<name>:<tag>.  In
+  ## addition, a second image with the database migration support will
+  ## be used as <repo>/<migrate>:<tag>.  It is not expected (unlike
+  ## the orderly image) that the 'name' and 'migrate' will need to be
+  ## changed often, and it is expected that a web image will go
+  ## together with corresponding migration and admin images.
+  image:
+    repo: vimc
+    name: orderly-web
+    tag: master
+    migrate: orderlyweb-migrate
+    admin: orderly-web-user-cli
+  ## Public-facing url for the whole web service, including protocol
+  ## (ideally https://), hostname and port (if not using standard
+  ## ports).  Here, we're going to use the same as for the proxy but
+  ## if you are using an external proxy then you'd use its hostname
+  ## and port.
+  url: https://localhost
+  ## If dev_mode is true then the port is exposed to the host (as
+  ## plain http).  Do not use in production.  The port is attached
+  ## only to the localhost and will not be available from other
+  ## machines
+  dev_mode: true
+  ## Port to use for the web service
+  port: 8888
+  ## Name of the web service (affects the UI)
+  name: OrderlyWeb
+  ## Email address of the web service
+  email: [email protected]
+  ## Authentication configuration
+  auth:
+    ## Enable or disable fine grained permissions
+    fine_grained: true
+    ## Enable montagu authentication provider
+    montagu: true
+    ## If using Montagu auth you also need the following
+    montagu_url: https://localhost
+    montagu_api_url: http://api:8080/v1/

tests/orderly-web.yml → tests/task_queue/orderly-web.yml

@@ -14,7 +14,7 @@ network: montagu-network
 ##
 ## (More volumes are anticipated as the tool develops)
 volumes:
-  orderly: orderly_web_volume
+  orderly: montagu_orderly_volume
   proxy_logs: orderly_web_proxy_logs
   redis: orderly_web_redis_data