Rubicon

.gitignore

@@ -10,6 +10,7 @@ __pycache__/
 .Python
 env/
 build/
+vendor/
 develop-eggs/
 dist/
 downloads/

.mega-linter.yml

@@ -9,6 +9,8 @@ ENABLE_LINTERS:
   - SNAKEMAKE_SNAKEFMT
   - MARKDOWN_MARKDOWNLINT
   - JUPYTER_JUPYFMT
+  - PERL_PERLCRITIC
+  - DOCKERFILE_HADOLINT
 
 VALIDATE_ALL_CODEBASE: true
 FORMATTERS_DISABLE_ERRORS: false

Dockerfile

@@ -24,6 +24,7 @@ ARG vpipe_path
 ARG envs_path
 ARG test_data
 
+# hadolint ignore=DL3008
 RUN apt-get update && apt-get install -y --no-install-recommends \
     jdupes
 
@@ -42,11 +43,11 @@ COPY tests/data ${test_data}
 WORKDIR /work
 
 # configuration: activate all steps
-RUN mkdir config
-RUN echo 'output:\n  snv: true\n  local: true\n  global: true\n  visualization: true\n  diversity: true\n  QA: true' > config/config.yaml
+RUN mkdir config \
+ && printf 'output:\n  snv: true\n  local: true\n  global: true\n  visualization: true\n  diversity: true\n  QA: true\n  upload: true\nupload:\n  orig_cram: true' > config/config.yaml
 
 # TODO harmonize list with CI tests and Docker tests
-RUN for virus in ${virus_download_list:-$(ls ${test_data}/)}; do echo "\n\n\e[36;1mvirus: ${virus}\e[0m\n" \
+RUN for virus in ${virus_download_list:-$(ls ${test_data}/)}; do printf '\n\n\e[36;1mvirus: %s\e[0m\n' "${virus}" \
  &&   ln -sf "${test_data}/${virus}/" ./samples \
  &&   if test -e samples/samples.tsv; then cp -f samples/samples.tsv ./config/samples.tsv; fi \
  &&   PYTHONUNBUFFERED=1 snakemake -s ${vpipe_path}/workflow/Snakefile -j 1 --conda-create-envs-only --use-conda --conda-prefix ${envs_path} --config "general={virus_base_config: ${virus}}" \
@@ -62,6 +63,7 @@ FROM snakemake/snakemake:${snaketag} AS vpipe-tests-base
 ARG install_path
 
 # NOTE rsync only used with local scratch
+# hadolint ignore=DL3008
 RUN apt-get update && apt-get install -y --no-install-recommends \
     rsync \
  && apt-get clean \
@@ -83,8 +85,8 @@ ARG test_data
 ENV virus=hiv
 
 WORKDIR /work
-RUN mkdir config
-RUN echo 'output:\n  snv: true\n  local: true\n  global: false\n  visualization: true\n  diversity: true\n  QA: true' > config/config.yaml
+RUN mkdir config \
+ && printf 'output:\n  snv: true\n  local: true\n  global: false\n  visualization: true\n  diversity: true\n  QA: true\n  upload: true\nupload:\n  orig_cram: true' > config/config.yaml
 COPY --from=create-envs ${test_data}/${virus} ./samples
 RUN if test -e samples/samples.tsv; then cp -f samples/samples.tsv ./config/samples.tsv; fi
 # NOTE see top comment if `--network=none` breaks build process
@@ -105,8 +107,8 @@ ARG test_data
 ENV virus=sars-cov-2
 
 WORKDIR /work
-RUN mkdir config
-RUN echo 'output:\n  snv: true\n  local: true\n  global: false\n  visualization: true\n  diversity: true\n  QA: true' > config/config.yaml
+RUN mkdir config \
+ && printf 'output:\n  snv: true\n  local: true\n  global: false\n  visualization: true\n  diversity: true\n  QA: true\n  upload: true\nupload:\n  orig_cram: true' > config/config.yaml
 COPY --from=create-envs ${test_data}/${virus} ./samples
 RUN if test -e samples/samples.tsv; then cp -f samples/samples.tsv ./config/samples.tsv; fi
 # NOTE see top comment if `--network=none` breaks build process
@@ -145,6 +147,7 @@ ARG install_path
 ARG vpipe_path
 ARG envs_path
 
+# hadolint ignore=DL3008
 RUN apt-get update && apt-get install -y --no-install-recommends \
     rsync \
  && apt-get clean \
@@ -153,7 +156,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 COPY --from=vpipe-final-base ${install_path} ${install_path}
 # =============================================
 
-MAINTAINER V-pipe Dev Team <[email protected]>
+LABEL maintainer="V-pipe Dev Team <[email protected]>"
 VOLUME /work
 WORKDIR /work
 

tests/regression_tests.sh

@@ -56,6 +56,10 @@ output:
     visualization: true
     diversity: true
     QA: true
+    upload: true
+
+upload:
+    orig_cram: true
 
 snv:
     threads: ${THREADS}

workflow/Snakefile

@@ -31,12 +31,14 @@ rule allfastqc:
         fastqc_files,
 
 
-include: "rules/clean.smk"
 include: "rules/quality_assurance.smk"
 include: "rules/align.smk"
 include: "rules/consensus.smk"
 include: "rules/mafs.smk"
 include: "rules/stats.smk"
+include: "rules/publish.smk"
+include: "rules/dehuman.smk"
+include: "rules/clean.smk"
 
 
 if config.output["snv"] or config.output["local"]:

workflow/envs/dehuman.yaml

@@ -0,0 +1,9 @@
+channels:
+  - conda-forge
+  - bioconda
+  - defaults
+dependencies:
+  - bwa=0.7.17
+  - samtools>1.11
+  - xxhash
+  - coreutils # [not linux]

workflow/envs/upload.yaml

@@ -0,0 +1,7 @@
+channels:
+  - conda-forge
+  - bioconda
+  - defaults
+dependencies:
+  - xxhash
+  - coreutils # [not linux]