Merge pull request #160 from nf-core/dev

Release candidate 1.4.0

.devcontainer/devcontainer.json

@@ -2,6 +2,7 @@
     "name": "nfcore",
     "image": "nfcore/gitpod:latest",
     "remoteUser": "gitpod",
+    "runArgs": ["--privileged"],
 
     // Configure tool-specific properties.
     "customizations": {

.github/CODEOWNERS

@@ -1,5 +1,7 @@
 # Maintainers
-* @matthdsm @emiller88
+* @nf-core/demultiplex
+*.nf.test* @nf-core/nf-test
+.github/workflows/ @nf-core/a-team
 
 # Element
 modules/nf-core/bases2fastq/ @blajoie

.github/CONTRIBUTING.md

@@ -9,7 +9,9 @@ Please use the pre-filled template to save time.
 However, don't be put off by this template - other more general issues and suggestions are welcome!
 Contributions to the code are even more welcome ;)
 
-> If you need help using or modifying nf-core/demultiplex then the best place to ask is on the nf-core Slack [#demultiplex](https://nfcore.slack.com/channels/demultiplex) channel ([join our Slack here](https://nf-co.re/join/slack)).
+:::info
+If you need help using or modifying nf-core/demultiplex then the best place to ask is on the nf-core Slack [#demultiplex](https://nfcore.slack.com/channels/demultiplex) channel ([join our Slack here](https://nf-co.re/join/slack)).
+:::
 
 ## Contribution workflow
 
@@ -116,4 +118,3 @@ To get started:
 Devcontainer specs:
 
 - [DevContainer config](.devcontainer/devcontainer.json)
-- [Dockerfile](.devcontainer/Dockerfile)

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -42,7 +42,7 @@ body:
     attributes:
       label: System information
       description: |
-        * Nextflow version _(eg. 22.10.1)_
+        * Nextflow version _(eg. 23.04.0)_
         * Hardware _(eg. HPC, Desktop, Cloud)_
         * Executor _(eg. slurm, local, awsbatch)_
         * Container engine: _(e.g. Docker, Singularity, Conda, Podman, Shifter, Charliecloud, or Apptainer)_

.github/workflows/awsfulltest.yml

@@ -14,21 +14,26 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Launch workflow via tower
-        uses: seqeralabs/action-tower-launch@v1
+        uses: seqeralabs/action-tower-launch@v2
         # TODO nf-core: You can customise AWS full pipeline tests as required
         # Add full size test data (but still relatively small datasets for few samples)
         # on the `test_full.config` test runs with only one set of parameters
         with:
           workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }}
           access_token: ${{ secrets.TOWER_ACCESS_TOKEN }}
           compute_env: ${{ secrets.TOWER_COMPUTE_ENV }}
+          revision: ${{ github.sha }}
           workdir: s3://${{ secrets.AWS_S3_BUCKET }}/work/demultiplex/work-${{ github.sha }}
           parameters: |
             {
+              "hook_url": "${{ secrets.MEGATESTS_ALERTS_SLACK_HOOK_URL }}",
               "outdir": "s3://${{ secrets.AWS_S3_BUCKET }}/demultiplex/results-${{ github.sha }}"
             }
-          profiles: test_full,public_aws_ecr,aws_tower
+          profiles: test_full
+
       - uses: actions/upload-artifact@v3
         with:
           name: Tower debug log file
-          path: tower_action_*.log
+          path: |
+            tower_action_*.log
+            tower_action_*.json

.github/workflows/awstest.yml

@@ -12,18 +12,22 @@ jobs:
     steps:
       # Launch workflow using Tower CLI tool action
       - name: Launch workflow via tower
-        uses: seqeralabs/action-tower-launch@v1
+        uses: seqeralabs/action-tower-launch@v2
         with:
           workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }}
           access_token: ${{ secrets.TOWER_ACCESS_TOKEN }}
           compute_env: ${{ secrets.TOWER_COMPUTE_ENV }}
+          revision: ${{ github.sha }}
           workdir: s3://${{ secrets.AWS_S3_BUCKET }}/work/demultiplex/work-${{ github.sha }}
           parameters: |
             {
               "outdir": "s3://${{ secrets.AWS_S3_BUCKET }}/demultiplex/results-test-${{ github.sha }}"
             }
-          profiles: test,public_aws_ecr,aws_tower
+          profiles: test
+
       - uses: actions/upload-artifact@v3
         with:
           name: Tower debug log file
-          path: tower_action_*.log
+          path: |
+            tower_action_*.log
+            tower_action_*.json

.github/workflows/ci.yml

@@ -4,93 +4,73 @@ on:
   pull_request:
     branches:
       - dev
+      - master
   release:
     types: [published]
 
 env:
   NXF_ANSI_LOG: false
+  CAPSULE_LOG: none
+  NFTEST_VER: "0.8.2"
 
 concurrency:
   group: "${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}"
+  cancel-in-progress: true
 
 jobs:
-  changes:
-    name: Check for changes
+  define_nxf_versions:
+    name: Choose nextflow versions to test against depending on target branch
     runs-on: ubuntu-latest
     outputs:
-      # Expose matched filters as job 'modules' output variable
-      tags: ${{ steps.filter.outputs.changes }}
+      matrix: ${{ steps.nxf_versions.outputs.matrix }}
     steps:
-      - uses: actions/checkout@v2
-
-      - uses: dorny/paths-filter@v2
-        id: filter
-        with:
-          filters: "tests/config/tags.yml"
+      - id: nxf_versions
+        run: |
+          if [[ "${{ github.event_name }}" == "pull_request" && "${{ github.base_ref }}" == "dev" && "${{ matrix.NXF_VER }}" != "latest-everything" ]]; then
+            echo matrix='["latest-everything"]' | tee -a $GITHUB_OUTPUT
+          else
+            echo matrix='["latest-everything", "23.04.0"]' | tee -a $GITHUB_OUTPUT
+          fi
 
-  test_changes:
-    name: ${{ matrix.tags }} ${{ matrix.component }} ${{ matrix.profile }} ${{ matrix.NXF_VER }}
-    runs-on: ubuntu-20.04
-    needs: changes
-    if: needs.changes.outputs.tags != '[]'
+  test:
+    name: Run pipeline with test data
+    needs: define_nxf_versions
+    runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        NXF_VER:
-          - "22.10.1"
-          - "latest-everything"
-        tags: ["${{ fromJson(needs.changes.outputs.tags) }}"]
-        profile: ["docker"] # TODO Add singularity/conda
-        component: ["pipeline"] # TODO There aren't any subworkflows or modules tests YET
-        exclude:
-          - NXF_VER: "latest-everything"
-            profile: "conda"
+        NXF_VER: ${{ fromJson(needs.define_nxf_versions.outputs.matrix) }}
+        profile:
+          - docker
     steps:
       - name: Check out pipeline code
         uses: actions/checkout@v3
 
-      - uses: actions/cache@v2
-        with:
-          path: /usr/local/bin/
-          key: ${{ runner.os }}
-          restore-keys: |
-            ${{ runner.os }}-nextflow-
-
+      # Install Nextflow
       - name: Install Nextflow
-        env:
-          CAPSULE_LOG: none
-        run: |
-          wget -qO- get.nextflow.io | bash
-          sudo mv nextflow /usr/local/bin/
+        uses: nf-core/setup-nextflow@v1
+        with:
+          version: "${{ matrix.NXF_VER }}"
 
+      # Install nf-test
       - name: Install nf-test
         run: |
-          wget -qO- https://code.askimed.com/install/nf-test | bash
+          wget -qO- https://code.askimed.com/install/nf-test | bash -s $NFTEST_VER
           sudo mv nf-test /usr/local/bin/
 
-      - name: Set up Singularity
-        if: matrix.profile == 'singularity'
-        uses: eWaterCycle/setup-singularity@v5
-        with:
-          singularity-version: 3.7.1
-
-      - name: Set up miniconda
-        if: matrix.profile == 'conda'
-        uses: conda-incubator/setup-miniconda@v2
-        with:
-          auto-update-conda: true
-          channels: conda-forge,bioconda,defaults
-          python-version: ${{ matrix.python-version }}
-
-      - name: Conda clean
-        if: matrix.profile == 'conda'
-        run: conda clean -a
-
+      # Run nf-test
       - name: Run nf-test
-        run: nf-test test --profile=test,${{ matrix.profile }} tests/${{ matrix.component }}/${{ matrix.tags }}/*.nf.test --junitxml=${{ matrix.tags }}.xml
+        run: nf-test test tests/pipeline/ --profile=test,${{ matrix.profile }} --junitxml=test.xml
+
+      # If the test fails, output the software_versions.yml using the 'batcat' utility
+      - name: Output log on failure
+        if: failure()
+        run: |
+          sudo apt install bat > /dev/null
+          batcat --decorations=always --color=always .nf-test/tests/*/output/pipeline_info/software_versions.yml
 
       - name: Publish Test Report
         uses: mikepenz/action-junit-report@v3
         if: always() # always run even if the previous step fails
         with:
-          report_paths: "${{ matrix.tags }}.xml"
+          report_paths: test.xml

.github/workflows/linting.yml

@@ -78,7 +78,7 @@ jobs:
 
       - uses: actions/setup-python@v4
         with:
-          python-version: "3.8"
+          python-version: "3.11"
           architecture: "x64"
 
       - name: Install dependencies

.github/workflows/release-announcments.yml

@@ -0,0 +1,68 @@
+name: release-announcements
+# Automatic release toot and tweet anouncements
+on:
+  release:
+    types: [published]
+  workflow_dispatch:
+
+jobs:
+  toot:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: rzr/fediverse-action@master
+        with:
+          access-token: ${{ secrets.MASTODON_ACCESS_TOKEN }}
+          host: "mstdn.science" # custom host if not "mastodon.social" (default)
+          # GitHub event payload
+          # https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#release
+          message: |
+            Pipeline release! ${{ github.repository }} v${{ github.event.release.tag_name }} - ${{ github.event.release.name }}!
+
+            Please see the changelog: ${{ github.event.release.html_url }}
+
+  send-tweet:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+      - name: Install dependencies
+        run: pip install tweepy==4.14.0
+      - name: Send tweet
+        shell: python
+        run: |
+          import os
+          import tweepy
+
+          client = tweepy.Client(
+              access_token=os.getenv("TWITTER_ACCESS_TOKEN"),
+              access_token_secret=os.getenv("TWITTER_ACCESS_TOKEN_SECRET"),
+              consumer_key=os.getenv("TWITTER_CONSUMER_KEY"),
+              consumer_secret=os.getenv("TWITTER_CONSUMER_SECRET"),
+          )
+          tweet = os.getenv("TWEET")
+          client.create_tweet(text=tweet)
+        env:
+          TWEET: |
+            Pipeline release! ${{ github.repository }} v${{ github.event.release.tag_name }} - ${{ github.event.release.name }}!
+
+            Please see the changelog: ${{ github.event.release.html_url }}
+          TWITTER_CONSUMER_KEY: ${{ secrets.TWITTER_CONSUMER_KEY }}
+          TWITTER_CONSUMER_SECRET: ${{ secrets.TWITTER_CONSUMER_SECRET }}
+          TWITTER_ACCESS_TOKEN: ${{ secrets.TWITTER_ACCESS_TOKEN }}
+          TWITTER_ACCESS_TOKEN_SECRET: ${{ secrets.TWITTER_ACCESS_TOKEN_SECRET }}
+
+  bsky-post:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: zentered/[email protected]
+        with:
+          post: |
+            Pipeline release! ${{ github.repository }} v${{ github.event.release.tag_name }} - ${{ github.event.release.name }}!
+
+            Please see the changelog: ${{ github.event.release.html_url }}
+        env:
+          BSKY_IDENTIFIER: ${{ secrets.BSKY_IDENTIFIER }}
+          BSKY_PASSWORD: ${{ secrets.BSKY_PASSWORD }}
+          #

.gitpod.yml

@@ -1,4 +1,9 @@
 image: nfcore/gitpod:latest
+tasks:
+  - name: Update Nextflow and setup pre-commit
+    command: |
+      pre-commit install --install-hooks
+      nextflow self-update
 
 vscode:
   extensions: # based on nf-core.nf-core-extensionpack

.nf-core.yml

@@ -1,3 +1,5 @@
 repository_type: pipeline
 lint:
   actions_ci: False
+  files_unchanged:
+    - .github/ISSUE_TEMPLATE/bug_report.yml

CHANGELOG.md

@@ -3,6 +3,32 @@
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [Unpublished Version / DEV]
+
+## 1.4.0 - 2023-12-14
+
+### `Added`
+
+- [#148](https://github.com/nf-core/demultiplex/pull/148) Update CODEOWNERS to use GitHub teams
+
+### `Changed`
+
+- [#141](https://github.com/nf-core/demultiplex/pull/141) Updated template to nf-core/tools v2.10
+- [#152](https://github.com/nf-core/demultiplex/pull/152) Updated bclconvert module to 4.2.4
+
+### `Fixed`
+
+- [#127](https://github.com/nf-core/demultiplex/pull/127) Add `singularity.registry = 'quay.io'` and bump NF version to 23.04.0
+- [#140](https://github.com/nf-core/demultiplex/pull/140) Make it possible to skip MultiQC, fix error raising
+- [#145](https://github.com/nf-core/demultiplex/pull/145) Fix MultiQC report generation
+- [#152](https://github.com/nf-core/demultiplex/pull/152) Close [#150](https://github.com/nf-core/demultiplex/issues/150)
+- [#157](https://github.com/nf-core/demultiplex/pull/157) Fix bcl2fastq and bclconvert publishDir
+- [#158](https://github.com/nf-core/demultiplex/pull/158) Update all modules
+
+## `Removed`
+
+- [#130](https://github.com/nf-core/demultiplex/pull/130) Remove `public_aws_ecr` profile.
+
 ## 1.3.2 - 2023-06-07
 
 ### `Fixed`

CITATIONS.md

@@ -36,5 +36,8 @@
 
 - [Docker](https://dl.acm.org/doi/10.5555/2600239.2600241)
 
+  > Merkel, D. (2014). Docker: lightweight linux containers for consistent development and deployment. Linux Journal, 2014(239), 2. doi: 10.5555/2600239.2600241.
+
 - [Singularity](https://pubmed.ncbi.nlm.nih.gov/28494014/)
+
   > Kurtzer GM, Sochat V, Bauer MW. Singularity: Scientific containers for mobility of compute. PLoS One. 2017 May 11;12(5):e0177459. doi: 10.1371/journal.pone.0177459. eCollection 2017. PubMed PMID: 28494014; PubMed Central PMCID: PMC5426675.