diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index e244181..7830238 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -1,31 +1,42 @@ +# nf-core/imcyto bug report + Hi there! -Thanks for telling us about a problem with the pipeline. Please delete this text and anything that's not relevant from the template below: +Thanks for telling us about a problem with the pipeline. +Please delete this text and anything that's not relevant from the template below: + +## Describe the bug -#### Describe the bug A clear and concise description of what the bug is. -#### Steps to reproduce +## Steps to reproduce + Steps to reproduce the behaviour: + 1. Command line: `nextflow run ...` 2. See error: _Please provide your error message_ -#### Expected behaviour +## Expected behaviour + A clear and concise description of what you expected to happen. -#### System: - - Hardware: [e.g. HPC, Desktop, Cloud...] - - Executor: [e.g. slurm, local, awsbatch...] - - OS: [e.g. CentOS Linux, macOS, Linux Mint...] - - Version [e.g. 7, 10.13.6, 18.3...] +## System + +- Hardware: +- Executor: +- OS: +- Version + +## Nextflow Installation + +- Version: + +## Container engine -#### Nextflow Installation: - - Version: [e.g. 0.31.0] +- Engine: +- version: +- Image tag: -#### Container engine: - - Engine: [e.g. Conda, Docker or Singularity] - - version: [e.g. 1.0.0] - - Image tag: [e.g. nfcore/imcyto:1.0.0] +## Additional context -#### Additional context Add any other context about the problem here. diff --git a/.github/workflows/branch.yml b/.github/workflows/branch.yml old mode 100755 new mode 100644 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml old mode 100755 new mode 100644 index 6fe1261..6e982bd --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,28 +5,46 @@ on: [push, pull_request] jobs: test: - runs-on: ubuntu-18.04 + env: + NXF_VER: ${{ matrix.nxf_ver }} + NXF_ANSI_LOG: false + runs-on: ubuntu-latest strategy: matrix: # Nextflow versions: check pipeline minimum and current latest nxf_ver: ['19.10.0', ''] steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - name: Install Nextflow run: | - export NXF_VER=${{ matrix.nxf_ver }} - export NXF_ANSI_LOG=false wget -qO- get.nextflow.io | bash sudo mv nextflow /usr/local/bin/ - #- name: Pull image (Cant pull standard single image for this pipeline as we are using individual BioContainers) + #- name: Pull docker image (Cant pull standard single image for this pipeline as we are using individual containers) # run: | - # docker pull nfcore/imcyto:dev && docker tag nfcore/imcyto:dev nfcore/imcyto:dev + # docker pull nfcore/imcyto:dev + # docker tag nfcore/imcyto:dev nfcore/imcyto:1.0.0 - name: Run the pipeline with .mcd input run: | nextflow run ${GITHUB_WORKSPACE} -profile test,docker - - name: Run the pipeline with .txt input + + config: + env: + NXF_VER: '19.10.0' + NXF_ANSI_LOG: false + runs-on: ubuntu-latest + strategy: + matrix: + config: [test_txt, test_tiff, test_skip_ilastik] + steps: + - uses: actions/checkout@v2 + - name: Install Nextflow run: | - nextflow run ${GITHUB_WORKSPACE} -profile test_txt,docker - - name: Run the pipeline with .tiff input + wget -qO- get.nextflow.io | bash + sudo mv nextflow /usr/local/bin/ + #- name: Pull docker image (Cant pull standard single image for this pipeline as we are using individual containers) + # run: | + # docker pull nfcore/imcyto:dev + # docker tag nfcore/imcyto:dev nfcore/imcyto:1.0.0 + - name: Run pipeline with various test configs run: | - nextflow run ${GITHUB_WORKSPACE} -profile test_tiff,docker + nextflow run ${GITHUB_WORKSPACE} -profile ${{ matrix.config }},docker diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml old mode 100755 new mode 100644 index 7354dc7..1e0827a --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -1,26 +1,39 @@ name: nf-core linting # This workflow is triggered on pushes and PRs to the repository. # It runs the `nf-core lint` and markdown lint tests to ensure that the code meets the nf-core guidelines -on: [push, pull_request] +on: + push: + pull_request: + release: + types: [published] jobs: Markdown: - runs-on: ubuntu-18.04 + runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - uses: actions/setup-node@v1 with: node-version: '10' - name: Install markdownlint - run: | - npm install -g markdownlint-cli + run: npm install -g markdownlint-cli - name: Run Markdownlint - run: | - markdownlint ${GITHUB_WORKSPACE} -c ${GITHUB_WORKSPACE}/.github/markdownlint.yml - nf-core: + run: markdownlint ${GITHUB_WORKSPACE} -c ${GITHUB_WORKSPACE}/.github/markdownlint.yml + YAML: runs-on: ubuntu-latest steps: - uses: actions/checkout@v1 + - uses: actions/setup-node@v1 + with: + node-version: '10' + - name: Install yaml-lint + run: npm install -g yaml-lint + - name: Run yaml-lint + run: yamllint $(find ${GITHUB_WORKSPACE} -type f -name "*.yml") + nf-core: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 - name: Install Nextflow run: | wget -qO- get.nextflow.io | bash @@ -29,13 +42,9 @@ jobs: with: python-version: '3.6' architecture: 'x64' - - name: Install pip - run: | - sudo apt install python3-pip - pip install --upgrade pip - - name: Install nf-core tools + - name: Install dependencies run: | + python -m pip install --upgrade pip pip install nf-core - name: Run nf-core lint - run: | - nf-core lint ${GITHUB_WORKSPACE} + run: nf-core lint ${GITHUB_WORKSPACE} diff --git a/.gitignore b/.gitignore index 94e9d1e..6354f37 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,6 @@ work/ data/ results/ .DS_Store +tests/ +testing/ *.pyc diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index acf2908..0000000 --- a/.travis.yml +++ /dev/null @@ -1,47 +0,0 @@ -sudo: required -language: python -jdk: openjdk8 -services: docker -python: '3.6' -cache: pip -matrix: - fast_finish: true - -before_install: - # PRs to master are only ok if coming from dev branch - #- '[ $TRAVIS_PULL_REQUEST = "false" ] || [ $TRAVIS_BRANCH != "master" ] || ([ $TRAVIS_PULL_REQUEST_SLUG = $TRAVIS_REPO_SLUG ] && [ $TRAVIS_PULL_REQUEST_BRANCH = "dev" ]) || [ $TRAVIS_PULL_REQUEST_BRANCH = "patch" ]' - - '[ $TRAVIS_PULL_REQUEST = "false" ] || [ $TRAVIS_BRANCH != "master" ] || ([ $TRAVIS_PULL_REQUEST_SLUG = $TRAVIS_REPO_SLUG ] && ([ $TRAVIS_PULL_REQUEST_BRANCH = "dev" ] || [ $TRAVIS_PULL_REQUEST_BRANCH = "patch" ]))' - # Pull the docker image first so the test doesn't wait for this - #- docker pull nfcore/imcyto:dev - # Fake the tag locally so that the pipeline runs properly - # Looks weird when this is :dev to :dev, but makes sense when testing code for a release (:dev to :1.0.1) - #- docker tag nfcore/imcyto:dev nfcore/imcyto:dev - -install: - # Install Nextflow - - mkdir /tmp/nextflow && cd /tmp/nextflow - - wget -qO- get.nextflow.io | bash - - sudo ln -s /tmp/nextflow/nextflow /usr/local/bin/nextflow - # Install nf-core/tools - - pip install --upgrade pip - - pip install nf-core - # Reset - - mkdir ${TRAVIS_BUILD_DIR}/tests && cd ${TRAVIS_BUILD_DIR}/tests - # Install markdownlint-cli - - sudo apt-get install npm && npm install -g markdownlint-cli - -env: - - NXF_VER='19.10.0' # Specify a minimum NF version that should be tested and work - - NXF_VER='' # Plus: get the latest NF version and check that it works - -script: - # Lint the pipeline code - - nf-core lint ${TRAVIS_BUILD_DIR} - # Lint the documentation - - markdownlint ${TRAVIS_BUILD_DIR} -c ${TRAVIS_BUILD_DIR}/.github/markdownlint.yml - # Run the pipeline with .mcd input - - nextflow run ${TRAVIS_BUILD_DIR} -ansi-log false -profile test,docker - # Run the pipeline with .txt input - - nextflow run ${TRAVIS_BUILD_DIR} -ansi-log false -profile test_txt,docker - # Run the pipeline with .tiff input - - nextflow run ${TRAVIS_BUILD_DIR} -ansi-log false -profile test_tiff,docker diff --git a/CHANGELOG.md b/CHANGELOG.md index 16aa0db..9db9a73 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # nf-core/imcyto: Changelog -## v1.0dev - [date] +The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) +and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). + +## [1.0.0] - 2020-05-29 Initial release of nf-core/imcyto, created with the [nf-core](http://nf-co.re/) template. diff --git a/CITATIONS.md b/CITATIONS.md index c57e2ce..d41f4d8 100644 --- a/CITATIONS.md +++ b/CITATIONS.md @@ -1,9 +1,14 @@ # nf-core/imcyto: Citations -## Pipeline tools +## [nf-core](https://www.ncbi.nlm.nih.gov/pubmed/32055031/) + +> Ewels PA, Peltzer A, Fillinger S, Patel H, Alneberg J, Wilm A, Garcia MU, Di Tommaso P, Nahnsen S. The nf-core framework for community-curated bioinformatics pipelines. Nat Biotechnol. 2020 Mar;38(3):276-278. doi: 10.1038/s41587-020-0439-x. PubMed PMID: 32055031. ReadCube: [Full Access Link](https://rdcu.be/b1GjZ). + +## [Nextflow](https://www.ncbi.nlm.nih.gov/pubmed/28398311/) -* [Nextflow](https://www.ncbi.nlm.nih.gov/pubmed/28398311/) - > Di Tommaso P, Chatzou M, Floden EW, Barja PP, Palumbo E, Notredame C. Nextflow enables reproducible computational workflows. Nat Biotechnol. 2017 Apr 11;35(4):316-319. doi: 10.1038/nbt.3820. PubMed PMID: 28398311. +> Di Tommaso P, Chatzou M, Floden EW, Barja PP, Palumbo E, Notredame C. Nextflow enables reproducible computational workflows. Nat Biotechnol. 2017 Apr 11;35(4):316-319. doi: 10.1038/nbt.3820. PubMed PMID: 28398311. + +## Pipeline tools * [CellProfiler](https://www.ncbi.nlm.nih.gov/pubmed/29969450/) > McQuin C, Goodman A, Chernyshev V, Kamentsky L, Cimini BA, Karhohs KW, Doan M, Ding L, Rafelski SM, Thirstrup D, Wiegraebe W, Singh S, Becker T, Caicedo JC, Carpenter AE. CellProfiler 3.0: Next-generation image processing for biology. PLoS Biol. 2018 Jul 3;16(7):e2005970. doi: 10.1371/journal.pbio.2005970. eCollection 2018 Jul. PubMed PMID: 29969450; PubMed Central PMCID: PMC6029841. @@ -28,4 +33,4 @@ * [Singularity](https://www.ncbi.nlm.nih.gov/pubmed/28494014/) > Kurtzer GM, Sochat V, Bauer MW. Singularity: Scientific containers for mobility of compute. PLoS One. 2017 May 11;12(5):e0177459. doi: 10.1371/journal.pone.0177459. eCollection 2017. PubMed PMID: 28494014; PubMed Central PMCID: PMC5426675. -* [Docker](https://www.docker.com/) +* [Docker](https://dl.acm.org/doi/10.5555/2600239.2600241) diff --git a/Dockerfile b/Dockerfile index a50e8c1..88158bb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM nfcore/base:1.7 +FROM nfcore/base:1.9 LABEL authors="Harshil Patel" \ description="Docker image containing all software requirements for the nf-core/imcyto pipeline" @@ -11,7 +11,7 @@ COPY environment.yml / RUN conda env create -f /environment.yml && conda clean -a # Add conda installation dir to PATH (instead of doing 'conda activate') -ENV PATH /opt/conda/envs/nf-core-imcyto-1.0dev/bin:$PATH +ENV PATH /opt/conda/envs/nf-core-imcyto-1.0.0/bin:$PATH # Dump the details of the installed packages to a file for posterity -RUN conda env export --name nf-core-imcyto-1.0dev > nf-core-imcyto-1.0dev.yml +RUN conda env export --name nf-core-imcyto-1.0.0 > nf-core-imcyto-1.0.0.yml diff --git a/README.md b/README.md index f292593..fbd0226 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,18 @@ -# ![nfcore/imcyto](docs/images/nf-core-imcyto_logo.png) +# ![nf-core/imcyto](docs/images/nf-core-imcyto_logo.png) -[![Build Status](https://travis-ci.com/nf-core/imcyto.svg?branch=master)](https://travis-ci.com/nf-core/imcyto) [![GitHub Actions CI Status](https://github.com/nf-core/imcyto/workflows/nf-core%20CI/badge.svg)](https://github.com/nf-core/imcyto/actions) [![GitHub Actions Linting Status](https://github.com/nf-core/imcyto/workflows/nf-core%20linting/badge.svg)](https://github.com/nf-core/imcyto/actions) [![Nextflow](https://img.shields.io/badge/nextflow-%E2%89%A519.10.0-brightgreen.svg)](https://www.nextflow.io/) + ## Introduction -**nfcore/imcyto** is a bioinformatics analysis pipeline used for image segmentation and extraction of single cell expression data. This pipeline was generated for Imaging Mass Cytometry data, however, it is flexible enough to be applicable to other types of imaging data (e.g. immunofluorescence/immunohistochemistry data). +**nfcore/imcyto** is a bioinformatics analysis pipeline used for image segmentation and extraction of single cell expression data. This pipeline was generated for Imaging Mass Cytometry data, however, it is flexible enough to be applicable to other types of imaging data e.g. immunofluorescence/immunohistochemistry data. The pipeline is built using [Nextflow](https://www.nextflow.io), a workflow tool to run tasks across multiple compute infrastructures in a very portable manner. It comes with docker containers making installation trivial and results highly reproducible. ## Pipeline summary - - 1. Split image acquisition output files (`mcd`, `ome.tiff` or `txt`) by ROI and convert to individual `tiff` files for channels with names matching those defined in user-provided `metadata.csv` file. Full and ilastik stacks will be generated separately for all channels being analysed in single cell expression analysis, and for channels being used to generate the cell mask, respectively ([imctools](https://github.com/BodenmillerGroup/imctools)). 2. Apply pre-processing filters to full stack `tiff` files ([CellProfiler](https://cellprofiler.org/)). @@ -37,12 +35,10 @@ iii. Download the pipeline and test it on a minimal dataset with a single comman nextflow run nf-core/imcyto -profile test, ``` -> Please check [nf-core/configs](https://github.com/nf-core/configs#documentation) to see if a custom config file to run nf-core pipelines already exists for your Institute. If so, you can simply use `-profile institute` in your command. This will enable either `docker` or `singularity` and set the appropriate execution settings for your local compute environment. +> Please check [nf-core/configs](https://github.com/nf-core/configs#documentation) to see if a custom config file to run nf-core pipelines already exists for your Institute. If so, you can simply use `-profile ` in your command. This will enable either `docker` or `singularity` and set the appropriate execution settings for your local compute environment. iv. Start running your own analysis! - - ```bash nextflow run nf-core/imcyto \ --input "./inputs/*.mcd" \ @@ -90,7 +86,13 @@ For further information or help, don't hesitate to get in touch on [Slack](https -You can cite the `nf-core` pre-print as follows: -> Ewels PA, Peltzer A, Fillinger S, Alneberg JA, Patel H, Wilm A, Garcia MU, Di Tommaso P, Nahnsen S. **nf-core: Community curated bioinformatics pipelines**. *bioRxiv*. 2019. p. 610741. [doi: 10.1101/610741](https://www.biorxiv.org/content/10.1101/610741v1). +You can cite the `nf-core` publication as follows: + +> **The nf-core framework for community-curated bioinformatics pipelines.** +> +> Philip Ewels, Alexander Peltzer, Sven Fillinger, Harshil Patel, Johannes Alneberg, Andreas Wilm, Maxime Ulysse Garcia, Paolo Di Tommaso & Sven Nahnsen. +> +> _Nat Biotechnol._ 2020 Feb 13. doi: [10.1038/s41587-020-0439-x](https://dx.doi.org/10.1038/s41587-020-0439-x). +> ReadCube: [Full Access Link](https://rdcu.be/b1GjZ) An extensive list of references for the tools used by the pipeline can be found in the [`CITATIONS.md`](CITATIONS.md) file. diff --git a/assets/nf-core-imcyto_logo.png b/assets/nf-core-imcyto_logo.png old mode 100755 new mode 100644 diff --git a/assets/nf-core-imcyto_social_preview.png b/assets/nf-core-imcyto_social_preview.png new file mode 100644 index 0000000..cff8178 Binary files /dev/null and b/assets/nf-core-imcyto_social_preview.png differ diff --git a/assets/nf-core-imcyto_social_preview.svg b/assets/nf-core-imcyto_social_preview.svg new file mode 100644 index 0000000..834d7e8 --- /dev/null +++ b/assets/nf-core-imcyto_social_preview.svg @@ -0,0 +1,448 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + Image Mass Cytometry analysis pipeline + imcyto + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/bin/scrape_software_versions.py b/bin/scrape_software_versions.py index 4819832..141bd72 100755 --- a/bin/scrape_software_versions.py +++ b/bin/scrape_software_versions.py @@ -30,7 +30,7 @@ results[k] = False # Remove software set to false in results -for k in results: +for k in list(results): if not results[k]: del(results[k]) diff --git a/conf/base.config b/conf/base.config index 3081cf9..481cfed 100644 --- a/conf/base.config +++ b/conf/base.config @@ -38,23 +38,20 @@ process { withLabel:process_long { time = { check_max( 20.h * task.attempt, 'time' ) } } - withName:get_software_versions { - cache = false - } - withName:IMCTools { + withName:IMCTOOLS { container = 'quay.io/biocontainers/imctools:1.0.5--py_0' } - withName:PreprocessFullStack { + withName:PREPROCESS_FULL_STACK { container = 'cellprofiler/cellprofiler:3.1.9' } - withName:PreprocessIlastikStack { + withName:PREPROCESS_ILASTIK_STACK { container = 'cellprofiler/cellprofiler:3.1.9' } - withName:Ilastik { - container = 'ilastik/ilastik-from-binary:1.3.3post2' + withName:ILASTIK { + container = 'ilastik/ilastik-from-binary:1.3.3b1' } - withName:Segmentation { + withName:SEGMENTATION { container = 'cellprofiler/cellprofiler:3.1.9' } withName:output_documentation { diff --git a/conf/test.config b/conf/test.config index 13a64dc..93540f0 100644 --- a/conf/test.config +++ b/conf/test.config @@ -12,7 +12,7 @@ params { config_profile_name = 'Test profile' config_profile_description = 'Minimal test dataset to check pipeline function' - // Limit resources so that this can run on Travis + // Limit resources so that this can run on GitHub Actions max_cpus = 2 max_memory = 6.GB max_time = 12.h diff --git a/conf/test_skip_ilastik.config b/conf/test_skip_ilastik.config new file mode 100644 index 0000000..ebc3f07 --- /dev/null +++ b/conf/test_skip_ilastik.config @@ -0,0 +1,30 @@ +/* + * ------------------------------------------------- + * Nextflow config file for running tests + * ------------------------------------------------- + * Defines bundled input files and everything required + * to run a fast and simple test. Use as follows: + * nextflow run nf-core/imcyto -profile test_skip_ilastik, + */ + +params { + + config_profile_name = 'Test profile' + config_profile_description = 'Minimal test dataset to check pipeline function' + + // Limit resources so that this can run on GitHub Actions + max_cpus = 2 + max_memory = 6.GB + max_time = 12.h + + // Test input data in *.mcd format and --skip_ilastik parameter + input = 'https://raw.githubusercontent.com/nf-core/test-datasets/imcyto/inputs/20191121_BRAC3438.6f.mcd' + metadata = 'https://raw.githubusercontent.com/nf-core/test-datasets/imcyto/inputs/metadata.csv' + + full_stack_cppipe = 'https://raw.githubusercontent.com/nf-core/test-datasets/imcyto/plugins/full_stack_preprocessing.cppipe' + ilastik_stack_cppipe = 'https://raw.githubusercontent.com/nf-core/test-datasets/imcyto/plugins/ilastik_stack_preprocessing.cppipe' + segmentation_cppipe = 'https://raw.githubusercontent.com/nf-core/test-datasets/imcyto/plugins/segmentation_skip_ilastik.cppipe' + ilastik_training_ilp = 'https://raw.githubusercontent.com/nf-core/test-datasets/imcyto/plugins/ilastik_training_params.ilp' + + skip_ilastik = true +} diff --git a/conf/test_tiff.config b/conf/test_tiff.config index 29ac815..d0e119c 100755 --- a/conf/test_tiff.config +++ b/conf/test_tiff.config @@ -4,7 +4,7 @@ * ------------------------------------------------- * Defines bundled input files and everything required * to run a fast and simple test. Use as follows: - * nextflow run nf-core/imcyto -profile test, + * nextflow run nf-core/imcyto -profile test_tiff, */ params { @@ -12,7 +12,7 @@ params { config_profile_name = 'Test profile' config_profile_description = 'Minimal test dataset to check pipeline function' - // Limit resources so that this can run on Travis + // Limit resources so that this can run on GitHub Actions max_cpus = 2 max_memory = 6.GB max_time = 12.h diff --git a/conf/test_txt.config b/conf/test_txt.config index f6c4d1a..37c7f02 100755 --- a/conf/test_txt.config +++ b/conf/test_txt.config @@ -4,7 +4,7 @@ * ------------------------------------------------- * Defines bundled input files and everything required * to run a fast and simple test. Use as follows: - * nextflow run nf-core/imcyto -profile test, + * nextflow run nf-core/imcyto -profile test_txt, */ params { @@ -12,7 +12,7 @@ params { config_profile_name = 'Test profile' config_profile_description = 'Minimal test dataset to check pipeline function' - // Limit resources so that this can run on Travis + // Limit resources so that this can run on GitHub Actions max_cpus = 2 max_memory = 6.GB max_time = 12.h diff --git a/docs/images/nf-core-imcyto_logo.png b/docs/images/nf-core-imcyto_logo.png old mode 100755 new mode 100644 diff --git a/docs/output.md b/docs/output.md index 5153716..4fce111 100644 --- a/docs/output.md +++ b/docs/output.md @@ -6,7 +6,7 @@ The pipeline is built using [Nextflow](https://www.nextflow.io/). See main [`REA This is an automated image analysis pipeline that sequentially pre-processes and single cell segments imaging data to extract single cell expression data. This pipeline was generated for Imaging Mass Cytometry experiments, however, it is flexible enough to be applicable to other types of imaging data (e.g. immunofluorescence/immunohistochemistry data). -The input to the pipeline can be in either `mcd`, `ome.tiff` or `txt` file format from which stacks of `tiff` files are generated for subsequent analysis. The various stages of this pipeline allow the `tiff` images to be pre-processed, and segmented using multiple CellProfiler `cppipe` project files and the pixel-classification software Ilastik. The concept of this step-wise image segmentation by combining Ilastik with CellProfiler was based on the analysis pipeline as described by the Bodenmiller group [(Zanotelli & Bodenmiller, Jan 2019)](https://github.com/BodenmillerGroup/ImcSegmentationPipeline/blob/development/documentation/imcsegmentationpipeline_documentation.pdf). +The input to the pipeline can be in either `mcd`, `tiff` or `txt` file format from which stacks of `tiff` files are generated for subsequent analysis. The various stages of this pipeline allow the `tiff` images to be pre-processed, and segmented using multiple CellProfiler `cppipe` project files and the pixel-classification software Ilastik. The concept of this step-wise image segmentation by combining Ilastik with CellProfiler was based on the analysis pipeline as described by the Bodenmiller group [(Zanotelli & Bodenmiller, Jan 2019)](https://github.com/BodenmillerGroup/ImcSegmentationPipeline/blob/development/documentation/imcsegmentationpipeline_documentation.pdf). The project files supplied with the pipeline constitute the minimal requirements to generate a single cell mask. A more refined and comprehensive pipeline will be uploaded in due course. @@ -18,7 +18,7 @@ This pipeline is designed to run on most compute infrastructures without the nee ## File prerequisites -1. `mcd`, `ome.tiff` or `txt` data file(s) without any spaces in the file names. Associated antibody panel should contain metal and antibody information in the form of "metal_antibody" e.g. "89Y_CD45". +1. `mcd`, `tiff` or `txt` data file(s) without any spaces in the file names. Associated antibody panel should contain metal and antibody information in the form of "metal_antibody" e.g. "89Y_CD45". 2. `metadata.csv` file containing your antibody panel to identify which `tiff` files are to be used for the full and Ilastik stacks. See [`--metadata.csv`](usage.md#--metadata) for the required file format. @@ -55,8 +55,6 @@ Once you have created/obtained the files required to run the pipeline, the direc └── segmentation.cppipe ``` - - Providing you have installed the version of Nextflow required by the pipeline, and you either have Docker or Singularity installed and available on the `PATH`, you can then execute the pipeline on your compute infrastructure using the command below: ```bash @@ -81,7 +79,7 @@ Each step of the pipeline as depicted in the pipeline schematic is broken down a *Input files*: -* `mcd`, `ome.tiff` or `txt` file +* `mcd`, `tiff` or `txt` file * `metadata.csv` file *Output files*: diff --git a/docs/usage.md b/docs/usage.md index d39ace4..08c6191 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -2,8 +2,6 @@ ## Table of contents - - * [Table of contents](#table-of-contents) * [Introduction](#introduction) * [Running the pipeline](#running-the-pipeline) @@ -29,6 +27,7 @@ * [`--awscli`](#--awscli) * [Other command line parameters](#other-command-line-parameters) * [`--outdir`](#--outdir) + * [`--publish_dir_mode`](#--publish_dir_mode) * [`--email`](#--email) * [`--email_on_fail`](#--email_on_fail) * [`-name`](#-name) @@ -41,7 +40,6 @@ * [`--max_cpus`](#--max_cpus) * [`--plaintext_email`](#--plaintext_email) * [`--monochrome_logs`](#--monochrome_logs) - ## Introduction @@ -100,9 +98,16 @@ This version number will be logged in reports when you run the pipeline, so that ### `-profile` -Use this parameter to choose a configuration profile. Profiles can give configuration presets for different compute environments. Note that multiple profiles can be loaded, for example: `-profile docker` - the order of arguments is important! +Use this parameter to choose a configuration profile. Profiles can give configuration presets for different compute environments. -If `-profile` is not specified at all the pipeline will be run locally and expects all software to be installed and available on the `PATH`. +Several generic profiles are bundled with the pipeline which instruct the pipeline to use software packaged using different methods (Docker, Singularity) - see below. + +The pipeline also dynamically loads configurations from [https://github.com/nf-core/configs](https://github.com/nf-core/configs) when it runs, making multiple config profiles for various institutional clusters available at run time. For more information and to see if your system is available in these configs please see the [nf-core/configs documentation](https://github.com/nf-core/configs#documentation). + +Note that multiple profiles can be loaded, for example: `-profile test,docker` - the order of arguments is important! +They are loaded in sequence, so later profiles can overwrite earlier profiles. + +If `-profile` is not specified, the pipeline will run locally and expect all software to be installed and available on the `PATH`. This is _not_ recommended. * `docker` * A generic configuration profile to be used with [Docker](http://docker.com/) @@ -114,11 +119,9 @@ If `-profile` is not specified at all the pipeline will be run locally and expec * A profile with a complete configuration for automated testing * Includes links to test data so needs no other parameters - - ### `--input` -Path to input data file(s) (globs must be surrounded with quotes). Currently supported formats are `*.mcd`. +Path to input data file(s) (globs must be surrounded with quotes). Currently supported formats are `*.mcd`, `*.tiff` and `*.txt`. ```bash --input "./mcd/*.mcd" @@ -126,7 +129,7 @@ Path to input data file(s) (globs must be surrounded with quotes). Currently sup ### `--metadata` -Path to metadata `csv` file indicating which images to merge in full and/or Ilastik stack. The file should only contain 3 columns i.e. 'metal', 'full_stack' and 'ilastik_stack'. The `metal` column should contain all the metals used in your antibody panel. The `full_stack` and `ilastik_stack` entries should be `1` or `0` to indicate whehter to include or exclude a metal for a given stack, respectively. See [`metadata.csv`](https://github.com/nf-core/test-datasets/blob/imcyto/inputs/metadata.csv) for an example. +Path to metadata `csv` file indicating which images to merge in full and/or Ilastik stack. The file should only contain 3 columns i.e. 'metal', 'full_stack' and 'ilastik_stack'. The `metal` column should contain all the metals used in your antibody panel. The `full_stack` and `ilastik_stack` entries should be `1` or `0` to indicate whether to include or exclude a metal for a given stack, respectively. See [`metadata.csv`](https://github.com/nf-core/test-datasets/blob/imcyto/inputs/metadata.csv) for an example. ```bash --metadata 'metadata.csv' @@ -198,7 +201,7 @@ Wherever process-specific requirements are set in the pipeline, the default valu If you are likely to be running `nf-core` pipelines regularly it may be a good idea to request that your custom config file is uploaded to the `nf-core/configs` git repository. Before you do this please can you test that the config file works with your pipeline of choice using the `-c` parameter (see definition below). You can then create a pull request to the `nf-core/configs` repository with the addition of your config file, associated documentation file (see examples in [`nf-core/configs/docs`](https://github.com/nf-core/configs/tree/master/docs)), and amending [`nfcore_custom.config`](https://github.com/nf-core/configs/blob/master/nfcore_custom.config) to include your custom profile. -If you have any questions or issues please send us a message on [Slack](https://nf-co.re/join/slack/). +If you have any questions or issues please send us a message on [Slack](https://nf-co.re/join/slack). ## AWS Batch specific parameters @@ -210,7 +213,7 @@ The JobQueue that you intend to use on AWS Batch. ### `--awsregion` -The AWS region to run your job in. Default is set to `eu-west-1` but can be adjusted to your needs. +The AWS region in which to run your job. Default is set to `eu-west-1` but can be adjusted to your needs. ### `--awscli` @@ -224,6 +227,10 @@ Please make sure to also set the `-w/--work-dir` and `--outdir` parameters to a The output directory where the results will be saved. +### `--publish_dir_mode` + +Value passed to Nextflow [`publishDir`](https://www.nextflow.io/docs/latest/process.html#publishdir) directive for publishing results in the output directory. Available: 'symlink', 'rellink', 'link', 'copy', 'copyNoFollow' and 'move' (Default: 'copy'). + ### `--email` Set this parameter to your e-mail address to get a summary e-mail with details of the run sent to you when the workflow exits. If set in your user config file (`~/.nextflow/config`) then you don't need to specify this on the command line for every run. @@ -258,7 +265,7 @@ Note - you can use this to override pipeline defaults. ### `--custom_config_version` -Provide git commit id for custom Institutional configs hosted at `nf-core/configs`. This was implemented for reproducibility purposes. Default is set to `master`. +Provide git commit id for custom Institutional configs hosted at `nf-core/configs`. This was implemented for reproducibility purposes. Default: `master`. ```bash ## Download and use config file with following git commid id diff --git a/main.nf b/main.nf index 0ec36a1..d1e4c93 100644 --- a/main.nf +++ b/main.nf @@ -41,6 +41,7 @@ def helpMessage() { --skip_ilastik [bool] Skip Ilastik processing step --plugins [file] Path to directory with plugin files required for CellProfiler. Default: assets/plugins --outdir [file] The output directory where the results will be saved + --publish_dir_mode [str] Mode for publishing results in the output directory. Available: symlink, rellink, link, copy, copyNoFollow, move (Default: copy) --email [email] Set this parameter to your e-mail address to get a summary e-mail with details of the run sent to you when the workflow exits --email_on_fail [email] Same as --email, except only send mail if the workflow is not successful -name [str] Name for the pipeline run. If not specified, Nextflow will automatically generate a random mnemonic. @@ -59,7 +60,7 @@ if (params.help) { } // Has the run name been specified by the user? -// this has the bonus effect of catching both -name and --name +// this has the bonus effect of catching both -name and --name custom_runName = params.name if (!(workflow.runName ==~ /[a-z]+_[a-z]+/)) { custom_runName = workflow.runName @@ -67,6 +68,7 @@ if (!(workflow.runName ==~ /[a-z]+_[a-z]+/)) { // Stage config files ch_output_docs = file("$baseDir/docs/output.md", checkIfExists: true) +ch_output_docs_images = file("$baseDir/docs/images/", checkIfExists: true) /* * Validate inputs @@ -160,27 +162,27 @@ log.info "-\033[2m--------------------------------------------------\033[0m-" checkHostname() /* - * STEP 1 - IMCTOOLS + * STEP 1: imctools */ -process IMCTools { +process IMCTOOLS { tag "$name" label 'process_medium' - publishDir "${params.outdir}/imctools/${name}", mode: 'copy', + publishDir "${params.outdir}/imctools/${name}", mode: params.publish_dir_mode, saveAs: { filename -> if (filename.indexOf("version.txt") > 0) null else filename } input: - set val(name), file(mcd) from ch_mcd - file metadata from ch_metadata + tuple val(name), path(mcd) from ch_mcd + path metadata from ch_metadata output: - set val(name), file("*/full_stack/*") into ch_full_stack_tiff - set val(name), file("*/ilastik_stack/*") into ch_ilastik_stack_tiff - file "*/*ome.tiff" - file "*.csv" - file "*version.txt" into ch_imctools_version + tuple val(name), path("*/full_stack/*") into ch_full_stack_tiff + tuple val(name), path("*/ilastik_stack/*") into ch_ilastik_stack_tiff + path "*/*ome.tiff" + path "*.csv" + path "*version.txt" into ch_imctools_version script: // This script is bundled with the pipeline, in nf-core/imcyto/bin/ """ @@ -223,26 +225,26 @@ ch_ilastik_stack_tiff .set { ch_ilastik_stack_tiff } /* -* STEP 2 - PREPROCESS FULL STACK IMAGES WITH CELLPROFILER -*/ -process PreprocessFullStack { + * STEP 2: Preprocess full stack images with CellProfiler + */ +process PREPROCESS_FULL_STACK { tag "${name}.${roi}" label 'process_medium' - publishDir "${params.outdir}/preprocess/${name}/${roi}", mode: 'copy', + publishDir "${params.outdir}/preprocess/${name}/${roi}", mode: params.publish_dir_mode, saveAs: { filename -> if (filename.indexOf("version.txt") > 0) null else filename } input: - set val(name), val(roi), file(tiff) from ch_full_stack_tiff - file ctiff from ch_compensation_full_stack.collect().ifEmpty([]) - file cppipe from ch_full_stack_cppipe - file plugin_dir from ch_preprocess_full_stack_plugin.collect() + tuple val(name), val(roi), path(tiff) from ch_full_stack_tiff + path ctiff from ch_compensation_full_stack.collect().ifEmpty([]) + path cppipe from ch_full_stack_cppipe + path plugin_dir from ch_preprocess_full_stack_plugin.collect() output: - set val(name), val(roi), file("full_stack/*") into ch_preprocess_full_stack_tiff - file "*version.txt" into ch_cellprofiler_version + tuple val(name), val(roi), path("full_stack/*") into ch_preprocess_full_stack_tiff + path "*version.txt" into ch_cellprofiler_version script: """ @@ -261,21 +263,21 @@ process PreprocessFullStack { } /* -* STEP 3 - PREPROCESS ILASTIK STACK IMAGES WITH CELLPROFILER -*/ -process PreprocessIlastikStack { + * STEP 3: Preprocess Ilastik stack images with CellProfiler + */ +process PREPROCESS_ILASTIK_STACK { tag "${name}.${roi}" label 'process_medium' - publishDir "${params.outdir}/preprocess/${name}/${roi}", mode: 'copy' + publishDir "${params.outdir}/preprocess/${name}/${roi}", mode: params.publish_dir_mode input: - set val(name), val(roi), file(tiff) from ch_ilastik_stack_tiff - file ctiff from ch_compensation_ilastik_stack.collect().ifEmpty([]) - file cppipe from ch_ilastik_stack_cppipe - file plugin_dir from ch_preprocess_ilastik_stack_plugin.collect() + tuple val(name), val(roi), path(tiff) from ch_ilastik_stack_tiff + path ctiff from ch_compensation_ilastik_stack.collect().ifEmpty([]) + path cppipe from ch_ilastik_stack_cppipe + path plugin_dir from ch_preprocess_ilastik_stack_plugin.collect() output: - set val(name), val(roi), file("ilastik_stack/*") into ch_preprocess_ilastik_stack_tiff + tuple val(name), val(roi), path("ilastik_stack/*") into ch_preprocess_ilastik_stack_tiff script: """ @@ -292,7 +294,7 @@ process PreprocessIlastikStack { } /* - * STEP 4 - ILASTIK + * STEP 4: Ilastik */ if (params.skip_ilastik) { ch_preprocess_full_stack_tiff @@ -301,22 +303,22 @@ if (params.skip_ilastik) { .set { ch_preprocess_full_stack_tiff } ch_ilastik_version = Channel.empty() } else { - process Ilastik { + process ILASTIK { tag "${name}.${roi}" label 'process_medium' - publishDir "${params.outdir}/ilastik/${name}/${roi}", mode: 'copy', + publishDir "${params.outdir}/ilastik/${name}/${roi}", mode: params.publish_dir_mode, saveAs: { filename -> if (filename.indexOf("version.txt") > 0) null else filename } input: - set val(name), val(roi), file(tiff) from ch_preprocess_ilastik_stack_tiff - file ilastik_training_ilp from ch_ilastik_training_ilp + tuple val(name), val(roi), path(tiff) from ch_preprocess_ilastik_stack_tiff + path ilastik_training_ilp from ch_ilastik_training_ilp output: - set val(name), val(roi), file("*.tiff") into ch_ilastik_tiff - file "*version.txt" into ch_ilastik_version + tuple val(name), val(roi), path("*.tiff") into ch_ilastik_tiff + path "*version.txt" into ch_ilastik_version script: """ @@ -342,21 +344,20 @@ if (params.skip_ilastik) { } /* - * STEP 5 - SEGMENTATION WITH CELLPROFILER + * STEP 5: Segmentation with CellProfiler */ -process Segmentation { +process SEGMENTATION { tag "${name}.${roi}" label 'process_high' - publishDir "${params.outdir}/segmentation/${name}/${roi}", mode: 'copy' + publishDir "${params.outdir}/segmentation/${name}/${roi}", mode: params.publish_dir_mode input: - set val(name), val(roi), file(tiff) from ch_preprocess_full_stack_tiff - file cppipe from ch_segmentation_cppipe - file plugin_dir from ch_segmentation_plugin.collect() + tuple val(name), val(roi), path(tiff) from ch_preprocess_full_stack_tiff + path cppipe from ch_segmentation_cppipe + path plugin_dir from ch_segmentation_plugin.collect() output: - set val(name), val(roi), file("*.csv") - set val(name), val(roi), file("*.tiff") + path "*.{csv,tiff}" script: """ @@ -373,16 +374,17 @@ process Segmentation { } /* - * STEP 6 - Output Description HTML + * STEP 6: Output Description HTML */ process output_documentation { - publishDir "${params.outdir}/pipeline_info", mode: 'copy' + publishDir "${params.outdir}/pipeline_info", mode: params.publish_dir_mode input: - file output_docs from ch_output_docs + path output_docs from ch_output_docs + path images from ch_output_docs_images output: - file "results_description.html" + path "results_description.html" script: """ @@ -394,19 +396,19 @@ process output_documentation { * Parse software version numbers */ process get_software_versions { - publishDir "${params.outdir}/pipeline_info", mode: 'copy', + publishDir "${params.outdir}/pipeline_info", mode: params.publish_dir_mode, saveAs: { filename -> if (filename.indexOf(".csv") > 0) filename else null } input: - file imctools from ch_imctools_version.first() - file cellprofiler from ch_cellprofiler_version.first() - file ilastik from ch_ilastik_version.first().ifEmpty([]) + path imctools from ch_imctools_version.first() + path cellprofiler from ch_cellprofiler_version.first() + path ilastik from ch_ilastik_version.first().ifEmpty([]) output: - file "software_versions.csv" + path "software_versions.csv" script: """ @@ -424,7 +426,7 @@ workflow.onComplete { // Set up the e-mail variables def subject = "[nf-core/imcyto] Successful: $workflow.runName" if (!workflow.success) { - subject = "[nf-core/imcyto] FAILED: $workflow.runName" + subject = "[nf-core/imcyto] FAILED: $workflow.runName" } def email_fields = [:] email_fields['version'] = workflow.manifest.version @@ -445,7 +447,6 @@ workflow.onComplete { if (workflow.repository) email_fields['summary']['Pipeline repository Git URL'] = workflow.repository if (workflow.commitId) email_fields['summary']['Pipeline repository Git Commit'] = workflow.commitId if (workflow.revision) email_fields['summary']['Pipeline Git branch/tag'] = workflow.revision - if (workflow.container) email_fields['summary']['Docker image'] = workflow.container email_fields['summary']['Nextflow Version'] = workflow.nextflow.version email_fields['summary']['Nextflow Build'] = workflow.nextflow.build email_fields['summary']['Nextflow Compile Timestamp'] = workflow.nextflow.timestamp @@ -456,6 +457,12 @@ workflow.onComplete { email_address = params.email_on_fail } + // Check if we are only sending emails on failure + email_address = params.email + if (!params.email && params.email_on_fail && !workflow.success) { + email_address = params.email_on_fail + } + // Render the TXT template def engine = new groovy.text.GStringTemplateEngine() def tf = new File("$baseDir/assets/email_template.txt") @@ -476,58 +483,58 @@ workflow.onComplete { // Send the HTML e-mail if (email_address) { try { - if (params.plaintext_email) { throw GroovyException('Send plaintext e-mail, not HTML') } - // Try to send HTML e-mail using sendmail - [ 'sendmail', '-t' ].execute() << sendmail_html - log.info "[nf-core/imcyto] Sent summary e-mail to $email_address (sendmail)" + if (params.plaintext_email) { throw GroovyException('Send plaintext e-mail, not HTML') } + // Try to send HTML e-mail using sendmail + [ 'sendmail', '-t' ].execute() << sendmail_html + log.info "[nf-core/imcyto] Sent summary e-mail to $email_address (sendmail)" } catch (all) { - // Catch failures and try with plaintext - [ 'mail', '-s', subject, email_address ].execute() << email_txt - log.info "[nf-core/imcyto] Sent summary e-mail to $email_address (mail)" + // Catch failures and try with plaintext + [ 'mail', '-s', subject, email_address ].execute() << email_txt + log.info "[nf-core/imcyto] Sent summary e-mail to $email_address (mail)" } } // Write summary e-mail HTML to a file def output_d = new File("${params.outdir}/pipeline_info/") if (!output_d.exists()) { - output_d.mkdirs() + output_d.mkdirs() } def output_hf = new File(output_d, "pipeline_report.html") output_hf.withWriter { w -> w << email_html } def output_tf = new File(output_d, "pipeline_report.txt") output_tf.withWriter { w -> w << email_txt } - c_reset = params.monochrome_logs ? '' : "\033[0m"; - c_purple = params.monochrome_logs ? '' : "\033[0;35m"; c_green = params.monochrome_logs ? '' : "\033[0;32m"; + c_purple = params.monochrome_logs ? '' : "\033[0;35m"; c_red = params.monochrome_logs ? '' : "\033[0;31m"; + c_reset = params.monochrome_logs ? '' : "\033[0m"; if (workflow.stats.ignoredCount > 0 && workflow.success) { - log.info "${c_purple}Warning, pipeline completed, but with errored process(es) ${c_reset}" - log.info "${c_red}Number of ignored errored process(es) : ${workflow.stats.ignoredCount} ${c_reset}" - log.info "${c_green}Number of successfully ran process(es) : ${workflow.stats.succeedCount} ${c_reset}" + log.info "-${c_purple}Warning, pipeline completed, but with errored process(es) ${c_reset}-" + log.info "-${c_red}Number of ignored errored process(es) : ${workflow.stats.ignoredCount} ${c_reset}-" + log.info "-${c_green}Number of successfully ran process(es) : ${workflow.stats.succeedCount} ${c_reset}-" } if (workflow.success) { - log.info "${c_purple}[nf-core/imcyto]${c_green} Pipeline completed successfully${c_reset}" + log.info "-${c_purple}[nf-core/imcyto]${c_green} Pipeline completed successfully${c_reset}-" } else { checkHostname() - log.info "${c_purple}[nf-core/imcyto]${c_red} Pipeline completed with errors${c_reset}" + log.info "-${c_purple}[nf-core/imcyto]${c_red} Pipeline completed with errors${c_reset}-" } } def nfcoreHeader() { // Log colors ANSI codes - c_reset = params.monochrome_logs ? '' : "\033[0m"; - c_dim = params.monochrome_logs ? '' : "\033[2m"; c_black = params.monochrome_logs ? '' : "\033[0;30m"; - c_green = params.monochrome_logs ? '' : "\033[0;32m"; - c_yellow = params.monochrome_logs ? '' : "\033[0;33m"; c_blue = params.monochrome_logs ? '' : "\033[0;34m"; - c_purple = params.monochrome_logs ? '' : "\033[0;35m"; c_cyan = params.monochrome_logs ? '' : "\033[0;36m"; + c_dim = params.monochrome_logs ? '' : "\033[2m"; + c_green = params.monochrome_logs ? '' : "\033[0;32m"; + c_purple = params.monochrome_logs ? '' : "\033[0;35m"; + c_reset = params.monochrome_logs ? '' : "\033[0m"; c_white = params.monochrome_logs ? '' : "\033[0;37m"; + c_yellow = params.monochrome_logs ? '' : "\033[0;33m"; return """ -${c_dim}--------------------------------------------------${c_reset}- ${c_green},--.${c_black}/${c_green},-.${c_reset} diff --git a/nextflow.config b/nextflow.config index 8d897cc..d295040 100644 --- a/nextflow.config +++ b/nextflow.config @@ -21,6 +21,7 @@ params { // Boilerplate options outdir = './results' + publish_dir_mode = 'copy' name = false email = false email_on_fail = false @@ -67,9 +68,10 @@ profiles { singularity.enabled = true singularity.autoMounts = true } - test { includeConfig 'conf/test.config' } - test_txt { includeConfig 'conf/test_txt.config' } - test_tiff { includeConfig 'conf/test_tiff.config' } + test { includeConfig 'conf/test.config' } + test_txt { includeConfig 'conf/test_txt.config' } + test_tiff { includeConfig 'conf/test_tiff.config' } + test_skip_ilastik { includeConfig 'conf/test_skip_ilastik.config' } } // Export this variable to prevent local Python libraries from conflicting with those in the container @@ -104,7 +106,7 @@ manifest { description = 'Image Mass Cytometry analysis pipeline.' mainScript = 'main.nf' nextflowVersion = '>=19.10.0' - version = '1.0dev' + version = '1.0.0' } // Function to ensure that resource requirements don't go beyond