Add paired-end test data, config profile, and CI matrix testing

.github/workflows/ci.yml

@@ -20,21 +20,42 @@ jobs:
       matrix:
         # Test both aligners and multiple Nextflow versions
         include:
-          # Test pipeline minimum Nextflow version with BWA
+          # Test pipeline minimum Nextflow version with BWA (single-end)
           - NXF_VER: "21.10.3"
             NXF_EDGE: ""
             aligner: "bwa"
+            test_profile: "test"
             test_type: "single-end"
-          # Test pipeline minimum Nextflow version with Kallisto
+          # Test pipeline minimum Nextflow version with Kallisto (single-end)
           - NXF_VER: "21.10.3"
             NXF_EDGE: ""
             aligner: "kallisto"
+            test_profile: "test"
             test_type: "single-end"
-          # Test with latest Nextflow version
+          # Test pipeline minimum Nextflow version with BWA (paired-end)
+          - NXF_VER: "21.10.3"
+            NXF_EDGE: ""
+            aligner: "bwa"
+            test_profile: "test_pe"
+            test_type: "paired-end"
+          # Test pipeline minimum Nextflow version with Kallisto (paired-end)
+          - NXF_VER: "21.10.3"
+            NXF_EDGE: ""
+            aligner: "kallisto"
+            test_profile: "test_pe"
+            test_type: "paired-end"
+          # Test with latest Nextflow version (single-end)
           - NXF_VER: "25.04.0"
             NXF_EDGE: ""
             aligner: "bwa"
+            test_profile: "test"
             test_type: "single-end"
+          # Test with latest Nextflow version (paired-end)
+          - NXF_VER: "25.04.0"
+            NXF_EDGE: ""
+            aligner: "bwa"
+            test_profile: "test_pe"
+            test_type: "paired-end"
     steps:
       - name: Check out pipeline code
         uses: actions/checkout@v4
@@ -72,19 +93,19 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
 
-      - name: Run pipeline with test data (${{ matrix.aligner }})
+      - name: Run pipeline with test data (${{ matrix.aligner }} - ${{ matrix.test_type }})
         run: |
           if [ "${{ matrix.aligner }}" == "kallisto" ]; then
-            nextflow run ${GITHUB_WORKSPACE} -profile test,docker --aligner ${{ matrix.aligner }} --ref_genome https://raw.githubusercontent.com/adamd3/BactSeq/develop/test_data/Mabs_CDS.fasta --outdir ./results
+            nextflow run ${GITHUB_WORKSPACE} -profile ${{ matrix.test_profile }},docker --aligner ${{ matrix.aligner }} --ref_genome https://raw.githubusercontent.com/adamd3/BactSeq/develop/test_data/Mabs_CDS.fasta --outdir ./results
           else
-            nextflow run ${GITHUB_WORKSPACE} -profile test,docker --aligner ${{ matrix.aligner }} --ref_genome https://raw.githubusercontent.com/adamd3/BactSeq/develop/test_data/Mabs.fasta --outdir ./results
+            nextflow run ${GITHUB_WORKSPACE} -profile ${{ matrix.test_profile }},docker --aligner ${{ matrix.aligner }} --ref_genome https://raw.githubusercontent.com/adamd3/BactSeq/develop/test_data/Mabs.fasta --outdir ./results
           fi
 
       - name: Upload test results
         if: always()
         uses: actions/upload-artifact@v4
         with:
-          name: test-results-${{ matrix.aligner }}-nf${{ matrix.NXF_VER }}
+          name: test-results-${{ matrix.aligner }}-${{ matrix.test_type }}-nf${{ matrix.NXF_VER }}
           path: |
             ./results
             .nextflow.log

bin/count_reads.R

@@ -39,7 +39,6 @@ opt <- parse_args(opt_parser)
 
 meta_f <- opt$metadata
 gff_f <- opt$gff
-ispaired <- if (opt$is_paired == "TRUE") TRUE else FALSE
 strandedness <- opt$strandedness
 threads <- opt$threads
 
@@ -76,6 +75,14 @@ parse_gff_attributes <- function(
 ## -----------------------------------------------------------------------------
 meta_tab <- read_tsv(meta_f)
 
+if (!is.null(opt$is_paired)) {
+    ispaired <- if (opt$is_paired == "TRUE") TRUE else FALSE
+} else if ("paired" %in% colnames(meta_tab)) {
+    ispaired <- any(meta_tab$paired %in% c("1", 1, "TRUE", "true", TRUE))
+} else {
+    ispaired <- FALSE
+}
+
 merged_total_counts <- meta_tab$sample %>%
     set_names() %>%
     paste0(".counts") %>%

bin/make_meta_file.py

@@ -43,15 +43,19 @@ def make_meta(sample_file, data_dir, outf):
         print(f"ERROR: Could not read sample file: {e}", file=sys.stdout)
         sys.exit(1)
 
-    sample_dat = pd.read_csv(sample_file, sep="\t")
-
     # Remove empty rows
     sample_dat.replace("", float("NaN"), inplace=True)
     sample_dat.dropna(subset=["sample"], inplace=True)
 
-    if len(sample_dat.file2.value_counts()) > 0:
-        f2_bnames = [os.path.basename(f) for f in sample_dat["file2"].to_list()]
-        f2_full_paths = [join_path_or_url(data_dir, b) for b in f2_bnames]
+    if "file2" in sample_dat.columns and len(sample_dat["file2"].dropna()) > 0:
+        f2_bnames = [
+            os.path.basename(f) if pd.notna(f) else ""
+            for f in sample_dat["file2"].to_list()
+        ]
+        f2_full_paths = [
+            join_path_or_url(data_dir, b) if b != "" else ""
+            for b in f2_bnames
+        ]
         sample_dat["file2"] = f2_full_paths
         sample_dat["paired"] = "1"
     else:

configs/test_pe.config

@@ -0,0 +1,26 @@
+/*
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    Config file for testing (paired-end)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    Defines input files and everything required to run a fast and simple
+    paired-end pipeline test.
+--------------------------------------------------------------------------------
+*/
+
+params {
+    config_profile_name        = "Paired-end test profile"
+    config_profile_description = "Paired-end test dataset to check pipeline function"
+
+    // Limit resources so that this can run on GitHub Actions
+    max_cpus   = 2
+    max_memory = "6.GB"
+    max_time   = "6.h"
+
+    // Input data
+    data_dir    = "https://raw.githubusercontent.com/adamd3/BactSeq/develop/test_data/"
+    ref_genome  = "https://raw.githubusercontent.com/adamd3/BactSeq/develop/test_data/Mabs.fasta"
+    ref_ann     = "https://raw.githubusercontent.com/adamd3/BactSeq/develop/test_data/Mabs.gff3"
+    sample_file = "https://raw.githubusercontent.com/adamd3/BactSeq/develop/test_data/sample_sheet_2.tsv"
+    contrast_file = "https://raw.githubusercontent.com/adamd3/BactSeq/develop/test_data/contrast_table.tsv"
+    func_file   = "https://raw.githubusercontent.com/adamd3/BactSeq/develop/test_data/functional_annotation.csv"
+}

main.nf

@@ -201,7 +201,6 @@ workflow {
             ch_bwa_out_count,
             ch_metadata,
             ch_gff_file,
-            params.paired,
             params.strandedness
         )
         ch_readcounts_df = COUNT_READS.out.counts_df
@@ -327,7 +326,6 @@ def helpMessage() {
     --func_file [file]              Path to GFF3-format file containing functional annotations.
     --l2fc_thresh [str]             Absolute log2(FoldChange) threshold for identifying differentially expressed genes. Default = 1.
     --outdir [file]                 The output directory where the results will be saved (Default: './results').
-    --paired [str]                  Data are paired-end.
     --p_thresh [str]                Adjusted p-value threshold for identifying differentially expressed genes. Default = 0.05.
     --skip_trimming [bool]          Do not trim adaptors from FastQ files.
     --strandedness [str]            Is data stranded? Options: `unstranded`, `forward`, `reverse`. Default = reverse.

modules/bwa_align.nf

@@ -66,7 +66,6 @@ process COUNT_READS {
     path counts
     path meta
     path gff
-    val paired
     val strandedness
 
     output:
@@ -78,14 +77,7 @@ process COUNT_READS {
     path 'library_composition_proportions.png', emit: libcomp_plot_prop, optional: true
 
     script:
-
-    if (paired) {
-        """
-        count_reads.R -p TRUE -s $strandedness -m $meta -g $gff -t ${task.cpus}
-        """
-    } else {
-        """
-        count_reads.R -p FALSE -s $strandedness -m $meta -g $gff -t ${task.cpus}
-        """
-    }
+    """
+    count_reads.R -s $strandedness -m $meta -g $gff -t ${task.cpus}
+    """
 }

nextflow.config

@@ -9,7 +9,6 @@ params {
 
     // Read aligning + counting
     aligner                    = 'bwa'
-    paired                     = false
     strandedness               = 'reverse'
     fragment_len               = 150 
     fragment_sd                = 20
@@ -80,6 +79,9 @@ profiles {
     test { 
         includeConfig 'configs/test.config'
     }
+    test_pe {
+        includeConfig 'configs/test_pe.config'
+    }
 }
 
 // Export these variables to prevent local Python/R libraries from 

test_data/sample_sheet_2.tsv

@@ -0,0 +1,5 @@
+sample	file1	file2	group	rep_no
+1D3	1D3_S9_R1_001.fastq.gz	1D3_S9_R2_001.fastq.gz	DMSO_6hr	1
+2C3	2C3_S16_R1_001.fastq.gz	2C3_S16_R2_001.fastq.gz	CLARI_6hr	1
+3C3	3C3_S25_R1_001.fastq.gz	3C3_S25_R2_001.fastq.gz	CLARI_6hr	2
+3D3	3D3_S27_R1_001.fastq.gz	3D3_S27_R2_001.fastq.gz	DMSO_6hr	2