Mercurial > repos > devteam > fastq_filter

changeset 0:0017fa63af6c draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Imported from capsule None
author devteam
date Thu, 23 Jan 2014 12:31:42 -0500
parents
children a30fc80df16c
files fastq_filter.py fastq_filter.xml test-data/empty_file.dat test-data/sanger_full_range_as_cssanger.fastqcssanger test-data/sanger_full_range_original_sanger.fastqsanger test-data/solexa_full_range_original_solexa.fastqsolexa tool_dependencies.xml
diffstat 6 files changed, 386 insertions(+), 0 deletions(-) [+]
line wrap: on
line diff
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/fastq_filter.py	Thu Jan 23 12:31:42 2014 -0500
@@ -0,0 +1,36 @@
+#Dan Blankenberg
+import sys, os, shutil
+from galaxy_utils.sequence.fastq import fastqReader, fastqWriter
+
+def main():
+    #Read command line arguments
+    input_filename = sys.argv[1]
+    script_filename = sys.argv[2]
+    output_filename = sys.argv[3]
+    additional_files_path = sys.argv[4]
+    input_type = sys.argv[5] or 'sanger'
+    
+    #Save script file for debuging/verification info later
+    os.mkdir( additional_files_path )
+    shutil.copy( script_filename, os.path.join( additional_files_path, 'debug.txt' ) )
+    
+    ## Dan, Others: Can we simply drop the "format=input_type" here since it is specified in reader.
+    ## This optimization would cut runtime roughly in half (for my test case anyway). -John
+    out = fastqWriter( open( output_filename, 'wb' ), format = input_type )
+    
+    i = None
+    reads_kept = 0
+    execfile(script_filename, globals())
+    for i, fastq_read in enumerate( fastqReader( open( input_filename ), format = input_type ) ):
+        ret_val = fastq_read_pass_filter( fastq_read )  ## fastq_read_pass_filter defined in script_filename
+        if ret_val:
+            out.write( fastq_read )
+            reads_kept += 1
+    out.close()
+    if i is None:
+        print "Your file contains no valid fastq reads."
+    else:
+        print 'Kept %s of %s reads (%.2f%%).' % ( reads_kept, i + 1, float( reads_kept ) / float( i + 1 ) * 100.0 )
+
+if __name__ == "__main__":
+    main()
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/fastq_filter.xml	Thu Jan 23 12:31:42 2014 -0500
@@ -0,0 +1,320 @@
+<tool id="fastq_filter" name="Filter FASTQ" version="1.0.0">
+  <description>reads by quality score and length</description>
+  <requirements>
+    <requirement type="package" version="1.0.0">galaxy_sequence_utils</requirement>
+  </requirements>
+  <command interpreter="python">fastq_filter.py $input_file $fastq_filter_file $output_file $output_file.files_path '${input_file.extension[len( 'fastq' ):]}'</command>
+  <inputs>
+    <page>
+      <param name="input_file" type="data" format="fastqsanger,fastqcssanger" label="FASTQ File" help="Requires groomed data: if your data does not appear here try using the FASTQ groomer."/>
+       <param name="min_size" label="Minimum Size" value="0" type="integer">
+        <validator type="in_range" message="Minimum size must be positive" min="0"/>
+      </param>
+      <param name="max_size" label="Maximum Size" value="0" type="integer" help="A maximum size less than 1 indicates no limit."/>
+      <param name="min_quality" label="Minimum Quality" value="0" type="float"/>
+      <param name="max_quality" label="Maximum Quality" value="0" type="float" help="A maximum quality less than 1 indicates no limit."/>
+      <param name="max_num_deviants" label="Maximum number of bases allowed outside of quality range" value="0" type="integer">
+        <validator type="in_range" message="Maximum number of deviate bases must be positive" min="0"/>
+      </param>
+     <param name="paired_end" label="This is paired end data" type="boolean" truevalue="paired_end" falsevalue="single_end" checked="False"/>
+      <repeat name="fastq_filters" title="Quality Filter on a Range of Bases" help="The above settings do not apply to these filters.">
+        <conditional name="offset_type">
+          <param name="base_offset_type" type="select" label="Define Base Offsets as" help="Use Absolute for fixed length reads (Illumina, SOLiD)&lt;br&gt;Use Percentage for variable length reads (Roche/454)">
+            <option value="offsets_absolute" selected="true">Absolute Values</option>
+            <option value="offsets_percent">Percentage of Read Length</option>
+          </param>
+          <when value="offsets_absolute">
+            <param name="left_column_offset" label="Offset from 5' end" value="0" type="integer" help="Values start at 0, increasing from the left">
+              <validator type="in_range" message="Base Offsets must be positive" min="0" max="inf"/>
+              <validator type="expression" message="An integer is required.">int( float( value ) ) == float( value )</validator>
+            </param>
+            <param name="right_column_offset" label="Offset from 3' end" value="0" type="integer" help="Values start at 0, increasing from the right">
+              <validator type="in_range" message="Base Offsets must be positive" min="0" max="inf"/>
+              <validator type="expression" message="An integer is required.">int( float( value ) ) == float( value )</validator>
+            </param>
+          </when>
+          <when value="offsets_percent">
+            <param name="left_column_offset" label="Offset from 5' end" value="0" type="float">
+              <validator type="in_range" message="Base Offsets must be between 0 and 100" min="0" max="100"/>
+            </param>
+            <param name="right_column_offset" label="Offset from 3' end" value="0" type="float">
+              <validator type="in_range" message="Base Offsets must be between 0 and 100" min="0" max="100"/>
+            </param>
+          </when>
+        </conditional>
+        <param name="score_operation" type="select" label="Aggregate read score for specified range">
+          <option value="min" selected="True">min score</option>
+          <option value="max">max score</option>
+          <option value="sum">sum of scores</option>
+          <option value="mean">mean of scores</option>
+        </param>
+        <param name="score_comparison" type="select" label="Keep read when aggregate score is">
+          <option value="&gt;">&gt;</option>
+          <option value="&gt;=" selected="true">&gt;=</option>
+          <option value="==">==</option>
+          <option value="&lt;">&lt;</option>
+          <option value="&lt;=">&lt;=</option>
+          <sanitizer sanitize="False"/>
+        </param>
+        <param name="score" label="Quality Score" value="0" type="float" />
+      </repeat>
+    </page>
+  </inputs>
+  <configfiles>
+    <configfile name="fastq_filter_file">
+def fastq_read_pass_filter( fastq_read ):
+    def mean( score_list ):
+        return float( sum( score_list ) ) / float( len( score_list ) )
+    if len( fastq_read ) &lt; $min_size:
+        return False
+    if $max_size &gt; 0 and len( fastq_read ) &gt; $max_size:
+        return False
+    num_deviates = $max_num_deviants
+    qual_scores = fastq_read.get_decimal_quality_scores()
+    for qual_score in qual_scores:
+        if qual_score &lt; $min_quality or ( $max_quality &gt; 0 and qual_score &gt; $max_quality ):
+            if num_deviates == 0:
+                return False
+            else:
+                num_deviates -= 1
+#if not $paired_end:
+    qual_scores_split = [ qual_scores ]
+#else:
+    qual_scores_split = [ qual_scores[ 0:int( len( qual_scores ) / 2 ) ], qual_scores[ int( len( qual_scores ) / 2 ): ] ]
+#end if
+#for $fastq_filter in $fastq_filters:
+    for split_scores in qual_scores_split:
+        left_column_offset = $fastq_filter[ 'offset_type' ][ 'left_column_offset' ]
+        right_column_offset = $fastq_filter[ 'offset_type' ][ 'right_column_offset' ]
+#if $fastq_filter[ 'offset_type' ]['base_offset_type'] == 'offsets_percent':
+        left_column_offset = int( round( float( left_column_offset ) / 100.0 * float( len( split_scores ) ) ) )
+        right_column_offset = int( round( float( right_column_offset ) / 100.0 * float( len( split_scores ) ) ) )
+#end if
+        if right_column_offset > 0:
+            split_scores = split_scores[ left_column_offset:-right_column_offset]
+        else:
+            split_scores = split_scores[ left_column_offset:]
+        if split_scores: ##if a read doesn't have enough columns, it passes by default 
+            if not ( ${fastq_filter[ 'score_operation' ]}( split_scores ) $fastq_filter[ 'score_comparison' ] $fastq_filter[ 'score' ]  ):
+                return False
+#end for
+    return True
+</configfile>
+  </configfiles>
+  <outputs>
+    <data format="input" name="output_file" />
+  </outputs>
+  <tests>
+    <!-- Do nothing filter -->
+    <test>
+      <param name="input_file" value="sanger_full_range_original_sanger.fastqsanger" ftype="fastqsanger"/>
+      <param name="min_size" value="0"/>
+      <param name="max_size" value="0"/>
+      <param name="min_quality" value="0"/>
+      <param name="max_quality" value="0"/>
+      <param name="max_num_deviants" value="0"/>
+      <param name="paired_end" value="False"/>
+      <param name="base_offset_type" value="offsets_absolute"/>
+      <param name="left_column_offset" value="0"/>
+      <param name="right_column_offset" value="0"/>
+      <param name="score_operation" value="min"/>
+      <param name="score_comparison" value="&gt;="/>
+      <param name="score" value="0"/>
+      <output name="out_file1" file="sanger_full_range_original_sanger.fastqsanger"/>
+    </test>
+    <!-- crippled input types prevent this test <test>
+      <param name="input_file" value="solexa_full_range_original_solexa.fastqsolexa" ftype="fastqsolexa"/>
+      <param name="min_size" value="0"/>
+      <param name="max_size" value="0"/>
+      <param name="min_quality" value="-5"/>
+      <param name="max_quality" value="0"/>
+      <param name="max_num_deviants" value="0"/>
+      <param name="paired_end" value="False"/>
+      <param name="base_offset_type" value="offsets_absolute"/>
+      <param name="left_column_offset" value="0"/>
+      <param name="right_column_offset" value="0"/>
+      <param name="score_operation" value="min"/>
+      <param name="score_comparison" value="&gt;="/>
+      <param name="score" value="-5"/>
+      <output name="out_file1" file="solexa_full_range_original_solexa.fastqsolexa"/>
+    </test> -->
+    <!-- No trim, so does not remove Adapter from cssanger -->
+    <test>
+      <param name="input_file" value="sanger_full_range_as_cssanger.fastqcssanger" ftype="fastqcssanger"/>
+      <param name="min_size" value="0"/>
+      <param name="max_size" value="0"/>
+      <param name="min_quality" value="0"/>
+      <param name="max_quality" value="0"/>
+      <param name="max_num_deviants" value="0"/>
+      <param name="paired_end" value="False"/>
+      <param name="base_offset_type" value="offsets_absolute"/>
+      <param name="left_column_offset" value="0"/>
+      <param name="right_column_offset" value="0"/>
+      <param name="score_operation" value="min"/>
+      <param name="score_comparison" value="&gt;="/>
+      <param name="score" value="0"/>
+      <output name="out_file1" file="sanger_full_range_as_cssanger.fastqcssanger"/>
+    </test>
+    <!-- Remove all Filter -->
+    <test>
+      <param name="input_file" value="sanger_full_range_original_sanger.fastqsanger" ftype="fastqsanger"/>
+      <param name="min_size" value="0"/>
+      <param name="max_size" value="0"/>
+      <param name="min_quality" value="1"/>
+      <param name="max_quality" value="0"/>
+      <param name="max_num_deviants" value="0"/>
+      <param name="paired_end" value="False"/>
+      <param name="base_offset_type" value="offsets_absolute"/>
+      <param name="left_column_offset" value="0"/>
+      <param name="right_column_offset" value="0"/>
+      <param name="score_operation" value="min"/>
+      <param name="score_comparison" value="&gt;="/>
+      <param name="score" value="0"/>
+      <output name="out_file1" file="empty_file.dat"/>
+    </test>
+    <!-- crippled input types prevent this test <test>
+      <param name="input_file" value="solexa_full_range_original_solexa.fastqsolexa" ftype="fastqsolexa"/>
+      <param name="min_size" value="0"/>
+      <param name="max_size" value="0"/>
+      <param name="min_quality" value="-4"/>
+      <param name="max_quality" value="0"/>
+      <param name="max_num_deviants" value="0"/>
+      <param name="paired_end" value="False"/>
+      <param name="base_offset_type" value="offsets_absolute"/>
+      <param name="left_column_offset" value="0"/>
+      <param name="right_column_offset" value="0"/>
+      <param name="score_operation" value="min"/>
+      <param name="score_comparison" value="&gt;="/>
+      <param name="score" value="-5"/>
+      <output name="out_file1" file="empty_file.dat"/>
+    </test> -->
+    <!-- Keep all by allowing 1 deviant -->
+    <test>
+      <param name="input_file" value="sanger_full_range_original_sanger.fastqsanger" ftype="fastqsanger"/>
+      <param name="min_size" value="0"/>
+      <param name="max_size" value="0"/>
+      <param name="min_quality" value="1"/>
+      <param name="max_quality" value="0"/>
+      <param name="max_num_deviants" value="1"/>
+      <param name="paired_end" value="False"/>
+      <param name="base_offset_type" value="offsets_absolute"/>
+      <param name="left_column_offset" value="0"/>
+      <param name="right_column_offset" value="0"/>
+      <param name="score_operation" value="min"/>
+      <param name="score_comparison" value="&gt;="/>
+      <param name="score" value="0"/>
+      <output name="out_file1" file="sanger_full_range_original_sanger.fastqsanger"/>
+    </test>
+    <!-- crippled input types prevent this test<test>
+      <param name="input_file" value="solexa_full_range_original_solexa.fastqsolexa" ftype="fastqsolexa"/>
+      <param name="min_size" value="0"/>
+      <param name="max_size" value="0"/>
+      <param name="min_quality" value="-5"/>
+      <param name="max_quality" value="61"/>
+      <param name="max_num_deviants" value="1"/>
+      <param name="paired_end" value="False"/>
+      <param name="base_offset_type" value="offsets_absolute"/>
+      <param name="left_column_offset" value="0"/>
+      <param name="right_column_offset" value="0"/>
+      <param name="score_operation" value="min"/>
+      <param name="score_comparison" value="&gt;="/>
+      <param name="score" value="-5"/>
+      <output name="out_file1" file="solexa_full_range_original_solexa.fastqsolexa"/>
+    </test> -->
+    <!-- Filter inner range -->
+    <test>
+      <param name="input_file" value="sanger_full_range_original_sanger.fastqsanger" ftype="fastqsanger"/>
+      <param name="min_size" value="0"/>
+      <param name="max_size" value="0"/>
+      <param name="min_quality" value="0"/>
+      <param name="max_quality" value="0"/>
+      <param name="max_num_deviants" value="0"/>
+      <param name="paired_end" value="False"/>
+      <param name="base_offset_type" value="offsets_absolute"/>
+      <param name="left_column_offset" value="1"/>
+      <param name="right_column_offset" value="1"/>
+      <param name="score_operation" value="min"/>
+      <param name="score_comparison" value="&gt;="/>
+      <param name="score" value="1"/>
+      <output name="out_file1" file="sanger_full_range_original_sanger.fastqsanger"/>
+    </test>
+    <test>
+      <param name="input_file" value="sanger_full_range_original_sanger.fastqsanger" ftype="fastqsanger"/>
+      <param name="min_size" value="0"/>
+      <param name="max_size" value="0"/>
+      <param name="min_quality" value="0"/>
+      <param name="max_quality" value="0"/>
+      <param name="max_num_deviants" value="0"/>
+      <param name="paired_end" value="False"/>
+      <param name="base_offset_type" value="offsets_absolute"/>
+      <param name="left_column_offset" value="1"/>
+      <param name="right_column_offset" value="1"/>
+      <param name="score_operation" value="max"/>
+      <param name="score_comparison" value="&lt;="/>
+      <param name="score" value="92"/>
+      <output name="out_file1" file="sanger_full_range_original_sanger.fastqsanger"/>
+    </test>
+    <!-- percent based offsets -->
+    <test>
+      <param name="input_file" value="sanger_full_range_original_sanger.fastqsanger" ftype="fastqsanger"/>
+      <param name="min_size" value="0"/>
+      <param name="max_size" value="0"/>
+      <param name="min_quality" value="0"/>
+      <param name="max_quality" value="0"/>
+      <param name="max_num_deviants" value="0"/>
+      <param name="paired_end" value="False"/>
+      <param name="base_offset_type" value="offsets_percent"/>
+      <param name="left_column_offset" value="1.075"/>
+      <param name="right_column_offset" value="1.075"/>
+      <param name="score_operation" value="min"/>
+      <param name="score_comparison" value="&gt;="/>
+      <param name="score" value="1"/>
+      <output name="out_file1" file="sanger_full_range_original_sanger.fastqsanger"/>
+    </test>
+    <test>
+      <param name="input_file" value="sanger_full_range_original_sanger.fastqsanger" ftype="fastqsanger"/>
+      <param name="min_size" value="0"/>
+      <param name="max_size" value="0"/>
+      <param name="min_quality" value="0"/>
+      <param name="max_quality" value="0"/>
+      <param name="max_num_deviants" value="0"/>
+      <param name="paired_end" value="True"/>
+      <param name="base_offset_type" value="offsets_percent"/>
+      <param name="left_column_offset" value="1"/>
+      <param name="right_column_offset" value="1"/>
+      <param name="score_operation" value="min"/>
+      <param name="score_comparison" value="&gt;="/>
+      <param name="score" value="1"/>
+      <output name="out_file1" file="empty_file.dat"/>
+    </test>
+  </tests>
+<help>
+This tool allows you to build complex filters to be applied to each read in a FASTQ file.
+
+**Basic Options:**
+    * You can specify a minimum and maximum read lengths.
+    * You can specify minimum and maximum per base quality scores, with optionally specifying the number of bases that are allowed to deviate from this range (default of 0 deviant bases).
+    * If your data is paired-end, select the proper checkbox; this will cause each read to be internally split down the middle and filters applied to each half using the offsets specified.
+
+**Advance Options:**
+    * You can specify any number of advanced filters. 
+    * 5' and 3' offsets are defined, starting at zero, increasing from the respective end of the reads. For example, a quality string of "ABCDEFG", with 5' and 3' offsets of 1 and 1, respectively, specified will yield "BCDEF".
+    * You can specify either absolute offset values, or percentage offset values. *Absolute Values* based offsets are useful for fixed length reads (e.g. Illumina or SOLiD data). *Percentage of Read Length* based offsets are useful for variable length reads (e.g. 454 data). When using the percent-based method, offsets are rounded to the nearest integer.
+    * The user specifies the aggregating action (min, max, sum, mean) to perform on the quality score values found between the specified offsets to be used with the user defined comparison operation and comparison value.
+    * If a set of offsets is specified that causes the remaining quality score list to be of length zero, then the read will **pass** the quality filter unless the size range filter is used to remove these reads.
+
+-----
+
+.. class:: warningmark
+
+Adapter bases in color space reads are excluded from filtering.
+
+------
+
+**Citation**
+
+If you use this tool, please cite `Blankenberg D, Gordon A, Von Kuster G, Coraor N, Taylor J, Nekrutenko A; Galaxy Team. Manipulation of FASTQ data with Galaxy. Bioinformatics. 2010 Jul 15;26(14):1783-5. &lt;http://www.ncbi.nlm.nih.gov/pubmed/20562416&gt;`_
+
+
+</help>
+</tool>
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/test-data/sanger_full_range_as_cssanger.fastqcssanger	Thu Jan 23 12:31:42 2014 -0500
@@ -0,0 +1,8 @@
+@FAKE0001 Original version has PHRED scores from 0 to 93 inclusive (in that order)
+G2131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131
++
+!"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~
+@FAKE0002 Original version has PHRED scores from 93 to 0 inclusive (in that order)
+G3131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131
++
+~}|{zyxwvutsrqponmlkjihgfedcba`_^]\[ZYXWVUTSRQPONMLKJIHGFEDCBA@?>=<;:9876543210/.-,+*)('&%$#"!
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/test-data/sanger_full_range_original_sanger.fastqsanger	Thu Jan 23 12:31:42 2014 -0500
@@ -0,0 +1,8 @@
+@FAKE0001 Original version has PHRED scores from 0 to 93 inclusive (in that order)
+ACGTACGTACGTACGTACGTACGTACGTACGTACGTACGTACGTACGTACGTACGTACGTACGTACGTACGTACGTACGTACGTACGTACGTAC
++
+!"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~
+@FAKE0002 Original version has PHRED scores from 93 to 0 inclusive (in that order)
+CATGCATGCATGCATGCATGCATGCATGCATGCATGCATGCATGCATGCATGCATGCATGCATGCATGCATGCATGCATGCATGCATGCATGCA
++
+~}|{zyxwvutsrqponmlkjihgfedcba`_^]\[ZYXWVUTSRQPONMLKJIHGFEDCBA@?>=<;:9876543210/.-,+*)('&%$#"!
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/test-data/solexa_full_range_original_solexa.fastqsolexa	Thu Jan 23 12:31:42 2014 -0500
@@ -0,0 +1,8 @@
+@FAKE0003 Original version has Solexa scores from -5 to 62 inclusive (in that order)
+ACGTACGTACGTACGTACGTACGTACGTACGTACGTACGTACGTACGTACGTACGTACGTACGTACGT
++
+;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~
+@FAKE0004 Original version has Solexa scores from 62 to -5 inclusive (in that order)
+TGCATGCATGCATGCATGCATGCATGCATGCATGCATGCATGCATGCATGCATGCATGCATGCATGCA
++
+~}|{zyxwvutsrqponmlkjihgfedcba`_^]\[ZYXWVUTSRQPONMLKJIHGFEDCBA@?>=<;
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tool_dependencies.xml	Thu Jan 23 12:31:42 2014 -0500
@@ -0,0 +1,6 @@
+<?xml version="1.0"?>
+<tool_dependency>
+  <package name="galaxy_sequence_utils" version="1.0.0">
+      <repository changeset_revision="195699b1562a" name="package_galaxy_utils_1_0" owner="devteam" prior_installation_required="False" toolshed="http://testtoolshed.g2.bx.psu.edu" />
+    </package>
+</tool_dependency>