Mercurial > repos > peterjc > seq_filter_by_id
changeset 10:b7ec1c9db38d draft
Uploaded v0.0.6g, renamed folder and README file
| author | peterjc |
|---|---|
| date | Thu, 26 Sep 2013 10:05:52 -0400 |
| parents | b1b4ad43caf1 |
| children | 8a34c565a473 |
| files | tools/filters/repository_dependencies.xml tools/filters/seq_filter_by_id.py tools/filters/seq_filter_by_id.rst tools/filters/seq_filter_by_id.xml tools/seq_filter_by_id/README.rst tools/seq_filter_by_id/repository_dependencies.xml tools/seq_filter_by_id/seq_filter_by_id.py tools/seq_filter_by_id/seq_filter_by_id.xml |
| diffstat | 8 files changed, 516 insertions(+), 516 deletions(-) [+] |
line wrap: on
line diff
--- a/tools/filters/repository_dependencies.xml Tue Sep 17 11:55:21 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,6 +0,0 @@ -<?xml version="1.0"?> -<repositories description="This requires Biopython as a dependency."> -<!-- Leave out the tool shed and revision to get the current - tool shed and latest revision at the time of upload --> -<repository changeset_revision="2f6c871cfa35" name="package_biopython_1_61" owner="biopython" toolshed="http://testtoolshed.g2.bx.psu.edu" /> -</repositories>
--- a/tools/filters/seq_filter_by_id.py Tue Sep 17 11:55:21 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,260 +0,0 @@ -#!/usr/bin/env python -"""Filter a FASTA, FASTQ or SSF file with IDs from a tabular file. - -Takes six command line options, tabular filename, ID column numbers (comma -separated list using one based counting), input filename, input type (e.g. -FASTA or SFF) and two output filenames (for records with and without the -given IDs, same format as input sequence file). - -If either output filename is just a minus sign, that file is not created. -This is intended to allow output for just the matched (or just the non-matched) -records. - -When filtering an SFF file, any Roche XML manifest in the input file is -preserved in both output files. - -Note in the default NCBI BLAST+ tabular output, the query sequence ID is -in column one, and the ID of the match from the database is in column two. -Here sensible values for the column numbers would therefore be "1" or "2". - -This tool is a short Python script which requires Biopython 1.54 or later -for SFF file support. If you use this tool in scientific work leading to a -publication, please cite the Biopython application note: - -Cock et al 2009. Biopython: freely available Python tools for computational -molecular biology and bioinformatics. Bioinformatics 25(11) 1422-3. -http://dx.doi.org/10.1093/bioinformatics/btp163 pmid:19304878. - -This script is copyright 2010-2013 by Peter Cock, The James Hutton Institute -(formerly the Scottish Crop Research Institute, SCRI), UK. All rights reserved. -See accompanying text file for licence details (MIT license). - -This is version 0.1.0 of the script, use -v or --version to get the version. -""" -import os -import sys - -def stop_err(msg, err=1): - sys.stderr.write(msg.rstrip() + "\n") - sys.exit(err) - -if "-v" in sys.argv or "--version" in sys.argv: - print "v0.1.0" - sys.exit(0) - -#Parse Command Line -if len(sys.argv) - 1 < 7 or len(sys.argv) % 2 == 1: - stop_err("Expected 7 or more arguments, 5 required " - "(in seq, seq format, out pos, out neg, logic) " - "then one or more pairs (tab file, columns), " - "got %i:\n%s" % (len(sys.argv)-1, " ".join(sys.argv))) - -in_file, seq_format, out_positive_file, out_negative_file, logic = sys.argv[1:6] - -if not os.path.isfile(in_file): - stop_err("Missing input file %r" % in_file) -if out_positive_file == "-" and out_negative_file == "-": - stop_err("Neither output file requested") -if logic not in ["UNION", "INTERSECTION"]: - stop_err("Fifth agrument should be 'UNION' or 'INTERSECTION', not %r" % logic) - -identifiers = [] -for i in range((len(sys.argv) - 6) // 2): - tabular_file = sys.argv[6+2*i] - cols_arg = sys.argv[7+2*i] - if not os.path.isfile(tabular_file): - stop_err("Missing tabular identifier file %r" % tabular_file) - try: - columns = [int(arg)-1 for arg in cols_arg.split(",")] - except ValueError: - stop_err("Expected list of columns (comma separated integers), got %r" % cols_arg) - if min(columns) < 0: - stop_err("Expect one-based column numbers (not zero-based counting), got %r" % cols_arg) - identifiers.append((tabular_file, columns)) - -#Read tabular file(s) and record all specified identifiers -ids = None #Will be a set -for tabular_file, columns in identifiers: - file_ids = set() - handle = open(tabular_file, "rU") - if len(columns)>1: - #General case of many columns - for line in handle: - if line.startswith("#"): - #Ignore comments - continue - parts = line.rstrip("\n").split("\t") - for col in columns: - file_ids.add(parts[col]) - else: - #Single column, special case speed up - col = columns[0] - for line in handle: - if not line.startswith("#"): - file_ids.add(line.rstrip("\n").split("\t")[col]) - print "Using %i IDs from column %s in tabular file" % (len(file_ids), ", ".join(str(col+1) for col in columns)) - if ids is None: - ids = file_ids - if logic == "UNION": - ids.update(file_ids) - else: - ids.intersection_update(file_ids) - handle.close() -if len(identifiers) > 1: - if logic == "UNION": - print "Have %i IDs combined from %i tabular files" % (len(ids), len(identifiers)) - else: - print "Have %i IDs in common from %i tabular files" % (len(ids), len(identifiers)) - - -def crude_fasta_iterator(handle): - """Yields tuples, record ID and the full record as a string.""" - while True: - line = handle.readline() - if line == "": - return # Premature end of file, or just empty? - if line[0] == ">": - break - - no_id_warned = False - while True: - if line[0] != ">": - raise ValueError( - "Records in Fasta files should start with '>' character") - try: - id = line[1:].split(None, 1)[0] - except IndexError: - if not no_id_warned: - sys.stderr.write("WARNING - Malformed FASTA entry with no identifier\n") - no_id_warned = True - id = None - lines = [line] - line = handle.readline() - while True: - if not line: - break - if line[0] == ">": - break - lines.append(line) - line = handle.readline() - yield id, "".join(lines) - if not line: - return # StopIteration - - -def fasta_filter(in_file, pos_file, neg_file, wanted): - """FASTA filter producing 60 character line wrapped outout.""" - pos_count = neg_count = 0 - #Galaxy now requires Python 2.5+ so can use with statements, - with open(in_file) as in_handle: - #Doing the if statement outside the loop for speed - #(with the downside of three very similar loops). - if pos_file != "-" and neg_file != "-": - print "Generating two FASTA files" - with open(pos_file, "w") as pos_handle: - with open(neg_file, "w") as neg_handle: - for identifier, record in crude_fasta_iterator(in_handle): - if identifier in wanted: - pos_handle.write(record) - pos_count += 1 - else: - neg_handle.write(record) - neg_count += 1 - elif pos_file != "-": - print "Generating matching FASTA file" - with open(pos_file, "w") as pos_handle: - for identifier, record in crude_fasta_iterator(in_handle): - if identifier in wanted: - pos_handle.write(record) - pos_count += 1 - else: - neg_count += 1 - else: - print "Generating non-matching FASTA file" - assert neg_file != "-" - with open(neg_file, "w") as neg_handle: - for identifier, record in crude_fasta_iterator(in_handle): - if identifier in wanted: - pos_count += 1 - else: - neg_handle.write(record) - neg_count += 1 - return pos_count, neg_count - - -if seq_format.lower()=="sff": - #Now write filtered SFF file based on IDs from BLAST file - try: - from Bio.SeqIO.SffIO import SffIterator, SffWriter - except ImportError: - stop_err("SFF filtering requires Biopython 1.54 or later") - - try: - from Bio.SeqIO.SffIO import ReadRocheXmlManifest - except ImportError: - #Prior to Biopython 1.56 this was a private function - from Bio.SeqIO.SffIO import _sff_read_roche_index_xml as ReadRocheXmlManifest - in_handle = open(in_file, "rb") #must be binary mode! - try: - manifest = ReadRocheXmlManifest(in_handle) - except ValueError: - manifest = None - #This makes two passes though the SFF file with isn't so efficient, - #but this makes the code simple. - pos_count = neg_count = 0 - if out_positive_file != "-": - out_handle = open(out_positive_file, "wb") - writer = SffWriter(out_handle, xml=manifest) - in_handle.seek(0) #start again after getting manifest - pos_count = writer.write_file(rec for rec in SffIterator(in_handle) if rec.id in ids) - out_handle.close() - if out_negative_file != "-": - out_handle = open(out_negative_file, "wb") - writer = SffWriter(out_handle, xml=manifest) - in_handle.seek(0) #start again - neg_count = writer.write_file(rec for rec in SffIterator(in_handle) if rec.id not in ids) - out_handle.close() - #And we're done - in_handle.close() - #At the time of writing, Galaxy doesn't show SFF file read counts, - #so it is useful to put them in stdout and thus shown in job info. - print "%i with and %i without specified IDs" % (pos_count, neg_count) -elif seq_format.lower()=="fasta": - #Write filtered FASTA file based on IDs from tabular file - pos_count, neg_count = fasta_filter(in_file, out_positive_file, out_negative_file, ids) - print "%i with and %i without specified IDs" % (pos_count, neg_count) -elif seq_format.lower().startswith("fastq"): - #Write filtered FASTQ file based on IDs from tabular file - from galaxy_utils.sequence.fastq import fastqReader, fastqWriter - reader = fastqReader(open(in_file, "rU")) - if out_positive_file != "-" and out_negative_file != "-": - print "Generating two FASTQ files" - positive_writer = fastqWriter(open(out_positive_file, "w")) - negative_writer = fastqWriter(open(out_negative_file, "w")) - for record in reader: - #The [1:] is because the fastaReader leaves the > on the identifier. - if record.identifier and record.identifier.split()[0][1:] in ids: - positive_writer.write(record) - else: - negative_writer.write(record) - positive_writer.close() - negative_writer.close() - elif out_positive_file != "-": - print "Generating matching FASTQ file" - positive_writer = fastqWriter(open(out_positive_file, "w")) - for record in reader: - #The [1:] is because the fastaReader leaves the > on the identifier. - if record.identifier and record.identifier.split()[0][1:] in ids: - positive_writer.write(record) - positive_writer.close() - elif out_negative_file != "-": - print "Generating non-matching FASTQ file" - negative_writer = fastqWriter(open(out_negative_file, "w")) - for record in reader: - #The [1:] is because the fastaReader leaves the > on the identifier. - if not record.identifier or record.identifier.split()[0][1:] not in ids: - negative_writer.write(record) - negative_writer.close() - reader.close() -else: - stop_err("Unsupported file type %r" % seq_format)
--- a/tools/filters/seq_filter_by_id.rst Tue Sep 17 11:55:21 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,124 +0,0 @@ -Galaxy tool to filter FASTA, FASTQ or SFF sequences by ID -========================================================= - -This tool is copyright 2010-2013 by Peter Cock, The James Hutton Institute -(formerly SCRI, Scottish Crop Research Institute), UK. All rights reserved. -See the licence text below. - -This tool is a short Python script (using both the Galaxy and Biopython library -functions) which divides a FASTA, FASTQ, or SFF file in two, those sequences with -or without an ID present in the specified column(s) of a tabular file. Example uses -include filtering based on search results from a tool like NCBI BLAST before -assembly. - -This tool is available from the Galaxy Tool Shed at: - -* http://toolshed.g2.bx.psu.edu/view/peterjc/seq_filter_by_id - -See also sister tools: - -* http://toolshed.g2.bx.psu.edu/view/peterjc/seq_select_by_id -* http://toolshed.g2.bx.psu.edu/view/peterjc/seq_rename - - -Automated Installation -====================== - -This should be straightforward using the Galaxy Tool Shed, which should be -able to automatically install the dependency on Biopython, and then install -this tool and run its unit tests. - - -Manual Installation -=================== - -There are just two files to install to use this tool from within Galaxy: - -* seq_filter_by_id.py (the Python script) -* seq_filter_by_id.xml (the Galaxy tool definition) - -The suggested location is in the Galaxy folder tools/filters next to the tool -for calling sff_extract.py for converting SFF to FASTQ or FASTA + QUAL. - -You will also need to modify the tools_conf.xml file to tell Galaxy to offer the -tool. One suggested location is in the filters section. Simply add the line:: - - <tool file="filters/seq_filter_by_id.xml" /> - -If you wish to run the unit tests, also add this to tools_conf.xml.sample -and move/copy the test-data files under Galaxy's test-data folder. Then:: - - $ ./run_functional_tests.sh -id seq_filter_by_id - -You will also need to install Biopython 1.54 or later. That's it. - - -History -======= - -======= ====================================================================== -Version Changes -------- ---------------------------------------------------------------------- -v0.0.1 - Initial version, combining three separate scripts for each file format. -v0.0.4 - Record script version when run from Galaxy. - - Faster FASTA code which preserves the original line wrapping. - - Basic unit test included. -v0.0.5 - Check for errors using Python script's return code. - - Cope with malformed FASTA entries without an identifier. -v0.0.6 - Link to Tool Shed added to help text and this documentation. - - Automated installation of the Biopython dependency. - - Use reStructuredText for this README file. - - Adopt standard MIT License. - - Updated citation information (Cock et al. 2013). - - Development moved to GitHub, https://github.com/peterjc/pico_galaxy -======= ====================================================================== - - - -Developers -========== - -This script and related tools were initially developed on the following hg branches: -http://bitbucket.org/peterjc/galaxy-central/src/fasta_filter -http://bitbucket.org/peterjc/galaxy-central/src/tools - -Development has now moved to a dedicated GitHub repository: -https://github.com/peterjc/pico_galaxy/tree/master/tools - -For making the "Galaxy Tool Shed" http://toolshed.g2.bx.psu.edu/ tarball use -the following command from the Galaxy root folder:: - - $ tar -czf seq_filter_by_id.tar.gz tools/filters/seq_filter_by_id.* tools/filters/repository_dependencies.xml test-data/k12_ten_proteins.fasta test-data/k12_hypothetical.fasta test-data/k12_hypothetical.tabular - -Check this worked:: - - $ tar -tzf seq_filter_by_id.tar.gz - tools/filters/seq_filter_by_id.py - tools/filters/seq_filter_by_id.rst - tools/filters/seq_filter_by_id.xml - tools/filters/repository_dependencies.xml - test-data/k12_ten_proteins.fasta - test-data/k12_hypothetical.fasta - test-data/k12_hypothetical.tabular - - -Licence (MIT) -============= - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE.
--- a/tools/filters/seq_filter_by_id.xml Tue Sep 17 11:55:21 2013 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,126 +0,0 @@ -<tool id="seq_filter_by_id" name="Filter sequences by ID" version="0.0.6"> - <description>from a tabular file</description> - <requirements> - <requirement type="package" version="1.61">biopython</requirement> - <requirement type="python-module">Bio</requirement> - </requirements> - <version_command interpreter="python">seq_filter_by_id.py --version</version_command> - <command interpreter="python"> -seq_filter_by_id.py "$input_file" "$input_file.ext" -#if $output_choice_cond.output_choice=="both" - $output_pos $output_neg -#elif $output_choice_cond.output_choice=="pos" - $output_pos - -#elif $output_choice_cond.output_choice=="neg" - - $output_neg -#end if -## TODO - Decide on best way to expose multiple ID files via the XML wrapper. -## Single tabular file, can call the Python script with either UNION or INTERSECTION -UNION "$input_tabular" "$columns" - </command> - <stdio> - <!-- Anything other than zero is an error --> - <exit_code range="1:" /> - <exit_code range=":-1" /> - </stdio> - <inputs> - <param name="input_file" type="data" format="fasta,fastq,sff" label="Sequence file to filter on the identifiers" help="FASTA, FASTQ, or SFF format." /> - <param name="input_tabular" type="data" format="tabular" label="Tabular file containing sequence identifiers"/> - <param name="columns" type="data_column" data_ref="input_tabular" multiple="True" numerical="False" label="Column(s) containing sequence identifiers" help="Multi-select list - hold the appropriate key while clicking to select multiple columns"> - <validator type="no_options" message="Pick at least one column"/> - </param> - <conditional name="output_choice_cond"> - <param name="output_choice" type="select" label="Output positive matches, negative matches, or both?"> - <option value="both">Both positive matches (ID on list) and negative matches (ID not on list), as two files</option> - <option value="pos">Just positive matches (ID on list), as a single file</option> - <option value="neg">Just negative matches (ID not on list), as a single file</option> - </param> - <!-- Seems need these dummy entries here, compare this to indels/indel_sam2interval.xml --> - <when value="both" /> - <when value="pos" /> - <when value="neg" /> - </conditional> - </inputs> - <outputs> - <data name="output_pos" format="fasta" label="With matched ID"> - <!-- TODO - Replace this with format="input:input_fastq" if/when that works --> - <change_format> - <when input_dataset="input_file" attribute="extension" value="sff" format="sff" /> - <when input_dataset="input_file" attribute="extension" value="fastq" format="fastq" /> - <when input_dataset="input_file" attribute="extension" value="fastqsanger" format="fastqsanger" /> - <when input_dataset="input_file" attribute="extension" value="fastqsolexa" format="fastqsolexa" /> - <when input_dataset="input_file" attribute="extension" value="fastqillumina" format="fastqillumina" /> - <when input_dataset="input_file" attribute="extension" value="fastqcssanger" format="fastqcssanger" /> - </change_format> - <filter>output_choice_cond["output_choice"] != "neg"</filter> - </data> - <data name="output_neg" format="fasta" label="Without matched ID"> - <!-- TODO - Replace this with format="input:input_fastq" if/when that works --> - <change_format> - <when input_dataset="input_file" attribute="extension" value="sff" format="sff" /> - <when input_dataset="input_file" attribute="extension" value="fastq" format="fastq" /> - <when input_dataset="input_file" attribute="extension" value="fastqsanger" format="fastqsanger" /> - <when input_dataset="input_file" attribute="extension" value="fastqsolexa" format="fastqsolexa" /> - <when input_dataset="input_file" attribute="extension" value="fastqillumina" format="fastqillumina" /> - <when input_dataset="input_file" attribute="extension" value="fastqcssanger" format="fastqcssanger" /> - </change_format> - <filter>output_choice_cond["output_choice"] != "pos"</filter> - </data> - </outputs> - <tests> - <test> - <param name="input_file" value="k12_ten_proteins.fasta" ftype="fasta" /> - <param name="input_tabular" value="k12_hypothetical.tabular" ftype="tabular" /> - <param name="columns" value="1" /> - <param name="output_choice" value="pos" /> - <output name="output_pos" file="k12_hypothetical.fasta" ftype="fasta" /> - </test> - </tests> - <help> - -**What it does** - -By default it divides a FASTA, FASTQ or Standard Flowgram Format (SFF) file in -two, those sequences with or without an ID present in the tabular file column(s) -specified. You can opt to have a single output file of just the matching records, -or just the non-matching ones. - -Note that the order of sequences in the original sequence file is preserved, as -is any Roche XML Manifest in an SFF file. Also, if any sequences share an -identifier (which would be very unusual in SFF files), duplicates are not removed. - -**Example Usage** - -You may have performed some kind of contamination search, for example running -BLASTN against a database of cloning vectors or bacteria, giving you a tabular -file containing read identifiers. You could use this tool to extract only the -reads without BLAST matches (i.e. those which do not match your contaminant -database). - -You may have a file of FASTA sequences which has been used with some analysis -tool giving tabular output, which has then been filtered on some criteria. -You can then use this tool to divide the original FASTA file into those entries -matching or not matching your criteria (those with or without their identifier -in the filtered tabular file). - -**References** - -If you use this Galaxy tool in work leading to a scientific publication please -cite the following papers: - -Peter J.A. Cock, Björn A. Grüning, Konrad Paszkiewicz and Leighton Pritchard (2013). -Galaxy tools and workflows for sequence analysis with applications -in molecular plant pathology. PeerJ 1:e167 -http://dx.doi.org/10.7717/peerj.167 - -This tool uses Biopython to read and write SFF files, so you may also wish to -cite the Biopython application note (and Galaxy too of course): - -Cock et al (2009). Biopython: freely available Python tools for computational -molecular biology and bioinformatics. Bioinformatics 25(11) 1422-3. -http://dx.doi.org/10.1093/bioinformatics/btp163 pmid:19304878. - -This tool is available to install into other Galaxy Instances via the Galaxy -Tool Shed at http://toolshed.g2.bx.psu.edu/view/peterjc/seq_filter_by_id - </help> -</tool>
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tools/seq_filter_by_id/README.rst Thu Sep 26 10:05:52 2013 -0400 @@ -0,0 +1,124 @@ +Galaxy tool to filter FASTA, FASTQ or SFF sequences by ID +========================================================= + +This tool is copyright 2010-2013 by Peter Cock, The James Hutton Institute +(formerly SCRI, Scottish Crop Research Institute), UK. All rights reserved. +See the licence text below. + +This tool is a short Python script (using both the Galaxy and Biopython library +functions) which divides a FASTA, FASTQ, or SFF file in two, those sequences with +or without an ID present in the specified column(s) of a tabular file. Example uses +include filtering based on search results from a tool like NCBI BLAST before +assembly. + +This tool is available from the Galaxy Tool Shed at: + +* http://toolshed.g2.bx.psu.edu/view/peterjc/seq_filter_by_id + +See also sister tools: + +* http://toolshed.g2.bx.psu.edu/view/peterjc/seq_select_by_id +* http://toolshed.g2.bx.psu.edu/view/peterjc/seq_rename + + +Automated Installation +====================== + +This should be straightforward using the Galaxy Tool Shed, which should be +able to automatically install the dependency on Biopython, and then install +this tool and run its unit tests. + + +Manual Installation +=================== + +There are just two files to install to use this tool from within Galaxy: + +* seq_filter_by_id.py (the Python script) +* seq_filter_by_id.xml (the Galaxy tool definition) + +The suggested location is a dedicated tools/seq_filter_by_id folder. + +You will also need to modify the tools_conf.xml file to tell Galaxy to offer the +tool. One suggested location is in the filters section. Simply add the line:: + + <tool file="seq_filter_by_id/seq_filter_by_id.xml" /> + +If you wish to run the unit tests, also add this to tools_conf.xml.sample +and move/copy the test-data files under Galaxy's test-data folder. Then:: + + $ ./run_functional_tests.sh -id seq_filter_by_id + +You will also need to install Biopython 1.54 or later. That's it. + + +History +======= + +======= ====================================================================== +Version Changes +------- ---------------------------------------------------------------------- +v0.0.1 - Initial version, combining three separate scripts for each file format. +v0.0.4 - Record script version when run from Galaxy. + - Faster FASTA code which preserves the original line wrapping. + - Basic unit test included. +v0.0.5 - Check for errors using Python script's return code. + - Cope with malformed FASTA entries without an identifier. +v0.0.6 - Link to Tool Shed added to help text and this documentation. + - Automated installation of the Biopython dependency. + - Use reStructuredText for this README file. + - Adopt standard MIT License. + - Updated citation information (Cock et al. 2013). + - Development moved to GitHub, https://github.com/peterjc/pico_galaxy + - Renamed folder and adopted README.rst naming. +======= ====================================================================== + + + +Developers +========== + +This script and related tools were initially developed on the following hg branches: +http://bitbucket.org/peterjc/galaxy-central/src/fasta_filter +http://bitbucket.org/peterjc/galaxy-central/src/tools + +Development has now moved to a dedicated GitHub repository: +https://github.com/peterjc/pico_galaxy/tree/master/tools + +For making the "Galaxy Tool Shed" http://toolshed.g2.bx.psu.edu/ tarball use +the following command from the Galaxy root folder:: + + $ tar -czf seq_filter_by_id.tar.gz tools/seq_filter_by_id/README.rst tools/seq_filter_by_id/seq_filter_by_id.* tools/seq_filter_by_id/repository_dependencies.xml test-data/k12_ten_proteins.fasta test-data/k12_hypothetical.fasta test-data/k12_hypothetical.tabular + +Check this worked:: + + $ tar -tzf seq_filter_by_id.tar.gz + tools/seq_filter_by_id/README.rst + tools/seq_filter_by_id/seq_filter_by_id.py + tools/seq_filter_by_id/seq_filter_by_id.xml + tools/seq_filter_by_id/repository_dependencies.xml + test-data/k12_ten_proteins.fasta + test-data/k12_hypothetical.fasta + test-data/k12_hypothetical.tabular + + +Licence (MIT) +============= + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE.
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tools/seq_filter_by_id/repository_dependencies.xml Thu Sep 26 10:05:52 2013 -0400 @@ -0,0 +1,6 @@ +<?xml version="1.0"?> +<repositories description="This requires Biopython as a dependency."> +<!-- Leave out the tool shed and revision to get the current + tool shed and latest revision at the time of upload --> +<repository changeset_revision="2f6c871cfa35" name="package_biopython_1_61" owner="biopython" toolshed="http://testtoolshed.g2.bx.psu.edu" /> +</repositories>
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tools/seq_filter_by_id/seq_filter_by_id.py Thu Sep 26 10:05:52 2013 -0400 @@ -0,0 +1,260 @@ +#!/usr/bin/env python +"""Filter a FASTA, FASTQ or SSF file with IDs from a tabular file. + +Takes six command line options, tabular filename, ID column numbers (comma +separated list using one based counting), input filename, input type (e.g. +FASTA or SFF) and two output filenames (for records with and without the +given IDs, same format as input sequence file). + +If either output filename is just a minus sign, that file is not created. +This is intended to allow output for just the matched (or just the non-matched) +records. + +When filtering an SFF file, any Roche XML manifest in the input file is +preserved in both output files. + +Note in the default NCBI BLAST+ tabular output, the query sequence ID is +in column one, and the ID of the match from the database is in column two. +Here sensible values for the column numbers would therefore be "1" or "2". + +This tool is a short Python script which requires Biopython 1.54 or later +for SFF file support. If you use this tool in scientific work leading to a +publication, please cite the Biopython application note: + +Cock et al 2009. Biopython: freely available Python tools for computational +molecular biology and bioinformatics. Bioinformatics 25(11) 1422-3. +http://dx.doi.org/10.1093/bioinformatics/btp163 pmid:19304878. + +This script is copyright 2010-2013 by Peter Cock, The James Hutton Institute +(formerly the Scottish Crop Research Institute, SCRI), UK. All rights reserved. +See accompanying text file for licence details (MIT license). + +This is version 0.1.0 of the script, use -v or --version to get the version. +""" +import os +import sys + +def stop_err(msg, err=1): + sys.stderr.write(msg.rstrip() + "\n") + sys.exit(err) + +if "-v" in sys.argv or "--version" in sys.argv: + print "v0.1.0" + sys.exit(0) + +#Parse Command Line +if len(sys.argv) - 1 < 7 or len(sys.argv) % 2 == 1: + stop_err("Expected 7 or more arguments, 5 required " + "(in seq, seq format, out pos, out neg, logic) " + "then one or more pairs (tab file, columns), " + "got %i:\n%s" % (len(sys.argv)-1, " ".join(sys.argv))) + +in_file, seq_format, out_positive_file, out_negative_file, logic = sys.argv[1:6] + +if not os.path.isfile(in_file): + stop_err("Missing input file %r" % in_file) +if out_positive_file == "-" and out_negative_file == "-": + stop_err("Neither output file requested") +if logic not in ["UNION", "INTERSECTION"]: + stop_err("Fifth agrument should be 'UNION' or 'INTERSECTION', not %r" % logic) + +identifiers = [] +for i in range((len(sys.argv) - 6) // 2): + tabular_file = sys.argv[6+2*i] + cols_arg = sys.argv[7+2*i] + if not os.path.isfile(tabular_file): + stop_err("Missing tabular identifier file %r" % tabular_file) + try: + columns = [int(arg)-1 for arg in cols_arg.split(",")] + except ValueError: + stop_err("Expected list of columns (comma separated integers), got %r" % cols_arg) + if min(columns) < 0: + stop_err("Expect one-based column numbers (not zero-based counting), got %r" % cols_arg) + identifiers.append((tabular_file, columns)) + +#Read tabular file(s) and record all specified identifiers +ids = None #Will be a set +for tabular_file, columns in identifiers: + file_ids = set() + handle = open(tabular_file, "rU") + if len(columns)>1: + #General case of many columns + for line in handle: + if line.startswith("#"): + #Ignore comments + continue + parts = line.rstrip("\n").split("\t") + for col in columns: + file_ids.add(parts[col]) + else: + #Single column, special case speed up + col = columns[0] + for line in handle: + if not line.startswith("#"): + file_ids.add(line.rstrip("\n").split("\t")[col]) + print "Using %i IDs from column %s in tabular file" % (len(file_ids), ", ".join(str(col+1) for col in columns)) + if ids is None: + ids = file_ids + if logic == "UNION": + ids.update(file_ids) + else: + ids.intersection_update(file_ids) + handle.close() +if len(identifiers) > 1: + if logic == "UNION": + print "Have %i IDs combined from %i tabular files" % (len(ids), len(identifiers)) + else: + print "Have %i IDs in common from %i tabular files" % (len(ids), len(identifiers)) + + +def crude_fasta_iterator(handle): + """Yields tuples, record ID and the full record as a string.""" + while True: + line = handle.readline() + if line == "": + return # Premature end of file, or just empty? + if line[0] == ">": + break + + no_id_warned = False + while True: + if line[0] != ">": + raise ValueError( + "Records in Fasta files should start with '>' character") + try: + id = line[1:].split(None, 1)[0] + except IndexError: + if not no_id_warned: + sys.stderr.write("WARNING - Malformed FASTA entry with no identifier\n") + no_id_warned = True + id = None + lines = [line] + line = handle.readline() + while True: + if not line: + break + if line[0] == ">": + break + lines.append(line) + line = handle.readline() + yield id, "".join(lines) + if not line: + return # StopIteration + + +def fasta_filter(in_file, pos_file, neg_file, wanted): + """FASTA filter producing 60 character line wrapped outout.""" + pos_count = neg_count = 0 + #Galaxy now requires Python 2.5+ so can use with statements, + with open(in_file) as in_handle: + #Doing the if statement outside the loop for speed + #(with the downside of three very similar loops). + if pos_file != "-" and neg_file != "-": + print "Generating two FASTA files" + with open(pos_file, "w") as pos_handle: + with open(neg_file, "w") as neg_handle: + for identifier, record in crude_fasta_iterator(in_handle): + if identifier in wanted: + pos_handle.write(record) + pos_count += 1 + else: + neg_handle.write(record) + neg_count += 1 + elif pos_file != "-": + print "Generating matching FASTA file" + with open(pos_file, "w") as pos_handle: + for identifier, record in crude_fasta_iterator(in_handle): + if identifier in wanted: + pos_handle.write(record) + pos_count += 1 + else: + neg_count += 1 + else: + print "Generating non-matching FASTA file" + assert neg_file != "-" + with open(neg_file, "w") as neg_handle: + for identifier, record in crude_fasta_iterator(in_handle): + if identifier in wanted: + pos_count += 1 + else: + neg_handle.write(record) + neg_count += 1 + return pos_count, neg_count + + +if seq_format.lower()=="sff": + #Now write filtered SFF file based on IDs from BLAST file + try: + from Bio.SeqIO.SffIO import SffIterator, SffWriter + except ImportError: + stop_err("SFF filtering requires Biopython 1.54 or later") + + try: + from Bio.SeqIO.SffIO import ReadRocheXmlManifest + except ImportError: + #Prior to Biopython 1.56 this was a private function + from Bio.SeqIO.SffIO import _sff_read_roche_index_xml as ReadRocheXmlManifest + in_handle = open(in_file, "rb") #must be binary mode! + try: + manifest = ReadRocheXmlManifest(in_handle) + except ValueError: + manifest = None + #This makes two passes though the SFF file with isn't so efficient, + #but this makes the code simple. + pos_count = neg_count = 0 + if out_positive_file != "-": + out_handle = open(out_positive_file, "wb") + writer = SffWriter(out_handle, xml=manifest) + in_handle.seek(0) #start again after getting manifest + pos_count = writer.write_file(rec for rec in SffIterator(in_handle) if rec.id in ids) + out_handle.close() + if out_negative_file != "-": + out_handle = open(out_negative_file, "wb") + writer = SffWriter(out_handle, xml=manifest) + in_handle.seek(0) #start again + neg_count = writer.write_file(rec for rec in SffIterator(in_handle) if rec.id not in ids) + out_handle.close() + #And we're done + in_handle.close() + #At the time of writing, Galaxy doesn't show SFF file read counts, + #so it is useful to put them in stdout and thus shown in job info. + print "%i with and %i without specified IDs" % (pos_count, neg_count) +elif seq_format.lower()=="fasta": + #Write filtered FASTA file based on IDs from tabular file + pos_count, neg_count = fasta_filter(in_file, out_positive_file, out_negative_file, ids) + print "%i with and %i without specified IDs" % (pos_count, neg_count) +elif seq_format.lower().startswith("fastq"): + #Write filtered FASTQ file based on IDs from tabular file + from galaxy_utils.sequence.fastq import fastqReader, fastqWriter + reader = fastqReader(open(in_file, "rU")) + if out_positive_file != "-" and out_negative_file != "-": + print "Generating two FASTQ files" + positive_writer = fastqWriter(open(out_positive_file, "w")) + negative_writer = fastqWriter(open(out_negative_file, "w")) + for record in reader: + #The [1:] is because the fastaReader leaves the > on the identifier. + if record.identifier and record.identifier.split()[0][1:] in ids: + positive_writer.write(record) + else: + negative_writer.write(record) + positive_writer.close() + negative_writer.close() + elif out_positive_file != "-": + print "Generating matching FASTQ file" + positive_writer = fastqWriter(open(out_positive_file, "w")) + for record in reader: + #The [1:] is because the fastaReader leaves the > on the identifier. + if record.identifier and record.identifier.split()[0][1:] in ids: + positive_writer.write(record) + positive_writer.close() + elif out_negative_file != "-": + print "Generating non-matching FASTQ file" + negative_writer = fastqWriter(open(out_negative_file, "w")) + for record in reader: + #The [1:] is because the fastaReader leaves the > on the identifier. + if not record.identifier or record.identifier.split()[0][1:] not in ids: + negative_writer.write(record) + negative_writer.close() + reader.close() +else: + stop_err("Unsupported file type %r" % seq_format)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tools/seq_filter_by_id/seq_filter_by_id.xml Thu Sep 26 10:05:52 2013 -0400 @@ -0,0 +1,126 @@ +<tool id="seq_filter_by_id" name="Filter sequences by ID" version="0.0.6"> + <description>from a tabular file</description> + <requirements> + <requirement type="package" version="1.61">biopython</requirement> + <requirement type="python-module">Bio</requirement> + </requirements> + <version_command interpreter="python">seq_filter_by_id.py --version</version_command> + <command interpreter="python"> +seq_filter_by_id.py "$input_file" "$input_file.ext" +#if $output_choice_cond.output_choice=="both" + $output_pos $output_neg +#elif $output_choice_cond.output_choice=="pos" + $output_pos - +#elif $output_choice_cond.output_choice=="neg" + - $output_neg +#end if +## TODO - Decide on best way to expose multiple ID files via the XML wrapper. +## Single tabular file, can call the Python script with either UNION or INTERSECTION +UNION "$input_tabular" "$columns" + </command> + <stdio> + <!-- Anything other than zero is an error --> + <exit_code range="1:" /> + <exit_code range=":-1" /> + </stdio> + <inputs> + <param name="input_file" type="data" format="fasta,fastq,sff" label="Sequence file to filter on the identifiers" help="FASTA, FASTQ, or SFF format." /> + <param name="input_tabular" type="data" format="tabular" label="Tabular file containing sequence identifiers"/> + <param name="columns" type="data_column" data_ref="input_tabular" multiple="True" numerical="False" label="Column(s) containing sequence identifiers" help="Multi-select list - hold the appropriate key while clicking to select multiple columns"> + <validator type="no_options" message="Pick at least one column"/> + </param> + <conditional name="output_choice_cond"> + <param name="output_choice" type="select" label="Output positive matches, negative matches, or both?"> + <option value="both">Both positive matches (ID on list) and negative matches (ID not on list), as two files</option> + <option value="pos">Just positive matches (ID on list), as a single file</option> + <option value="neg">Just negative matches (ID not on list), as a single file</option> + </param> + <!-- Seems need these dummy entries here, compare this to indels/indel_sam2interval.xml --> + <when value="both" /> + <when value="pos" /> + <when value="neg" /> + </conditional> + </inputs> + <outputs> + <data name="output_pos" format="fasta" label="With matched ID"> + <!-- TODO - Replace this with format="input:input_fastq" if/when that works --> + <change_format> + <when input_dataset="input_file" attribute="extension" value="sff" format="sff" /> + <when input_dataset="input_file" attribute="extension" value="fastq" format="fastq" /> + <when input_dataset="input_file" attribute="extension" value="fastqsanger" format="fastqsanger" /> + <when input_dataset="input_file" attribute="extension" value="fastqsolexa" format="fastqsolexa" /> + <when input_dataset="input_file" attribute="extension" value="fastqillumina" format="fastqillumina" /> + <when input_dataset="input_file" attribute="extension" value="fastqcssanger" format="fastqcssanger" /> + </change_format> + <filter>output_choice_cond["output_choice"] != "neg"</filter> + </data> + <data name="output_neg" format="fasta" label="Without matched ID"> + <!-- TODO - Replace this with format="input:input_fastq" if/when that works --> + <change_format> + <when input_dataset="input_file" attribute="extension" value="sff" format="sff" /> + <when input_dataset="input_file" attribute="extension" value="fastq" format="fastq" /> + <when input_dataset="input_file" attribute="extension" value="fastqsanger" format="fastqsanger" /> + <when input_dataset="input_file" attribute="extension" value="fastqsolexa" format="fastqsolexa" /> + <when input_dataset="input_file" attribute="extension" value="fastqillumina" format="fastqillumina" /> + <when input_dataset="input_file" attribute="extension" value="fastqcssanger" format="fastqcssanger" /> + </change_format> + <filter>output_choice_cond["output_choice"] != "pos"</filter> + </data> + </outputs> + <tests> + <test> + <param name="input_file" value="k12_ten_proteins.fasta" ftype="fasta" /> + <param name="input_tabular" value="k12_hypothetical.tabular" ftype="tabular" /> + <param name="columns" value="1" /> + <param name="output_choice" value="pos" /> + <output name="output_pos" file="k12_hypothetical.fasta" ftype="fasta" /> + </test> + </tests> + <help> + +**What it does** + +By default it divides a FASTA, FASTQ or Standard Flowgram Format (SFF) file in +two, those sequences with or without an ID present in the tabular file column(s) +specified. You can opt to have a single output file of just the matching records, +or just the non-matching ones. + +Note that the order of sequences in the original sequence file is preserved, as +is any Roche XML Manifest in an SFF file. Also, if any sequences share an +identifier (which would be very unusual in SFF files), duplicates are not removed. + +**Example Usage** + +You may have performed some kind of contamination search, for example running +BLASTN against a database of cloning vectors or bacteria, giving you a tabular +file containing read identifiers. You could use this tool to extract only the +reads without BLAST matches (i.e. those which do not match your contaminant +database). + +You may have a file of FASTA sequences which has been used with some analysis +tool giving tabular output, which has then been filtered on some criteria. +You can then use this tool to divide the original FASTA file into those entries +matching or not matching your criteria (those with or without their identifier +in the filtered tabular file). + +**References** + +If you use this Galaxy tool in work leading to a scientific publication please +cite the following papers: + +Peter J.A. Cock, Björn A. Grüning, Konrad Paszkiewicz and Leighton Pritchard (2013). +Galaxy tools and workflows for sequence analysis with applications +in molecular plant pathology. PeerJ 1:e167 +http://dx.doi.org/10.7717/peerj.167 + +This tool uses Biopython to read and write SFF files, so you may also wish to +cite the Biopython application note (and Galaxy too of course): + +Cock et al (2009). Biopython: freely available Python tools for computational +molecular biology and bioinformatics. Bioinformatics 25(11) 1422-3. +http://dx.doi.org/10.1093/bioinformatics/btp163 pmid:19304878. + +This tool is available to install into other Galaxy Instances via the Galaxy +Tool Shed at http://toolshed.g2.bx.psu.edu/view/peterjc/seq_filter_by_id + </help> +</tool>