# HG changeset patch
# User peterjc
# Date 1380204352 14400
# Node ID b7ec1c9db38db5235977dd5de13abcd7fa2def88
# Parent  b1b4ad43caf191812117a4d144b89e1518b21787
Uploaded v0.0.6g, renamed folder and README file

diff -r b1b4ad43caf1 -r b7ec1c9db38d tools/filters/repository_dependencies.xml
--- a/tools/filters/repository_dependencies.xml	Tue Sep 17 11:55:21 2013 -0400
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,6 +0,0 @@
-<?xml version="1.0"?>
-<repositories description="This requires Biopython as a dependency.">
-<!-- Leave out the tool shed and revision to get the current
-     tool shed and latest revision at the time of upload -->
-<repository changeset_revision="2f6c871cfa35" name="package_biopython_1_61" owner="biopython" toolshed="http://testtoolshed.g2.bx.psu.edu" />
-</repositories>
diff -r b1b4ad43caf1 -r b7ec1c9db38d tools/filters/seq_filter_by_id.py
--- a/tools/filters/seq_filter_by_id.py	Tue Sep 17 11:55:21 2013 -0400
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,260 +0,0 @@
-#!/usr/bin/env python
-"""Filter a FASTA, FASTQ or SSF file with IDs from a tabular file.
-
-Takes six command line options, tabular filename, ID column numbers (comma
-separated list using one based counting), input filename, input type (e.g.
-FASTA or SFF) and two output filenames (for records with and without the
-given IDs, same format as input sequence file).
-
-If either output filename is just a minus sign, that file is not created.
-This is intended to allow output for just the matched (or just the non-matched)
-records.
-
-When filtering an SFF file, any Roche XML manifest in the input file is
-preserved in both output files.
-
-Note in the default NCBI BLAST+ tabular output, the query sequence ID is
-in column one, and the ID of the match from the database is in column two.
-Here sensible values for the column numbers would therefore be "1" or "2".
-
-This tool is a short Python script which requires Biopython 1.54 or later
-for SFF file support. If you use this tool in scientific work leading to a
-publication, please cite the Biopython application note:
-
-Cock et al 2009. Biopython: freely available Python tools for computational
-molecular biology and bioinformatics. Bioinformatics 25(11) 1422-3.
-http://dx.doi.org/10.1093/bioinformatics/btp163 pmid:19304878.
-
-This script is copyright 2010-2013 by Peter Cock, The James Hutton Institute
-(formerly the Scottish Crop Research Institute, SCRI), UK. All rights reserved.
-See accompanying text file for licence details (MIT license).
-
-This is version 0.1.0 of the script, use -v or --version to get the version.
-"""
-import os
-import sys
-
-def stop_err(msg, err=1):
-    sys.stderr.write(msg.rstrip() + "\n")
-    sys.exit(err)
-
-if "-v" in sys.argv or "--version" in sys.argv:
-    print "v0.1.0"
-    sys.exit(0)
-
-#Parse Command Line
-if len(sys.argv) - 1 < 7 or len(sys.argv) % 2 == 1:
-    stop_err("Expected 7 or more arguments, 5 required "
-             "(in seq, seq format, out pos, out neg, logic) "
-             "then one or more pairs (tab file, columns), "
-             "got %i:\n%s" % (len(sys.argv)-1, " ".join(sys.argv)))
-
-in_file, seq_format, out_positive_file, out_negative_file, logic = sys.argv[1:6]
-
-if not os.path.isfile(in_file):
-    stop_err("Missing input file %r" % in_file)
-if out_positive_file == "-" and out_negative_file == "-":
-    stop_err("Neither output file requested")
-if logic not in ["UNION", "INTERSECTION"]:
-    stop_err("Fifth agrument should be 'UNION' or 'INTERSECTION', not %r" % logic)
-
-identifiers = []
-for i in range((len(sys.argv) - 6) // 2):
-    tabular_file = sys.argv[6+2*i]
-    cols_arg = sys.argv[7+2*i]
-    if not os.path.isfile(tabular_file):
-        stop_err("Missing tabular identifier file %r" % tabular_file)
-    try:
-        columns = [int(arg)-1 for arg in cols_arg.split(",")]
-    except ValueError:
-        stop_err("Expected list of columns (comma separated integers), got %r" % cols_arg)
-    if min(columns) < 0:
-        stop_err("Expect one-based column numbers (not zero-based counting), got %r" % cols_arg)
-    identifiers.append((tabular_file, columns))
-
-#Read tabular file(s) and record all specified identifiers
-ids = None #Will be a set
-for tabular_file, columns in identifiers:
-    file_ids = set()
-    handle = open(tabular_file, "rU")
-    if len(columns)>1:
-        #General case of many columns
-        for line in handle:
-            if line.startswith("#"):
-                #Ignore comments
-                continue
-            parts = line.rstrip("\n").split("\t")
-            for col in columns:
-                file_ids.add(parts[col])
-    else:
-        #Single column, special case speed up
-        col = columns[0]
-        for line in handle:
-            if not line.startswith("#"):
-                file_ids.add(line.rstrip("\n").split("\t")[col])
-    print "Using %i IDs from column %s in tabular file" % (len(file_ids), ", ".join(str(col+1) for col in columns))
-    if ids is None:
-        ids = file_ids
-    if logic == "UNION":
-        ids.update(file_ids)
-    else:
-        ids.intersection_update(file_ids)
-    handle.close()
-if len(identifiers) > 1:
-    if logic == "UNION":
-        print "Have %i IDs combined from %i tabular files" % (len(ids), len(identifiers))
-    else:
-        print "Have %i IDs in common from %i tabular files" % (len(ids), len(identifiers))
-
-
-def crude_fasta_iterator(handle):
-    """Yields tuples, record ID and the full record as a string."""
-    while True:
-        line = handle.readline()
-        if line == "":
-            return # Premature end of file, or just empty?
-        if line[0] == ">":
-            break
-
-    no_id_warned = False
-    while True:
-        if line[0] != ">":
-            raise ValueError(
-                "Records in Fasta files should start with '>' character")
-        try:
-            id = line[1:].split(None, 1)[0]
-        except IndexError:
-            if not no_id_warned:
-                sys.stderr.write("WARNING - Malformed FASTA entry with no identifier\n")
-                no_id_warned = True
-            id = None
-        lines = [line]
-        line = handle.readline()
-        while True:
-            if not line:
-                break
-            if line[0] == ">":
-                break
-            lines.append(line)
-            line = handle.readline()
-        yield id, "".join(lines)
-        if not line:
-            return # StopIteration
-
-
-def fasta_filter(in_file, pos_file, neg_file, wanted):
-    """FASTA filter producing 60 character line wrapped outout."""
-    pos_count = neg_count = 0
-    #Galaxy now requires Python 2.5+ so can use with statements,
-    with open(in_file) as in_handle:
-        #Doing the if statement outside the loop for speed
-        #(with the downside of three very similar loops).
-        if pos_file != "-" and neg_file != "-":
-            print "Generating two FASTA files"
-            with open(pos_file, "w") as pos_handle:
-                with open(neg_file, "w") as neg_handle:
-                    for identifier, record in crude_fasta_iterator(in_handle):
-                        if identifier in wanted:
-                            pos_handle.write(record)
-                            pos_count += 1
-                        else:
-                            neg_handle.write(record)
-                            neg_count += 1
-        elif pos_file != "-":
-            print "Generating matching FASTA file"
-            with open(pos_file, "w") as pos_handle:
-                for identifier, record in crude_fasta_iterator(in_handle):
-                    if identifier in wanted:
-                        pos_handle.write(record)
-                        pos_count += 1
-                    else:
-                        neg_count += 1
-        else:
-            print "Generating non-matching FASTA file"
-            assert neg_file != "-"
-            with open(neg_file, "w") as neg_handle:
-                for identifier, record in crude_fasta_iterator(in_handle):
-                    if identifier in wanted:
-                        pos_count += 1
-                    else:
-                        neg_handle.write(record)
-                        neg_count += 1
-    return pos_count, neg_count
-
-
-if seq_format.lower()=="sff":
-    #Now write filtered SFF file based on IDs from BLAST file
-    try:
-        from Bio.SeqIO.SffIO import SffIterator, SffWriter
-    except ImportError:
-        stop_err("SFF filtering requires Biopython 1.54 or later")
-
-    try:
-        from Bio.SeqIO.SffIO import ReadRocheXmlManifest
-    except ImportError:
-        #Prior to Biopython 1.56 this was a private function
-        from Bio.SeqIO.SffIO import _sff_read_roche_index_xml as ReadRocheXmlManifest
-    in_handle = open(in_file, "rb") #must be binary mode!
-    try:
-        manifest = ReadRocheXmlManifest(in_handle)
-    except ValueError:
-        manifest = None
-    #This makes two passes though the SFF file with isn't so efficient,
-    #but this makes the code simple.
-    pos_count = neg_count = 0
-    if out_positive_file != "-":
-        out_handle = open(out_positive_file, "wb")
-        writer = SffWriter(out_handle, xml=manifest)
-        in_handle.seek(0) #start again after getting manifest
-        pos_count = writer.write_file(rec for rec in SffIterator(in_handle) if rec.id in ids)
-        out_handle.close()
-    if out_negative_file != "-":
-        out_handle = open(out_negative_file, "wb")
-        writer = SffWriter(out_handle, xml=manifest)
-        in_handle.seek(0) #start again
-        neg_count = writer.write_file(rec for rec in SffIterator(in_handle) if rec.id not in ids)
-        out_handle.close()
-    #And we're done
-    in_handle.close()
-    #At the time of writing, Galaxy doesn't show SFF file read counts,
-    #so it is useful to put them in stdout and thus shown in job info.
-    print "%i with and %i without specified IDs" % (pos_count, neg_count)
-elif seq_format.lower()=="fasta":
-    #Write filtered FASTA file based on IDs from tabular file
-    pos_count, neg_count = fasta_filter(in_file, out_positive_file, out_negative_file, ids)
-    print "%i with and %i without specified IDs" % (pos_count, neg_count)
-elif seq_format.lower().startswith("fastq"):
-    #Write filtered FASTQ file based on IDs from tabular file
-    from galaxy_utils.sequence.fastq import fastqReader, fastqWriter
-    reader = fastqReader(open(in_file, "rU"))
-    if out_positive_file != "-" and out_negative_file != "-":
-        print "Generating two FASTQ files"
-        positive_writer = fastqWriter(open(out_positive_file, "w"))
-        negative_writer = fastqWriter(open(out_negative_file, "w"))
-        for record in reader:
-            #The [1:] is because the fastaReader leaves the > on the identifier.
-            if record.identifier and record.identifier.split()[0][1:] in ids:
-                positive_writer.write(record)
-            else:
-                negative_writer.write(record)
-        positive_writer.close()
-        negative_writer.close()
-    elif out_positive_file != "-":
-        print "Generating matching FASTQ file"
-        positive_writer = fastqWriter(open(out_positive_file, "w"))
-        for record in reader:
-            #The [1:] is because the fastaReader leaves the > on the identifier.
-            if record.identifier and record.identifier.split()[0][1:] in ids:
-                positive_writer.write(record)
-        positive_writer.close()
-    elif out_negative_file != "-":
-        print "Generating non-matching FASTQ file"
-        negative_writer = fastqWriter(open(out_negative_file, "w"))
-        for record in reader:
-            #The [1:] is because the fastaReader leaves the > on the identifier.
-            if not record.identifier or record.identifier.split()[0][1:] not in ids:
-                negative_writer.write(record)
-        negative_writer.close()
-    reader.close()
-else:
-    stop_err("Unsupported file type %r" % seq_format)
diff -r b1b4ad43caf1 -r b7ec1c9db38d tools/filters/seq_filter_by_id.rst
--- a/tools/filters/seq_filter_by_id.rst	Tue Sep 17 11:55:21 2013 -0400
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,124 +0,0 @@
-Galaxy tool to filter FASTA, FASTQ or SFF sequences by ID
-=========================================================
-
-This tool is copyright 2010-2013 by Peter Cock, The James Hutton Institute
-(formerly SCRI, Scottish Crop Research Institute), UK. All rights reserved.
-See the licence text below.
-
-This tool is a short Python script (using both the Galaxy and Biopython library
-functions) which divides a FASTA, FASTQ, or SFF file in two, those sequences with
-or without an ID present in the specified column(s) of a tabular file. Example uses
-include filtering based on search results from a tool like NCBI BLAST before
-assembly.
-
-This tool is available from the Galaxy Tool Shed at:
-
-* http://toolshed.g2.bx.psu.edu/view/peterjc/seq_filter_by_id
-
-See also sister tools:
-
-* http://toolshed.g2.bx.psu.edu/view/peterjc/seq_select_by_id
-* http://toolshed.g2.bx.psu.edu/view/peterjc/seq_rename
-
-
-Automated Installation
-======================
-
-This should be straightforward using the Galaxy Tool Shed, which should be
-able to automatically install the dependency on Biopython, and then install
-this tool and run its unit tests.
-
-
-Manual Installation
-===================
-
-There are just two files to install to use this tool from within Galaxy:
-
-* seq_filter_by_id.py (the Python script)
-* seq_filter_by_id.xml (the Galaxy tool definition)
-
-The suggested location is in the Galaxy folder tools/filters next to the tool
-for calling sff_extract.py for converting SFF to FASTQ or FASTA + QUAL.
-
-You will also need to modify the tools_conf.xml file to tell Galaxy to offer the
-tool. One suggested location is in the filters section. Simply add the line::
-
-    <tool file="filters/seq_filter_by_id.xml" />
-
-If you wish to run the unit tests, also add this to tools_conf.xml.sample
-and move/copy the test-data files under Galaxy's test-data folder. Then::
-
-    $ ./run_functional_tests.sh -id seq_filter_by_id
-
-You will also need to install Biopython 1.54 or later. That's it.
-
-
-History
-=======
-
-======= ======================================================================
-Version Changes
-------- ----------------------------------------------------------------------
-v0.0.1  - Initial version, combining three separate scripts for each file format.
-v0.0.4  - Record script version when run from Galaxy.
-        - Faster FASTA code which preserves the original line wrapping.
-        - Basic unit test included.
-v0.0.5  - Check for errors using Python script's return code.
-        - Cope with malformed FASTA entries without an identifier.
-v0.0.6  - Link to Tool Shed added to help text and this documentation.
-        - Automated installation of the Biopython dependency.
-        - Use reStructuredText for this README file.
-        - Adopt standard MIT License.
-        - Updated citation information (Cock et al. 2013).
-        - Development moved to GitHub, https://github.com/peterjc/pico_galaxy
-======= ======================================================================
-
-
-
-Developers
-==========
-
-This script and related tools were initially developed on the following hg branches:
-http://bitbucket.org/peterjc/galaxy-central/src/fasta_filter
-http://bitbucket.org/peterjc/galaxy-central/src/tools
-
-Development has now moved to a dedicated GitHub repository:
-https://github.com/peterjc/pico_galaxy/tree/master/tools
-
-For making the "Galaxy Tool Shed" http://toolshed.g2.bx.psu.edu/ tarball use
-the following command from the Galaxy root folder::
-
-    $ tar -czf seq_filter_by_id.tar.gz tools/filters/seq_filter_by_id.* tools/filters/repository_dependencies.xml test-data/k12_ten_proteins.fasta test-data/k12_hypothetical.fasta test-data/k12_hypothetical.tabular
-
-Check this worked::
-
-    $ tar -tzf seq_filter_by_id.tar.gz
-    tools/filters/seq_filter_by_id.py
-    tools/filters/seq_filter_by_id.rst
-    tools/filters/seq_filter_by_id.xml
-    tools/filters/repository_dependencies.xml
-    test-data/k12_ten_proteins.fasta
-    test-data/k12_hypothetical.fasta
-    test-data/k12_hypothetical.tabular
-
-
-Licence (MIT)
-=============
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
diff -r b1b4ad43caf1 -r b7ec1c9db38d tools/filters/seq_filter_by_id.xml
--- a/tools/filters/seq_filter_by_id.xml	Tue Sep 17 11:55:21 2013 -0400
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,126 +0,0 @@
-<tool id="seq_filter_by_id" name="Filter sequences by ID" version="0.0.6">
-	<description>from a tabular file</description>
-	<requirements>
-		<requirement type="package" version="1.61">biopython</requirement>
-		<requirement type="python-module">Bio</requirement>
-	</requirements>
-	<version_command interpreter="python">seq_filter_by_id.py --version</version_command>
-	<command interpreter="python">
-seq_filter_by_id.py "$input_file" "$input_file.ext"
-#if $output_choice_cond.output_choice=="both"
- $output_pos $output_neg
-#elif $output_choice_cond.output_choice=="pos"
- $output_pos -
-#elif $output_choice_cond.output_choice=="neg"
- - $output_neg
-#end if
-## TODO - Decide on best way to expose multiple ID files via the XML wrapper.
-## Single tabular file, can call the Python script with either UNION or INTERSECTION
-UNION "$input_tabular" "$columns"
-	</command>
-        <stdio>
-                <!-- Anything other than zero is an error -->
-                <exit_code range="1:" />
-                <exit_code range=":-1" />
-        </stdio>
-	<inputs>
-		<param name="input_file" type="data" format="fasta,fastq,sff" label="Sequence file to filter on the identifiers" help="FASTA, FASTQ, or SFF format." />
-		<param name="input_tabular" type="data" format="tabular" label="Tabular file containing sequence identifiers"/>
-		<param name="columns" type="data_column" data_ref="input_tabular" multiple="True" numerical="False" label="Column(s) containing sequence identifiers" help="Multi-select list - hold the appropriate key while clicking to select multiple columns">
-			<validator type="no_options" message="Pick at least one column"/>
-		</param>
-		<conditional name="output_choice_cond">
-			<param name="output_choice" type="select" label="Output positive matches, negative matches, or both?">
-				<option value="both">Both positive matches (ID on list) and negative matches (ID not on list), as two files</option>
-				<option value="pos">Just positive matches (ID on list), as a single file</option>
-				<option value="neg">Just negative matches (ID not on list), as a single file</option>
-			</param>
-			<!-- Seems need these dummy entries here, compare this to indels/indel_sam2interval.xml -->
-			<when value="both" />
-			<when value="pos" />
-			<when value="neg" />
-		</conditional>
-	</inputs>
-	<outputs>
-		<data name="output_pos" format="fasta" label="With matched ID">
-            <!-- TODO - Replace this with format="input:input_fastq" if/when that works -->
-            <change_format>
-                <when input_dataset="input_file" attribute="extension" value="sff" format="sff" />
-                <when input_dataset="input_file" attribute="extension" value="fastq" format="fastq" />
-                <when input_dataset="input_file" attribute="extension" value="fastqsanger" format="fastqsanger" />
-                <when input_dataset="input_file" attribute="extension" value="fastqsolexa" format="fastqsolexa" />
-                <when input_dataset="input_file" attribute="extension" value="fastqillumina" format="fastqillumina" />
-                <when input_dataset="input_file" attribute="extension" value="fastqcssanger" format="fastqcssanger" />
-            </change_format>
-			<filter>output_choice_cond["output_choice"] != "neg"</filter>
-		</data>
-		<data name="output_neg" format="fasta" label="Without matched ID">
-            <!-- TODO - Replace this with format="input:input_fastq" if/when that works -->
-            <change_format>
-                <when input_dataset="input_file" attribute="extension" value="sff" format="sff" />
-                <when input_dataset="input_file" attribute="extension" value="fastq" format="fastq" />
-                <when input_dataset="input_file" attribute="extension" value="fastqsanger" format="fastqsanger" />
-                <when input_dataset="input_file" attribute="extension" value="fastqsolexa" format="fastqsolexa" />
-                <when input_dataset="input_file" attribute="extension" value="fastqillumina" format="fastqillumina" />
-                <when input_dataset="input_file" attribute="extension" value="fastqcssanger" format="fastqcssanger" />
-            </change_format>
-			<filter>output_choice_cond["output_choice"] != "pos"</filter>
-		</data>
-	</outputs>
-	<tests>
-		<test>
-			<param name="input_file" value="k12_ten_proteins.fasta" ftype="fasta" />
-			<param name="input_tabular" value="k12_hypothetical.tabular" ftype="tabular" />
-			<param name="columns" value="1" />
-			<param name="output_choice" value="pos" />
-			<output name="output_pos" file="k12_hypothetical.fasta" ftype="fasta" />
-		</test>
-	</tests>
-	<help>
-
-**What it does**
-
-By default it divides a FASTA, FASTQ or Standard Flowgram Format (SFF) file in
-two, those sequences with or without an ID present in the tabular file column(s)
-specified. You can opt to have a single output file of just the matching records,
-or just the non-matching ones.
-
-Note that the order of sequences in the original sequence file is preserved, as
-is any Roche XML Manifest in an SFF file. Also, if any sequences share an
-identifier (which would be very unusual in SFF files), duplicates are not removed.
-
-**Example Usage**
-
-You may have performed some kind of contamination search, for example running
-BLASTN against a database of cloning vectors or bacteria, giving you a tabular
-file containing read identifiers. You could use this tool to extract only the
-reads without BLAST matches (i.e. those which do not match your contaminant
-database).
-
-You may have a file of FASTA sequences which has been used with some analysis
-tool giving tabular output, which has then been filtered on some criteria.
-You can then use this tool to divide the original FASTA file into those entries
-matching or not matching your criteria (those with or without their identifier
-in the filtered tabular file).
-
-**References**
-
-If you use this Galaxy tool in work leading to a scientific publication please
-cite the following papers:
-
-Peter J.A. Cock, Björn A. Grüning, Konrad Paszkiewicz and Leighton Pritchard (2013).
-Galaxy tools and workflows for sequence analysis with applications
-in molecular plant pathology. PeerJ 1:e167
-http://dx.doi.org/10.7717/peerj.167
-
-This tool uses Biopython to read and write SFF files, so you may also wish to
-cite the Biopython application note (and Galaxy too of course):
-
-Cock et al (2009). Biopython: freely available Python tools for computational
-molecular biology and bioinformatics. Bioinformatics 25(11) 1422-3.
-http://dx.doi.org/10.1093/bioinformatics/btp163 pmid:19304878.
-
-This tool is available to install into other Galaxy Instances via the Galaxy
-Tool Shed at http://toolshed.g2.bx.psu.edu/view/peterjc/seq_filter_by_id
-	</help>
-</tool>
diff -r b1b4ad43caf1 -r b7ec1c9db38d tools/seq_filter_by_id/README.rst
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tools/seq_filter_by_id/README.rst	Thu Sep 26 10:05:52 2013 -0400
@@ -0,0 +1,124 @@
+Galaxy tool to filter FASTA, FASTQ or SFF sequences by ID
+=========================================================
+
+This tool is copyright 2010-2013 by Peter Cock, The James Hutton Institute
+(formerly SCRI, Scottish Crop Research Institute), UK. All rights reserved.
+See the licence text below.
+
+This tool is a short Python script (using both the Galaxy and Biopython library
+functions) which divides a FASTA, FASTQ, or SFF file in two, those sequences with
+or without an ID present in the specified column(s) of a tabular file. Example uses
+include filtering based on search results from a tool like NCBI BLAST before
+assembly.
+
+This tool is available from the Galaxy Tool Shed at:
+
+* http://toolshed.g2.bx.psu.edu/view/peterjc/seq_filter_by_id
+
+See also sister tools:
+
+* http://toolshed.g2.bx.psu.edu/view/peterjc/seq_select_by_id
+* http://toolshed.g2.bx.psu.edu/view/peterjc/seq_rename
+
+
+Automated Installation
+======================
+
+This should be straightforward using the Galaxy Tool Shed, which should be
+able to automatically install the dependency on Biopython, and then install
+this tool and run its unit tests.
+
+
+Manual Installation
+===================
+
+There are just two files to install to use this tool from within Galaxy:
+
+* seq_filter_by_id.py (the Python script)
+* seq_filter_by_id.xml (the Galaxy tool definition)
+
+The suggested location is a dedicated tools/seq_filter_by_id folder.
+
+You will also need to modify the tools_conf.xml file to tell Galaxy to offer the
+tool. One suggested location is in the filters section. Simply add the line::
+
+    <tool file="seq_filter_by_id/seq_filter_by_id.xml" />
+
+If you wish to run the unit tests, also add this to tools_conf.xml.sample
+and move/copy the test-data files under Galaxy's test-data folder. Then::
+
+    $ ./run_functional_tests.sh -id seq_filter_by_id
+
+You will also need to install Biopython 1.54 or later. That's it.
+
+
+History
+=======
+
+======= ======================================================================
+Version Changes
+------- ----------------------------------------------------------------------
+v0.0.1  - Initial version, combining three separate scripts for each file format.
+v0.0.4  - Record script version when run from Galaxy.
+        - Faster FASTA code which preserves the original line wrapping.
+        - Basic unit test included.
+v0.0.5  - Check for errors using Python script's return code.
+        - Cope with malformed FASTA entries without an identifier.
+v0.0.6  - Link to Tool Shed added to help text and this documentation.
+        - Automated installation of the Biopython dependency.
+        - Use reStructuredText for this README file.
+        - Adopt standard MIT License.
+        - Updated citation information (Cock et al. 2013).
+        - Development moved to GitHub, https://github.com/peterjc/pico_galaxy
+        - Renamed folder and adopted README.rst naming.
+======= ======================================================================
+
+
+
+Developers
+==========
+
+This script and related tools were initially developed on the following hg branches:
+http://bitbucket.org/peterjc/galaxy-central/src/fasta_filter
+http://bitbucket.org/peterjc/galaxy-central/src/tools
+
+Development has now moved to a dedicated GitHub repository:
+https://github.com/peterjc/pico_galaxy/tree/master/tools
+
+For making the "Galaxy Tool Shed" http://toolshed.g2.bx.psu.edu/ tarball use
+the following command from the Galaxy root folder::
+
+    $ tar -czf seq_filter_by_id.tar.gz tools/seq_filter_by_id/README.rst tools/seq_filter_by_id/seq_filter_by_id.* tools/seq_filter_by_id/repository_dependencies.xml test-data/k12_ten_proteins.fasta test-data/k12_hypothetical.fasta test-data/k12_hypothetical.tabular
+
+Check this worked::
+
+    $ tar -tzf seq_filter_by_id.tar.gz
+    tools/seq_filter_by_id/README.rst
+    tools/seq_filter_by_id/seq_filter_by_id.py
+    tools/seq_filter_by_id/seq_filter_by_id.xml
+    tools/seq_filter_by_id/repository_dependencies.xml
+    test-data/k12_ten_proteins.fasta
+    test-data/k12_hypothetical.fasta
+    test-data/k12_hypothetical.tabular
+
+
+Licence (MIT)
+=============
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
diff -r b1b4ad43caf1 -r b7ec1c9db38d tools/seq_filter_by_id/repository_dependencies.xml
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tools/seq_filter_by_id/repository_dependencies.xml	Thu Sep 26 10:05:52 2013 -0400
@@ -0,0 +1,6 @@
+<?xml version="1.0"?>
+<repositories description="This requires Biopython as a dependency.">
+<!-- Leave out the tool shed and revision to get the current
+     tool shed and latest revision at the time of upload -->
+<repository changeset_revision="2f6c871cfa35" name="package_biopython_1_61" owner="biopython" toolshed="http://testtoolshed.g2.bx.psu.edu" />
+</repositories>
diff -r b1b4ad43caf1 -r b7ec1c9db38d tools/seq_filter_by_id/seq_filter_by_id.py
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tools/seq_filter_by_id/seq_filter_by_id.py	Thu Sep 26 10:05:52 2013 -0400
@@ -0,0 +1,260 @@
+#!/usr/bin/env python
+"""Filter a FASTA, FASTQ or SSF file with IDs from a tabular file.
+
+Takes six command line options, tabular filename, ID column numbers (comma
+separated list using one based counting), input filename, input type (e.g.
+FASTA or SFF) and two output filenames (for records with and without the
+given IDs, same format as input sequence file).
+
+If either output filename is just a minus sign, that file is not created.
+This is intended to allow output for just the matched (or just the non-matched)
+records.
+
+When filtering an SFF file, any Roche XML manifest in the input file is
+preserved in both output files.
+
+Note in the default NCBI BLAST+ tabular output, the query sequence ID is
+in column one, and the ID of the match from the database is in column two.
+Here sensible values for the column numbers would therefore be "1" or "2".
+
+This tool is a short Python script which requires Biopython 1.54 or later
+for SFF file support. If you use this tool in scientific work leading to a
+publication, please cite the Biopython application note:
+
+Cock et al 2009. Biopython: freely available Python tools for computational
+molecular biology and bioinformatics. Bioinformatics 25(11) 1422-3.
+http://dx.doi.org/10.1093/bioinformatics/btp163 pmid:19304878.
+
+This script is copyright 2010-2013 by Peter Cock, The James Hutton Institute
+(formerly the Scottish Crop Research Institute, SCRI), UK. All rights reserved.
+See accompanying text file for licence details (MIT license).
+
+This is version 0.1.0 of the script, use -v or --version to get the version.
+"""
+import os
+import sys
+
+def stop_err(msg, err=1):
+    sys.stderr.write(msg.rstrip() + "\n")
+    sys.exit(err)
+
+if "-v" in sys.argv or "--version" in sys.argv:
+    print "v0.1.0"
+    sys.exit(0)
+
+#Parse Command Line
+if len(sys.argv) - 1 < 7 or len(sys.argv) % 2 == 1:
+    stop_err("Expected 7 or more arguments, 5 required "
+             "(in seq, seq format, out pos, out neg, logic) "
+             "then one or more pairs (tab file, columns), "
+             "got %i:\n%s" % (len(sys.argv)-1, " ".join(sys.argv)))
+
+in_file, seq_format, out_positive_file, out_negative_file, logic = sys.argv[1:6]
+
+if not os.path.isfile(in_file):
+    stop_err("Missing input file %r" % in_file)
+if out_positive_file == "-" and out_negative_file == "-":
+    stop_err("Neither output file requested")
+if logic not in ["UNION", "INTERSECTION"]:
+    stop_err("Fifth agrument should be 'UNION' or 'INTERSECTION', not %r" % logic)
+
+identifiers = []
+for i in range((len(sys.argv) - 6) // 2):
+    tabular_file = sys.argv[6+2*i]
+    cols_arg = sys.argv[7+2*i]
+    if not os.path.isfile(tabular_file):
+        stop_err("Missing tabular identifier file %r" % tabular_file)
+    try:
+        columns = [int(arg)-1 for arg in cols_arg.split(",")]
+    except ValueError:
+        stop_err("Expected list of columns (comma separated integers), got %r" % cols_arg)
+    if min(columns) < 0:
+        stop_err("Expect one-based column numbers (not zero-based counting), got %r" % cols_arg)
+    identifiers.append((tabular_file, columns))
+
+#Read tabular file(s) and record all specified identifiers
+ids = None #Will be a set
+for tabular_file, columns in identifiers:
+    file_ids = set()
+    handle = open(tabular_file, "rU")
+    if len(columns)>1:
+        #General case of many columns
+        for line in handle:
+            if line.startswith("#"):
+                #Ignore comments
+                continue
+            parts = line.rstrip("\n").split("\t")
+            for col in columns:
+                file_ids.add(parts[col])
+    else:
+        #Single column, special case speed up
+        col = columns[0]
+        for line in handle:
+            if not line.startswith("#"):
+                file_ids.add(line.rstrip("\n").split("\t")[col])
+    print "Using %i IDs from column %s in tabular file" % (len(file_ids), ", ".join(str(col+1) for col in columns))
+    if ids is None:
+        ids = file_ids
+    if logic == "UNION":
+        ids.update(file_ids)
+    else:
+        ids.intersection_update(file_ids)
+    handle.close()
+if len(identifiers) > 1:
+    if logic == "UNION":
+        print "Have %i IDs combined from %i tabular files" % (len(ids), len(identifiers))
+    else:
+        print "Have %i IDs in common from %i tabular files" % (len(ids), len(identifiers))
+
+
+def crude_fasta_iterator(handle):
+    """Yields tuples, record ID and the full record as a string."""
+    while True:
+        line = handle.readline()
+        if line == "":
+            return # Premature end of file, or just empty?
+        if line[0] == ">":
+            break
+
+    no_id_warned = False
+    while True:
+        if line[0] != ">":
+            raise ValueError(
+                "Records in Fasta files should start with '>' character")
+        try:
+            id = line[1:].split(None, 1)[0]
+        except IndexError:
+            if not no_id_warned:
+                sys.stderr.write("WARNING - Malformed FASTA entry with no identifier\n")
+                no_id_warned = True
+            id = None
+        lines = [line]
+        line = handle.readline()
+        while True:
+            if not line:
+                break
+            if line[0] == ">":
+                break
+            lines.append(line)
+            line = handle.readline()
+        yield id, "".join(lines)
+        if not line:
+            return # StopIteration
+
+
+def fasta_filter(in_file, pos_file, neg_file, wanted):
+    """FASTA filter producing 60 character line wrapped outout."""
+    pos_count = neg_count = 0
+    #Galaxy now requires Python 2.5+ so can use with statements,
+    with open(in_file) as in_handle:
+        #Doing the if statement outside the loop for speed
+        #(with the downside of three very similar loops).
+        if pos_file != "-" and neg_file != "-":
+            print "Generating two FASTA files"
+            with open(pos_file, "w") as pos_handle:
+                with open(neg_file, "w") as neg_handle:
+                    for identifier, record in crude_fasta_iterator(in_handle):
+                        if identifier in wanted:
+                            pos_handle.write(record)
+                            pos_count += 1
+                        else:
+                            neg_handle.write(record)
+                            neg_count += 1
+        elif pos_file != "-":
+            print "Generating matching FASTA file"
+            with open(pos_file, "w") as pos_handle:
+                for identifier, record in crude_fasta_iterator(in_handle):
+                    if identifier in wanted:
+                        pos_handle.write(record)
+                        pos_count += 1
+                    else:
+                        neg_count += 1
+        else:
+            print "Generating non-matching FASTA file"
+            assert neg_file != "-"
+            with open(neg_file, "w") as neg_handle:
+                for identifier, record in crude_fasta_iterator(in_handle):
+                    if identifier in wanted:
+                        pos_count += 1
+                    else:
+                        neg_handle.write(record)
+                        neg_count += 1
+    return pos_count, neg_count
+
+
+if seq_format.lower()=="sff":
+    #Now write filtered SFF file based on IDs from BLAST file
+    try:
+        from Bio.SeqIO.SffIO import SffIterator, SffWriter
+    except ImportError:
+        stop_err("SFF filtering requires Biopython 1.54 or later")
+
+    try:
+        from Bio.SeqIO.SffIO import ReadRocheXmlManifest
+    except ImportError:
+        #Prior to Biopython 1.56 this was a private function
+        from Bio.SeqIO.SffIO import _sff_read_roche_index_xml as ReadRocheXmlManifest
+    in_handle = open(in_file, "rb") #must be binary mode!
+    try:
+        manifest = ReadRocheXmlManifest(in_handle)
+    except ValueError:
+        manifest = None
+    #This makes two passes though the SFF file with isn't so efficient,
+    #but this makes the code simple.
+    pos_count = neg_count = 0
+    if out_positive_file != "-":
+        out_handle = open(out_positive_file, "wb")
+        writer = SffWriter(out_handle, xml=manifest)
+        in_handle.seek(0) #start again after getting manifest
+        pos_count = writer.write_file(rec for rec in SffIterator(in_handle) if rec.id in ids)
+        out_handle.close()
+    if out_negative_file != "-":
+        out_handle = open(out_negative_file, "wb")
+        writer = SffWriter(out_handle, xml=manifest)
+        in_handle.seek(0) #start again
+        neg_count = writer.write_file(rec for rec in SffIterator(in_handle) if rec.id not in ids)
+        out_handle.close()
+    #And we're done
+    in_handle.close()
+    #At the time of writing, Galaxy doesn't show SFF file read counts,
+    #so it is useful to put them in stdout and thus shown in job info.
+    print "%i with and %i without specified IDs" % (pos_count, neg_count)
+elif seq_format.lower()=="fasta":
+    #Write filtered FASTA file based on IDs from tabular file
+    pos_count, neg_count = fasta_filter(in_file, out_positive_file, out_negative_file, ids)
+    print "%i with and %i without specified IDs" % (pos_count, neg_count)
+elif seq_format.lower().startswith("fastq"):
+    #Write filtered FASTQ file based on IDs from tabular file
+    from galaxy_utils.sequence.fastq import fastqReader, fastqWriter
+    reader = fastqReader(open(in_file, "rU"))
+    if out_positive_file != "-" and out_negative_file != "-":
+        print "Generating two FASTQ files"
+        positive_writer = fastqWriter(open(out_positive_file, "w"))
+        negative_writer = fastqWriter(open(out_negative_file, "w"))
+        for record in reader:
+            #The [1:] is because the fastaReader leaves the > on the identifier.
+            if record.identifier and record.identifier.split()[0][1:] in ids:
+                positive_writer.write(record)
+            else:
+                negative_writer.write(record)
+        positive_writer.close()
+        negative_writer.close()
+    elif out_positive_file != "-":
+        print "Generating matching FASTQ file"
+        positive_writer = fastqWriter(open(out_positive_file, "w"))
+        for record in reader:
+            #The [1:] is because the fastaReader leaves the > on the identifier.
+            if record.identifier and record.identifier.split()[0][1:] in ids:
+                positive_writer.write(record)
+        positive_writer.close()
+    elif out_negative_file != "-":
+        print "Generating non-matching FASTQ file"
+        negative_writer = fastqWriter(open(out_negative_file, "w"))
+        for record in reader:
+            #The [1:] is because the fastaReader leaves the > on the identifier.
+            if not record.identifier or record.identifier.split()[0][1:] not in ids:
+                negative_writer.write(record)
+        negative_writer.close()
+    reader.close()
+else:
+    stop_err("Unsupported file type %r" % seq_format)
diff -r b1b4ad43caf1 -r b7ec1c9db38d tools/seq_filter_by_id/seq_filter_by_id.xml
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tools/seq_filter_by_id/seq_filter_by_id.xml	Thu Sep 26 10:05:52 2013 -0400
@@ -0,0 +1,126 @@
+<tool id="seq_filter_by_id" name="Filter sequences by ID" version="0.0.6">
+	<description>from a tabular file</description>
+	<requirements>
+		<requirement type="package" version="1.61">biopython</requirement>
+		<requirement type="python-module">Bio</requirement>
+	</requirements>
+	<version_command interpreter="python">seq_filter_by_id.py --version</version_command>
+	<command interpreter="python">
+seq_filter_by_id.py "$input_file" "$input_file.ext"
+#if $output_choice_cond.output_choice=="both"
+ $output_pos $output_neg
+#elif $output_choice_cond.output_choice=="pos"
+ $output_pos -
+#elif $output_choice_cond.output_choice=="neg"
+ - $output_neg
+#end if
+## TODO - Decide on best way to expose multiple ID files via the XML wrapper.
+## Single tabular file, can call the Python script with either UNION or INTERSECTION
+UNION "$input_tabular" "$columns"
+	</command>
+        <stdio>
+                <!-- Anything other than zero is an error -->
+                <exit_code range="1:" />
+                <exit_code range=":-1" />
+        </stdio>
+	<inputs>
+		<param name="input_file" type="data" format="fasta,fastq,sff" label="Sequence file to filter on the identifiers" help="FASTA, FASTQ, or SFF format." />
+		<param name="input_tabular" type="data" format="tabular" label="Tabular file containing sequence identifiers"/>
+		<param name="columns" type="data_column" data_ref="input_tabular" multiple="True" numerical="False" label="Column(s) containing sequence identifiers" help="Multi-select list - hold the appropriate key while clicking to select multiple columns">
+			<validator type="no_options" message="Pick at least one column"/>
+		</param>
+		<conditional name="output_choice_cond">
+			<param name="output_choice" type="select" label="Output positive matches, negative matches, or both?">
+				<option value="both">Both positive matches (ID on list) and negative matches (ID not on list), as two files</option>
+				<option value="pos">Just positive matches (ID on list), as a single file</option>
+				<option value="neg">Just negative matches (ID not on list), as a single file</option>
+			</param>
+			<!-- Seems need these dummy entries here, compare this to indels/indel_sam2interval.xml -->
+			<when value="both" />
+			<when value="pos" />
+			<when value="neg" />
+		</conditional>
+	</inputs>
+	<outputs>
+		<data name="output_pos" format="fasta" label="With matched ID">
+            <!-- TODO - Replace this with format="input:input_fastq" if/when that works -->
+            <change_format>
+                <when input_dataset="input_file" attribute="extension" value="sff" format="sff" />
+                <when input_dataset="input_file" attribute="extension" value="fastq" format="fastq" />
+                <when input_dataset="input_file" attribute="extension" value="fastqsanger" format="fastqsanger" />
+                <when input_dataset="input_file" attribute="extension" value="fastqsolexa" format="fastqsolexa" />
+                <when input_dataset="input_file" attribute="extension" value="fastqillumina" format="fastqillumina" />
+                <when input_dataset="input_file" attribute="extension" value="fastqcssanger" format="fastqcssanger" />
+            </change_format>
+			<filter>output_choice_cond["output_choice"] != "neg"</filter>
+		</data>
+		<data name="output_neg" format="fasta" label="Without matched ID">
+            <!-- TODO - Replace this with format="input:input_fastq" if/when that works -->
+            <change_format>
+                <when input_dataset="input_file" attribute="extension" value="sff" format="sff" />
+                <when input_dataset="input_file" attribute="extension" value="fastq" format="fastq" />
+                <when input_dataset="input_file" attribute="extension" value="fastqsanger" format="fastqsanger" />
+                <when input_dataset="input_file" attribute="extension" value="fastqsolexa" format="fastqsolexa" />
+                <when input_dataset="input_file" attribute="extension" value="fastqillumina" format="fastqillumina" />
+                <when input_dataset="input_file" attribute="extension" value="fastqcssanger" format="fastqcssanger" />
+            </change_format>
+			<filter>output_choice_cond["output_choice"] != "pos"</filter>
+		</data>
+	</outputs>
+	<tests>
+		<test>
+			<param name="input_file" value="k12_ten_proteins.fasta" ftype="fasta" />
+			<param name="input_tabular" value="k12_hypothetical.tabular" ftype="tabular" />
+			<param name="columns" value="1" />
+			<param name="output_choice" value="pos" />
+			<output name="output_pos" file="k12_hypothetical.fasta" ftype="fasta" />
+		</test>
+	</tests>
+	<help>
+
+**What it does**
+
+By default it divides a FASTA, FASTQ or Standard Flowgram Format (SFF) file in
+two, those sequences with or without an ID present in the tabular file column(s)
+specified. You can opt to have a single output file of just the matching records,
+or just the non-matching ones.
+
+Note that the order of sequences in the original sequence file is preserved, as
+is any Roche XML Manifest in an SFF file. Also, if any sequences share an
+identifier (which would be very unusual in SFF files), duplicates are not removed.
+
+**Example Usage**
+
+You may have performed some kind of contamination search, for example running
+BLASTN against a database of cloning vectors or bacteria, giving you a tabular
+file containing read identifiers. You could use this tool to extract only the
+reads without BLAST matches (i.e. those which do not match your contaminant
+database).
+
+You may have a file of FASTA sequences which has been used with some analysis
+tool giving tabular output, which has then been filtered on some criteria.
+You can then use this tool to divide the original FASTA file into those entries
+matching or not matching your criteria (those with or without their identifier
+in the filtered tabular file).
+
+**References**
+
+If you use this Galaxy tool in work leading to a scientific publication please
+cite the following papers:
+
+Peter J.A. Cock, Björn A. Grüning, Konrad Paszkiewicz and Leighton Pritchard (2013).
+Galaxy tools and workflows for sequence analysis with applications
+in molecular plant pathology. PeerJ 1:e167
+http://dx.doi.org/10.7717/peerj.167
+
+This tool uses Biopython to read and write SFF files, so you may also wish to
+cite the Biopython application note (and Galaxy too of course):
+
+Cock et al (2009). Biopython: freely available Python tools for computational
+molecular biology and bioinformatics. Bioinformatics 25(11) 1422-3.
+http://dx.doi.org/10.1093/bioinformatics/btp163 pmid:19304878.
+
+This tool is available to install into other Galaxy Instances via the Galaxy
+Tool Shed at http://toolshed.g2.bx.psu.edu/view/peterjc/seq_filter_by_id
+	</help>
+</tool>