Mercurial > repos > peterjc > seq_filter_by_id

changeset 4:3b990ab4ae04 draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Uploaded v0.0.6, auto-dependencies, RST, MIT
author peterjc
date Wed, 24 Jul 2013 11:53:21 -0400
parents 66d1ca92fb38
children 00153c4efec3
files tools/filters/seq_filter_by_id.py tools/filters/seq_filter_by_id.rst tools/filters/seq_filter_by_id.txt tools/filters/seq_filter_by_id.xml
diffstat 4 files changed, 187 insertions(+), 124 deletions(-) [+]
line wrap: on
line diff
--- a/tools/filters/seq_filter_by_id.py	Wed Apr 24 11:33:48 2013 -0400
+++ b/tools/filters/seq_filter_by_id.py	Wed Jul 24 11:53:21 2013 -0400
@@ -29,8 +29,9 @@
 (formerly the Scottish Crop Research Institute, SCRI), UK. All rights reserved.
 See accompanying text file for licence details (MIT/BSD style).
 
-This is version 0.0.5 of the script, use -v or --version to get the version.
+This is version 0.1.0 of the script, use -v or --version to get the version.
 """
+import os
 import sys
 
 def stop_err(msg, err=1):
@@ -38,46 +39,72 @@
     sys.exit(err)
 
 if "-v" in sys.argv or "--version" in sys.argv:
-    print "v0.0.5"
+    print "v0.1.0"
     sys.exit(0)
 
 #Parse Command Line
-try:
-    tabular_file, cols_arg, in_file, seq_format, out_positive_file, out_negative_file = sys.argv[1:]
-except ValueError:
-    stop_err("Expected six arguments (tab file, columns, in seq, seq format, out pos, out neg), got %i:\n%s" % (len(sys.argv)-1, " ".join(sys.argv)))
-try:
-    columns = [int(arg)-1 for arg in cols_arg.split(",")]
-except ValueError:
-    stop_err("Expected list of columns (comma separated integers), got %s" % cols_arg)
-if min(columns) < 0:
-    stop_err("Expect one-based column numbers (not zero-based counting), got %s" % cols_arg)
+if len(sys.argv) - 1 < 7 or len(sys.argv) % 2 == 1:
+    stop_err("Expected 7 or more arguments, 5 required "
+             "(in seq, seq format, out pos, out neg, logic) "
+             "then one or more pairs (tab file, columns), "
+             "got %i:\n%s" % (len(sys.argv)-1, " ".join(sys.argv)))
 
+in_file, seq_format, out_positive_file, out_negative_file, logic = sys.argv[1:6]
+
+if not os.path.isfile(in_file):
+    stop_err("Missing input file %r" % in_file)
 if out_positive_file == "-" and out_negative_file == "-":
     stop_err("Neither output file requested")
+if logic not in ["UNION", "INTERSECTION"]:
+    stop_err("Fifth agrument should be 'UNION' or 'INTERSECTION', not %r" % logic)
 
+identifiers = []
+for i in range((len(sys.argv) - 6) // 2):
+    tabular_file = sys.argv[6+2*i]
+    cols_arg = sys.argv[7+2*i]
+    if not os.path.isfile(tabular_file):
+        stop_err("Missing tabular identifier file %r" % tabular_file)
+    try:
+        columns = [int(arg)-1 for arg in cols_arg.split(",")]
+    except ValueError:
+        stop_err("Expected list of columns (comma separated integers), got %r" % cols_arg)
+    if min(columns) < 0:
+        stop_err("Expect one-based column numbers (not zero-based counting), got %r" % cols_arg)
+    identifiers.append((tabular_file, columns))
 
-#Read tabular file and record all specified identifiers
-ids = set()
-handle = open(tabular_file, "rU")
-if len(columns)>1:
-    #General case of many columns
-    for line in handle:
-        if line.startswith("#"):
-            #Ignore comments
-            continue
-        parts = line.rstrip("\n").split("\t")
-        for col in columns:
-            ids.add(parts[col])
-    print "Using %i IDs from %i columns of tabular file" % (len(ids), len(columns))
-else:
-    #Single column, special case speed up
-    col = columns[0]
-    for line in handle:
-        if not line.startswith("#"):
-            ids.add(line.rstrip("\n").split("\t")[col])
-    print "Using %i IDs from tabular file" % (len(ids))
-handle.close()
+#Read tabular file(s) and record all specified identifiers
+ids = None #Will be a set
+for tabular_file, columns in identifiers:
+    file_ids = set()
+    handle = open(tabular_file, "rU")
+    if len(columns)>1:
+        #General case of many columns
+        for line in handle:
+            if line.startswith("#"):
+                #Ignore comments
+                continue
+            parts = line.rstrip("\n").split("\t")
+            for col in columns:
+                file_ids.add(parts[col])
+    else:
+        #Single column, special case speed up
+        col = columns[0]
+        for line in handle:
+            if not line.startswith("#"):
+                file_ids.add(line.rstrip("\n").split("\t")[col])
+    print "Using %i IDs from column %s in tabular file" % (len(file_ids), ", ".join(str(col+1) for col in columns))
+    if ids is None:
+        ids = file_ids
+    if logic == "UNION":
+        ids.update(file_ids)
+    else:
+        ids.intersection_update(file_ids)
+    handle.close()
+if len(identifiers) > 1:
+    if logic == "UNION":
+        print "Have %i IDs combined from %i tabular files" % (len(ids), len(identifiers))
+    else:
+        print "Have %i IDs in common from %i tabular files" % (len(ids), len(identifiers))
 
 
 def crude_fasta_iterator(handle):
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tools/filters/seq_filter_by_id.rst	Wed Jul 24 11:53:21 2013 -0400
@@ -0,0 +1,120 @@
+Galaxy tool to filter FASTA, FASTQ or SFF sequences by ID
+=========================================================
+
+This tool is copyright 2010-2013 by Peter Cock, The James Hutton Institute
+(formerly SCRI, Scottish Crop Research Institute), UK. All rights reserved.
+See the licence text below.
+
+This tool is a short Python script (using both the Galaxy and Biopython library
+functions) which divides a FASTA, FASTQ, or SFF file in two, those sequences with
+or without an ID present in the specified column(s) of a tabular file. Example uses
+include filtering based on search results from a tool like NCBI BLAST before
+assembly.
+
+This tool is available from the Galaxy Tool Shed at:
+
+* http://toolshed.g2.bx.psu.edu/view/peterjc/seq_filter_by_id
+
+See also sister tools:
+
+* http://toolshed.g2.bx.psu.edu/view/peterjc/seq_select_by_id
+* http://toolshed.g2.bx.psu.edu/view/peterjc/seq_rename
+
+
+Automated Installation
+======================
+
+This should be straightforward using the Galaxy Tool Shed, which should be
+able to automatically install the dependency on Biopython, and then install
+this tool and run its unit tests.
+
+
+Manual Installation
+===================
+
+There are just two files to install to use this tool from within Galaxy:
+
+* seq_filter_by_id.py (the Python script)
+* seq_filter_by_id.xml (the Galaxy tool definition)
+
+The suggested location is in the Galaxy folder tools/filters next to the tool
+for calling sff_extract.py for converting SFF to FASTQ or FASTA + QUAL.
+
+You will also need to modify the tools_conf.xml file to tell Galaxy to offer the
+tool. One suggested location is in the filters section. Simply add the line::
+
+    <tool file="filters/seq_filter_by_id.xml" />
+
+If you wish to run the unit tests, also add this to tools_conf.xml.sample
+and move/copy the test-data files under Galaxy's test-data folder. Then::
+
+    $ ./run_functional_tests.sh -id seq_filter_by_id
+
+You will also need to install Biopython 1.54 or later. That's it.
+
+
+History
+=======
+
+======= ======================================================================
+Version Changes
+------- ----------------------------------------------------------------------
+v0.0.1   - Initial version, combining three separate scripts for each file format.
+v0.0.4   - Record script version when run from Galaxy.
+         - Faster FASTA code which preserves the original line wrapping.
+         - Basic unit test included.
+v0.0.5   - Check for errors using Python script's return code.
+         - Cope with malformed FASTA entries without an identifier.
+v0.0.6   - Link to Tool Shed added to help text and this documentation.
+         - Automated installation of the Biopython dependency.
+         - Use reStructuredText for this README file.
+         - Adopt standard MIT License.
+======= ======================================================================
+
+
+
+Developers
+==========
+
+This script and related tools are being developed on the following hg branch:
+http://bitbucket.org/peterjc/galaxy-central/src/tools
+
+This incorporates the previously used hg branch:
+http://bitbucket.org/peterjc/galaxy-central/src/fasta_filter
+
+For making the "Galaxy Tool Shed" http://toolshed.g2.bx.psu.edu/ tarball use
+the following command from the Galaxy root folder::
+
+    $ tar -czf seq_filter_by_id.tar.gz tools/filters/seq_filter_by_id.* test-data/k12_ten_proteins.fasta test-data/k12_hypothetical.fasta test-data/k12_hypothetical.tabular
+
+Check this worked::
+
+    $ tar -tzf seq_filter_by_id.tar.gz
+    filter/seq_filter_by_id.py
+    filter/seq_filter_by_id.rst
+    filter/seq_filter_by_id.xml
+    test-data/k12_ten_proteins.fasta
+    test-data/k12_hypothetical.fasta
+    test-data/k12_hypothetical.tabular
+
+
+Licence (MIT)
+=============
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
--- a/tools/filters/seq_filter_by_id.txt	Wed Apr 24 11:33:48 2013 -0400
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,89 +0,0 @@
-Galaxy tool to filter FASTA, FASTQ or SFF sequences by ID
-=========================================================
-
-This tool is copyright 2010-2013 by Peter Cock, The James Hutton Institute
-(formerly SCRI, Scottish Crop Research Institute), UK. All rights reserved.
-See the licence text below.
-
-This tool is a short Python script (using both the Galaxy and Biopython library
-functions) which divides a FASTA, FASTQ, or SFF file in two, those sequences with
-or without an ID present in the specified column(s) of a tabular file. Example uses
-include filtering based on search results from a tool like NCBI BLAST before
-assembly.
-
-
-Installation
-============
-
-There are just two files to install:
-
-* seq_filter_by_id.py (the Python script)
-* seq_filter_by_id.xml (the Galaxy tool definition)
-
-The suggested location is in the Galaxy folder tools/filters next to the tool
-for calling sff_extract.py for converting SFF to FASTQ or FASTA + QUAL.
-
-You will also need to modify the tools_conf.xml file to tell Galaxy to offer the
-tool. One suggested location is in the filters section. Simply add the line:
-
-<tool file="filters/sff_filter_by_id.xml" />
-
-You will also need to install Biopython 1.54 or later. That's it.
-
-
-History
-=======
-
-v0.0.1 - Initial version, combining three separate scripts for each file format.
-v0.0.4 - Record script version when run from Galaxy.
-       - Faster FASTA code which preserves the original line wrapping.
-       - Basic unit test included.
-v0.0.5 - Check for errors using Python script's return code.
-       - Cope with malformed FASTA entries without an identifier.
-
-
-Developers
-==========
-
-This script and related tools are being developed on the following hg branch:
-http://bitbucket.org/peterjc/galaxy-central/src/tools
-
-This incorporates the previously used hg branch:
-http://bitbucket.org/peterjc/galaxy-central/src/fasta_filter
-
-For making the "Galaxy Tool Shed" http://toolshed.g2.bx.psu.edu/ tarball use
-the following command from the Galaxy root folder:
-
-$ tar -czf seq_filter_by_id.tar.gz tools/filters/seq_filter_by_id.* test-data/k12_ten_proteins.fasta test-data/k12_hypothetical.fasta test-data/k12_hypothetical.tabular
-
-Check this worked:
-
-$ tar -tzf seq_filter_by_id.tar.gz
-filter/seq_filter_by_id.py
-filter/seq_filter_by_id.txt
-filter/seq_filter_by_id.xml
-test-data/k12_ten_proteins.fasta
-test-data/k12_hypothetical.fasta
-test-data/k12_hypothetical.tabular
-
-
-Licence (MIT/BSD style)
-=======================
-
-Permission to use, copy, modify, and distribute this software and its
-documentation with or without modifications and for any purpose and
-without fee is hereby granted, provided that any copyright notices
-appear in all copies and that both those copyright notices and this
-permission notice appear in supporting documentation, and that the
-names of the contributors or copyright holders not be used in
-advertising or publicity pertaining to distribution of the software
-without specific prior permission.
-
-THE CONTRIBUTORS AND COPYRIGHT HOLDERS OF THIS SOFTWARE DISCLAIM ALL
-WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL THE
-CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY SPECIAL, INDIRECT
-OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
-OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
-OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE
-OR PERFORMANCE OF THIS SOFTWARE.
--- a/tools/filters/seq_filter_by_id.xml	Wed Apr 24 11:33:48 2013 -0400
+++ b/tools/filters/seq_filter_by_id.xml	Wed Jul 24 11:53:21 2013 -0400
@@ -1,8 +1,8 @@
-<tool id="seq_filter_by_id" name="Filter sequences by ID" version="0.0.5">
+<tool id="seq_filter_by_id" name="Filter sequences by ID" version="0.0.6">
 	<description>from a tabular file</description>
 	<version_command interpreter="python">seq_filter_by_id.py --version</version_command>
 	<command interpreter="python">
-seq_filter_by_id.py $input_tabular $columns $input_file $input_file.ext
+seq_filter_by_id.py "$input_file" "$input_file.ext"
 #if $output_choice_cond.output_choice=="both"
  $output_pos $output_neg
 #elif $output_choice_cond.output_choice=="pos"
@@ -10,6 +10,9 @@
 #elif $output_choice_cond.output_choice=="neg"
  - $output_neg
 #end if
+## TODO - Decide on best way to expose multiple ID files via the XML wrapper.
+## Single tabular file, can call the Python script with either UNION or INTERSECTION
+UNION "$input_tabular" "$columns"
 	</command>
         <stdio>
                 <!-- Anything other than zero is an error -->
@@ -109,5 +112,7 @@
 molecular biology and bioinformatics. Bioinformatics 25(11) 1422-3.
 http://dx.doi.org/10.1093/bioinformatics/btp163 pmid:19304878.
 
+This tool is available to install into other Galaxy Instances via the Galaxy
+Tool Shed at http://toolshed.g2.bx.psu.edu/view/peterjc/seq_filter_by_id
 	</help>
 </tool>