Mercurial > repos > devteam > subtract_query

changeset 0:8a3448727cec draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Imported from capsule None
author devteam
date Tue, 01 Apr 2014 09:12:49 -0400
parents
children df4bce472400
files subtract_query.py subtract_query.xml test-data/2.txt test-data/eq-removebeginning.dat test-data/eq-showbeginning.dat test-data/eq-showbeginning_e.dat test-data/eq-showtail.dat test-data/subtract-query-1.dat test-data/subtract-query-2.dat test-data/subtract-query-3.dat test-data/subtract-query-4.dat tool_dependencies.xml
diffstat 12 files changed, 376 insertions(+), 0 deletions(-) [+]
line wrap: on
line diff
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/subtract_query.py	Tue Apr 01 09:12:49 2014 -0400
@@ -0,0 +1,111 @@
+#!/usr/bin/env python
+# Greg Von Kuster
+
+"""
+Subtract an entire query from another query
+usage: %prog in_file_1 in_file_2 begin_col end_col output
+    --ignore-empty-end-cols: ignore empty end columns when subtracting
+"""
+import sys, re
+from bx.cookbook import doc_optparse
+
+# Older py compatibility
+try:
+    set()
+except:
+    from sets import Set as set
+
+assert sys.version_info[:2] >= ( 2, 4 )
+
+def get_lines(fname, begin_col='', end_col='', ignore_empty_end_cols=False):
+    lines = set([])
+    i = 0
+    for i, line in enumerate(file(fname)):
+        line = line.rstrip('\r\n')
+        if line and not line.startswith('#'):
+            if begin_col and end_col:
+                """Both begin_col and end_col must be integers at this point."""
+                try:
+                    line = line.split('\t')
+                    line = '\t'.join([line[j] for j in range(begin_col-1, end_col)])
+                    if ignore_empty_end_cols:
+                        # removing empty fields, we do not compare empty fields at the end of a line.
+                        line = line.rstrip()
+                    lines.add( line )
+                except: pass
+            else:
+                if ignore_empty_end_cols:
+                    # removing empty fields, we do not compare empty fields at the end of a line.
+                    line = line.rstrip()
+                lines.add( line )
+    if i: return (i+1, lines)
+    else: return (i, lines)
+
+def main():
+    
+    # Parsing Command Line here
+    options, args = doc_optparse.parse( __doc__ )
+
+    try:
+        inp1_file, inp2_file, begin_col, end_col, out_file = args
+    except:
+        doc_optparse.exception()
+    
+    begin_col = begin_col.strip()
+    end_col = end_col.strip()
+    
+    if begin_col != 'None' or end_col != 'None':
+        """
+        The user selected columns for restriction.  We'll allow default
+        values for both begin_col and end_col as long as the user selected
+        at least one of them for restriction.
+        """
+        if begin_col == 'None':
+            begin_col = end_col
+        elif end_col == 'None':
+            end_col = begin_col
+        begin_col = int(begin_col)
+        end_col = int(end_col)
+        """Make sure that begin_col <= end_col (switch if not)"""
+        if begin_col > end_col:
+            tmp_col = end_col
+            end_col = begin_col
+            begin_col = tmp_col
+    else:
+        begin_col = end_col = ''
+
+    try:
+        fo = open(out_file,'w')
+    except:
+        print >> sys.stderr, "Unable to open output file"
+        sys.exit()
+
+    """
+    len1 is the number of lines in inp1_file
+    lines1 is the set of unique lines in inp1_file
+    diff1 is the number of duplicate lines removed from inp1_file
+    """
+    len1, lines1 = get_lines(inp1_file, begin_col, end_col, options.ignore_empty_end_cols)
+    diff1 = len1 - len(lines1)
+    len2, lines2 = get_lines(inp2_file, begin_col, end_col, options.ignore_empty_end_cols)
+    
+    lines1.difference_update(lines2)
+    """lines1 is now the set of unique lines in inp1_file - the set of unique lines in inp2_file"""
+
+    for line in lines1:
+        print >> fo, line
+
+    fo.close()
+    
+    info_msg = 'Subtracted %d lines. ' %((len1 - diff1) - len(lines1))
+    
+    if begin_col and end_col:
+        info_msg += 'Restricted to columns c' + str(begin_col) + ' thru c' + str(end_col) + '. '
+
+    if diff1 > 0:
+        info_msg += 'Eliminated %d duplicate/blank/comment/invalid lines from first query.' %diff1
+    
+    print info_msg
+
+if __name__ == "__main__":
+    main()
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/subtract_query.xml	Tue Apr 01 09:12:49 2014 -0400
@@ -0,0 +1,131 @@
+<tool id="subtract_query1" name="Subtract Whole Dataset" version="0.1">
+  <description>from another dataset</description>
+  <requirements>
+    <requirement type="package" version="0.7.1">bx-python</requirement>
+    <requirement type="package" version="1.0.0">galaxy-ops</requirement>
+  </requirements>
+  <command interpreter="python">
+    subtract_query.py $input1 $input2 $begin_col $end_col $output
+    #if str($ignore_empty_end_cols) == 'true':
+      --ignore-empty-end-cols
+    #end if
+
+  </command>
+  <inputs>
+    <param format="txt" name="input2" type="data" label="Subtract" help="Second dataset" />
+    <param format="txt" name="input1" type="data" label="from" help="First dataset" />
+    <param name="begin_col" type="data_column" data_ref="input1" force_select="False" label="Restrict subtraction between 'begin column'" />
+    <param name="end_col" type="data_column" data_ref="input1" force_select="False" label="and 'end column'" help="Specifying columns for restricting subtraction is available only for tabular formatted datasets" />
+    <param name="ignore_empty_end_cols" type="boolean" label="Ignore empty columns and whitespace at end of line when subtracting"/>
+  </inputs>
+  <outputs>
+    <data format="input" name="output" metadata_source="input1" />
+  </outputs>
+  <tests>
+  	<!-- Subtract 2 non-tabular files with no column restrictions. -->
+  	<!-- Cannot figure out why this test won't pass, it works in real time... -->
+  	<!--
+    <test>
+      <param name="input1" value="1.txt" />
+      <param name="input2" value="2.txt" />
+      <param name="begin_col" value="None" />
+      <param name="end_col" value="None" />
+      <output name="output" file="subtract-query-1.dat" />
+    </test>
+    -->
+  	<!-- Subtract 2 tabular files with no column restrictions. -->
+    <test>
+      <param name="input1" value="eq-showbeginning.dat" />
+      <param name="input2" value="eq-showtail.dat" />
+      <param name="begin_col" value="None" />
+      <param name="end_col" value="None" />
+      <output name="output" file="subtract-query-2.dat" />
+    </test>
+  	<!-- Subtract 2 tabular files with column restrictions. -->
+    <test>
+      <param name="input1" value="eq-showbeginning.dat" />
+      <param name="input2" value="eq-removebeginning.dat" />
+      <param name="begin_col" value="c1" />
+      <param name="end_col" value="c3" />
+      <output name="output" file="subtract-query-3.dat" />
+    </test>
+  	<!-- Subtract a non-tabular file from a tabular file with no column restrictions. -->
+    <test>
+      <param name="input1" value="eq-showbeginning.dat" />
+      <param name="input2" value="2.txt" />
+      <param name="begin_col" value="None" />
+      <param name="end_col" value="None" />
+      <output name="output" file="subtract-query-4.dat" />
+    </test>
+    <!-- Subtract  2 tabular files with no column restrictions, ignoring empty end columns. -->
+    <test>
+      <param name="input1" value="eq-showbeginning_e.dat" />
+      <param name="input2" value="eq-showtail.dat" />
+      <param name="begin_col" value="None" />
+      <param name="end_col" value="None" />
+      <param name="ignore_empty_end_cols" value="true" />
+      <output name="output" file="subtract-query-2.dat" />
+    </test>
+  </tests>
+  <help>
+
+.. class:: infomark
+
+**TIP:** This tool complements the tool in the **Operate on Genomic Intervals** tool set which subtracts the intervals of two datasets.
+
+
+-----
+
+**Syntax**
+
+This tool subtracts an entire dataset from another dataset.  
+
+- Any text format is valid.
+- If both dataset formats are tabular, you may restrict the subtraction to specific columns **contained in both datasets** and the resulting dataset will include only the columns specified. 
+- The begin column must be less than or equal to the end column.  If it is not, begin column is switched with end column.
+- If begin column is specified but end column is not, end column will default to begin_column (and vice versa).
+- All blank and comment lines are skipped and not included in the resulting dataset (comment lines are lines beginning with a # character).
+- Duplicate lines are eliminated from both dataset prior to subtraction.  If any duplicate lines were eliminated from the first dataset, the number is displayed in the resulting history item.
+
+-----
+
+**Example**
+
+If this is the **First dataset**::
+
+  chr1            4225    19670
+  chr10           6       8
+  chr1            24417   24420
+  chr6_hla_hap2   0       150
+  chr2            1       5
+  chr10           2       10
+  chr1            30      55
+  chrY            1       20
+  chr1            1225979 42287290
+  chr10           7       8 
+
+and this is the **Second dataset**::
+
+  chr1            4225    19670
+  chr10           6       8
+  chr1            24417   24420
+  chr6_hla_hap2   0       150
+  chr2            1       5
+  chr1            30      55
+  chrY            1       20
+  chr1            1225979 42287290
+
+Subtracting the **Second dataset** from the **First dataset** (including all columns) will yield::
+
+  chr10           7       8 
+  chr10           2       10
+
+Conversely, subtracting the **First dataset** from the **Second dataset** (including all columns) will result in an empty dataset.
+
+Subtracting the **Second dataset** from the **First dataset** (restricting to columns c1 and c2) will yield::
+
+  chr10           7
+  chr10           2
+
+  </help>
+</tool>
\ No newline at end of file
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/test-data/2.txt	Tue Apr 01 09:12:49 2014 -0400
@@ -0,0 +1,8 @@
+chr1    4225    19670
+chr10   6       8
+chr1    24417   24420
+chr6_hla_hap2   0       150
+chr2    1       5
+chr1    30      55
+chrY    1       20
+chr1    1225979 42287290
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/test-data/eq-removebeginning.dat	Tue Apr 01 09:12:49 2014 -0400
@@ -0,0 +1,60 @@
+chr11	116124407	116124501	CCDS8374.1_cds_0_0_chr11_116124408_r	0	-
+chr11	116206508	116206563	CCDS8377.1_cds_0_0_chr11_116206509_f	0	+
+chr11	116211733	116212337	CCDS8378.1_cds_0_0_chr11_116211734_r	0	-
+chr11	1812377	1812407	CCDS7726.1_cds_0_0_chr11_1812378_f	0	+
+chr12	38440094	38440321	CCDS8736.1_cds_0_0_chr12_38440095_r	0	-
+chr13	112381694	112381953	CCDS9526.1_cds_0_0_chr13_112381695_f	0	+
+chr14	98710240	98712285	CCDS9949.1_cds_0_0_chr14_98710241_r	0	-
+chr15	41486872	41487060	CCDS10096.1_cds_0_0_chr15_41486873_r	0	-
+chr15	41673708	41673857	CCDS10097.1_cds_0_0_chr15_41673709_f	0	+
+chr15	41679161	41679250	CCDS10098.1_cds_0_0_chr15_41679162_r	0	-
+chr15	41826029	41826196	CCDS10101.1_cds_0_0_chr15_41826030_f	0	+
+chr16	142908	143003	CCDS10397.1_cds_0_0_chr16_142909_f	0	+
+chr16	179963	180135	CCDS10401.1_cds_0_0_chr16_179964_r	0	-
+chr16	244413	244681	CCDS10402.1_cds_0_0_chr16_244414_f	0	+
+chr16	259268	259383	CCDS10403.1_cds_0_0_chr16_259269_r	0	-
+chr18	23786114	23786321	CCDS11891.1_cds_0_0_chr18_23786115_r	0	-
+chr18	59406881	59407046	CCDS11985.1_cds_0_0_chr18_59406882_f	0	+
+chr18	59455932	59456337	CCDS11986.1_cds_0_0_chr18_59455933_r	0	-
+chr18	59600586	59600754	CCDS11988.1_cds_0_0_chr18_59600587_f	0	+
+chr19	59068595	59069564	CCDS12866.1_cds_0_0_chr19_59068596_f	0	+
+chr19	59236026	59236146	CCDS12872.1_cds_0_0_chr19_59236027_r	0	-
+chr19	59297998	59298008	CCDS12877.1_cds_0_0_chr19_59297999_f	0	+
+chr19	59302168	59302288	CCDS12878.1_cds_0_0_chr19_59302169_r	0	-
+chr2	118288583	118288668	CCDS2120.1_cds_0_0_chr2_118288584_f	0	+
+chr2	118394148	118394202	CCDS2121.1_cds_0_0_chr2_118394149_r	0	-
+chr2	220190202	220190242	CCDS2441.1_cds_0_0_chr2_220190203_f	0	+
+chr2	220229609	220230869	CCDS2443.1_cds_0_0_chr2_220229610_r	0	-
+chr20	33330413	33330423	CCDS13249.1_cds_0_0_chr20_33330414_r	0	-
+chr20	33513606	33513792	CCDS13255.1_cds_0_0_chr20_33513607_f	0	+
+chr20	33579500	33579527	CCDS13256.1_cds_0_0_chr20_33579501_r	0	-
+chr20	33593260	33593348	CCDS13257.1_cds_0_0_chr20_33593261_f	0	+
+chr21	32707032	32707192	CCDS13614.1_cds_0_0_chr21_32707033_f	0	+
+chr21	32869641	32870022	CCDS13615.1_cds_0_0_chr21_32869642_r	0	-
+chr21	33321040	33322012	CCDS13620.1_cds_0_0_chr21_33321041_f	0	+
+chr21	33744994	33745040	CCDS13625.1_cds_0_0_chr21_33744995_r	0	-
+chr22	30120223	30120265	CCDS13897.1_cds_0_0_chr22_30120224_f	0	+
+chr22	30160419	30160661	CCDS13898.1_cds_0_0_chr22_30160420_r	0	-
+chr22	30665273	30665360	CCDS13901.1_cds_0_0_chr22_30665274_f	0	+
+chr22	30939054	30939266	CCDS13903.1_cds_0_0_chr22_30939055_r	0	-
+chr5	131424298	131424460	CCDS4149.1_cds_0_0_chr5_131424299_f	0	+
+chr5	131556601	131556672	CCDS4151.1_cds_0_0_chr5_131556602_r	0	-
+chr5	131621326	131621419	CCDS4152.1_cds_0_0_chr5_131621327_f	0	+
+chr5	131847541	131847666	CCDS4155.1_cds_0_0_chr5_131847542_r	0	-
+chr6	108299600	108299744	CCDS5061.1_cds_0_0_chr6_108299601_r	0	-
+chr6	108594662	108594687	CCDS5063.1_cds_0_0_chr6_108594663_f	0	+
+chr6	108640045	108640151	CCDS5064.1_cds_0_0_chr6_108640046_r	0	-
+chr6	108722976	108723115	CCDS5067.1_cds_0_0_chr6_108722977_f	0	+
+chr7	113660517	113660685	CCDS5760.1_cds_0_0_chr7_113660518_f	0	+
+chr7	116512159	116512389	CCDS5771.1_cds_0_0_chr7_116512160_r	0	-
+chr7	116714099	116714152	CCDS5773.1_cds_0_0_chr7_116714100_f	0	+
+chr7	116945541	116945787	CCDS5774.1_cds_0_0_chr7_116945542_r	0	-
+chr8	118881131	118881317	CCDS6324.1_cds_0_0_chr8_118881132_r	0	-
+chr9	128764156	128764189	CCDS6914.1_cds_0_0_chr9_128764157_f	0	+
+chr9	128787519	128789136	CCDS6915.1_cds_0_0_chr9_128787520_r	0	-
+chr9	128882427	128882523	CCDS6917.1_cds_0_0_chr9_128882428_f	0	+
+chr9	128937229	128937445	CCDS6919.1_cds_0_0_chr9_128937230_r	0	-
+chrX	122745047	122745924	CCDS14606.1_cds_0_0_chrX_122745048_f	0	+
+chrX	152648964	152649196	CCDS14733.1_cds_0_0_chrX_152648965_r	0	-
+chrX	152691446	152691471	CCDS14735.1_cds_0_0_chrX_152691447_f	0	+
+chrX	152694029	152694263	CCDS14736.1_cds_0_0_chrX_152694030_r	0	-
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/test-data/eq-showbeginning.dat	Tue Apr 01 09:12:49 2014 -0400
@@ -0,0 +1,10 @@
+chr1	147962192	147962580	CCDS989.1_cds_0_0_chr1_147962193_r	0	-
+chr1	147984545	147984630	CCDS990.1_cds_0_0_chr1_147984546_f	0	+
+chr1	148078400	148078582	CCDS993.1_cds_0_0_chr1_148078401_r	0	-
+chr1	148185136	148185276	CCDS996.1_cds_0_0_chr1_148185137_f	0	+
+chr10	55251623	55253124	CCDS7248.1_cds_0_0_chr10_55251624_r	0	-
+chr11	116124407	116124501	CCDS8374.1_cds_0_0_chr11_116124408_r	0	-
+chr11	116206508	116206563	CCDS8377.1_cds_0_0_chr11_116206509_f	0	+
+chr11	116211733	116212337	CCDS8378.1_cds_0_0_chr11_116211734_r	0	-
+chr11	1812377	1812407	CCDS7726.1_cds_0_0_chr11_1812378_f	0	+
+chr12	38440094	38440321	CCDS8736.1_cds_0_0_chr12_38440095_r	0	-
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/test-data/eq-showbeginning_e.dat	Tue Apr 01 09:12:49 2014 -0400
@@ -0,0 +1,10 @@
+chr1	147962192	147962580	CCDS989.1_cds_0_0_chr1_147962193_r	0	-
+chr1	147984545	147984630	CCDS990.1_cds_0_0_chr1_147984546_f	0	+		
+chr1	148078400	148078582	CCDS993.1_cds_0_0_chr1_148078401_r	0	-		
+chr1	148185136	148185276	CCDS996.1_cds_0_0_chr1_148185137_f	0	+
+chr10	55251623	55253124	CCDS7248.1_cds_0_0_chr10_55251624_r	0	-		
+chr11	116124407	116124501	CCDS8374.1_cds_0_0_chr11_116124408_r	0	-
+chr11	116206508	116206563	CCDS8377.1_cds_0_0_chr11_116206509_f	0	+		
+chr11	116211733	116212337	CCDS8378.1_cds_0_0_chr11_116211734_r	0	-
+chr11	1812377	1812407	CCDS7726.1_cds_0_0_chr11_1812378_f	0	+		
+chr12	38440094	38440321	CCDS8736.1_cds_0_0_chr12_38440095_r	0	-							
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/test-data/eq-showtail.dat	Tue Apr 01 09:12:49 2014 -0400
@@ -0,0 +1,10 @@
+chr7	116945541	116945787	CCDS5774.1_cds_0_0_chr7_116945542_r	0	-
+chr8	118881131	118881317	CCDS6324.1_cds_0_0_chr8_118881132_r	0	-
+chr9	128764156	128764189	CCDS6914.1_cds_0_0_chr9_128764157_f	0	+
+chr9	128787519	128789136	CCDS6915.1_cds_0_0_chr9_128787520_r	0	-
+chr9	128882427	128882523	CCDS6917.1_cds_0_0_chr9_128882428_f	0	+
+chr9	128937229	128937445	CCDS6919.1_cds_0_0_chr9_128937230_r	0	-
+chrX	122745047	122745924	CCDS14606.1_cds_0_0_chrX_122745048_f	0	+
+chrX	152648964	152649196	CCDS14733.1_cds_0_0_chrX_152648965_r	0	-
+chrX	152691446	152691471	CCDS14735.1_cds_0_0_chrX_152691447_f	0	+
+chrX	152694029	152694263	CCDS14736.1_cds_0_0_chrX_152694030_r	0	-
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/test-data/subtract-query-1.dat	Tue Apr 01 09:12:49 2014 -0400
@@ -0,0 +1,2 @@
+chr10   7       8
+chr10   2       10
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/test-data/subtract-query-2.dat	Tue Apr 01 09:12:49 2014 -0400
@@ -0,0 +1,10 @@
+chr11	1812377	1812407	CCDS7726.1_cds_0_0_chr11_1812378_f	0	+
+chr11	116124407	116124501	CCDS8374.1_cds_0_0_chr11_116124408_r	0	-
+chr1	147962192	147962580	CCDS989.1_cds_0_0_chr1_147962193_r	0	-
+chr1	148185136	148185276	CCDS996.1_cds_0_0_chr1_148185137_f	0	+
+chr12	38440094	38440321	CCDS8736.1_cds_0_0_chr12_38440095_r	0	-
+chr10	55251623	55253124	CCDS7248.1_cds_0_0_chr10_55251624_r	0	-
+chr11	116211733	116212337	CCDS8378.1_cds_0_0_chr11_116211734_r	0	-
+chr11	116206508	116206563	CCDS8377.1_cds_0_0_chr11_116206509_f	0	+
+chr1	148078400	148078582	CCDS993.1_cds_0_0_chr1_148078401_r	0	-
+chr1	147984545	147984630	CCDS990.1_cds_0_0_chr1_147984546_f	0	+
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/test-data/subtract-query-3.dat	Tue Apr 01 09:12:49 2014 -0400
@@ -0,0 +1,5 @@
+chr1	147984545	147984630
+chr10	55251623	55253124
+chr1	148185136	148185276
+chr1	147962192	147962580
+chr1	148078400	148078582
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/test-data/subtract-query-4.dat	Tue Apr 01 09:12:49 2014 -0400
@@ -0,0 +1,10 @@
+chr11	1812377	1812407	CCDS7726.1_cds_0_0_chr11_1812378_f	0	+
+chr11	116124407	116124501	CCDS8374.1_cds_0_0_chr11_116124408_r	0	-
+chr1	147962192	147962580	CCDS989.1_cds_0_0_chr1_147962193_r	0	-
+chr1	148185136	148185276	CCDS996.1_cds_0_0_chr1_148185137_f	0	+
+chr12	38440094	38440321	CCDS8736.1_cds_0_0_chr12_38440095_r	0	-
+chr10	55251623	55253124	CCDS7248.1_cds_0_0_chr10_55251624_r	0	-
+chr11	116211733	116212337	CCDS8378.1_cds_0_0_chr11_116211734_r	0	-
+chr11	116206508	116206563	CCDS8377.1_cds_0_0_chr11_116206509_f	0	+
+chr1	148078400	148078582	CCDS993.1_cds_0_0_chr1_148078401_r	0	-
+chr1	147984545	147984630	CCDS990.1_cds_0_0_chr1_147984546_f	0	+
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tool_dependencies.xml	Tue Apr 01 09:12:49 2014 -0400
@@ -0,0 +1,9 @@
+<?xml version="1.0"?>
+<tool_dependency>
+  <package name="bx-python" version="0.7.1">
+      <repository changeset_revision="cdb5991f9790" name="package_bx_python_0_7" owner="devteam" prior_installation_required="False" toolshed="http://testtoolshed.g2.bx.psu.edu" />
+    </package>
+    <package name="galaxy-ops" version="1.0.0">
+      <repository changeset_revision="3287c55c02b8" name="package_galaxy_ops_1_0_0" owner="devteam" prior_installation_required="False" toolshed="http://testtoolshed.g2.bx.psu.edu" />
+    </package>
+</tool_dependency>