# HG changeset patch # User devteam # Date 1396357969 14400 # Node ID 8a3448727cecae6b6ccac5ac543bde2bfcd1f68f Imported from capsule None diff -r 000000000000 -r 8a3448727cec subtract_query.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/subtract_query.py Tue Apr 01 09:12:49 2014 -0400 @@ -0,0 +1,111 @@ +#!/usr/bin/env python +# Greg Von Kuster + +""" +Subtract an entire query from another query +usage: %prog in_file_1 in_file_2 begin_col end_col output + --ignore-empty-end-cols: ignore empty end columns when subtracting +""" +import sys, re +from bx.cookbook import doc_optparse + +# Older py compatibility +try: + set() +except: + from sets import Set as set + +assert sys.version_info[:2] >= ( 2, 4 ) + +def get_lines(fname, begin_col='', end_col='', ignore_empty_end_cols=False): + lines = set([]) + i = 0 + for i, line in enumerate(file(fname)): + line = line.rstrip('\r\n') + if line and not line.startswith('#'): + if begin_col and end_col: + """Both begin_col and end_col must be integers at this point.""" + try: + line = line.split('\t') + line = '\t'.join([line[j] for j in range(begin_col-1, end_col)]) + if ignore_empty_end_cols: + # removing empty fields, we do not compare empty fields at the end of a line. + line = line.rstrip() + lines.add( line ) + except: pass + else: + if ignore_empty_end_cols: + # removing empty fields, we do not compare empty fields at the end of a line. + line = line.rstrip() + lines.add( line ) + if i: return (i+1, lines) + else: return (i, lines) + +def main(): + + # Parsing Command Line here + options, args = doc_optparse.parse( __doc__ ) + + try: + inp1_file, inp2_file, begin_col, end_col, out_file = args + except: + doc_optparse.exception() + + begin_col = begin_col.strip() + end_col = end_col.strip() + + if begin_col != 'None' or end_col != 'None': + """ + The user selected columns for restriction. We'll allow default + values for both begin_col and end_col as long as the user selected + at least one of them for restriction. + """ + if begin_col == 'None': + begin_col = end_col + elif end_col == 'None': + end_col = begin_col + begin_col = int(begin_col) + end_col = int(end_col) + """Make sure that begin_col <= end_col (switch if not)""" + if begin_col > end_col: + tmp_col = end_col + end_col = begin_col + begin_col = tmp_col + else: + begin_col = end_col = '' + + try: + fo = open(out_file,'w') + except: + print >> sys.stderr, "Unable to open output file" + sys.exit() + + """ + len1 is the number of lines in inp1_file + lines1 is the set of unique lines in inp1_file + diff1 is the number of duplicate lines removed from inp1_file + """ + len1, lines1 = get_lines(inp1_file, begin_col, end_col, options.ignore_empty_end_cols) + diff1 = len1 - len(lines1) + len2, lines2 = get_lines(inp2_file, begin_col, end_col, options.ignore_empty_end_cols) + + lines1.difference_update(lines2) + """lines1 is now the set of unique lines in inp1_file - the set of unique lines in inp2_file""" + + for line in lines1: + print >> fo, line + + fo.close() + + info_msg = 'Subtracted %d lines. ' %((len1 - diff1) - len(lines1)) + + if begin_col and end_col: + info_msg += 'Restricted to columns c' + str(begin_col) + ' thru c' + str(end_col) + '. ' + + if diff1 > 0: + info_msg += 'Eliminated %d duplicate/blank/comment/invalid lines from first query.' %diff1 + + print info_msg + +if __name__ == "__main__": + main() diff -r 000000000000 -r 8a3448727cec subtract_query.xml --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/subtract_query.xml Tue Apr 01 09:12:49 2014 -0400 @@ -0,0 +1,131 @@ + + from another dataset + + bx-python + galaxy-ops + + + subtract_query.py $input1 $input2 $begin_col $end_col $output + #if str($ignore_empty_end_cols) == 'true': + --ignore-empty-end-cols + #end if + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +.. class:: infomark + +**TIP:** This tool complements the tool in the **Operate on Genomic Intervals** tool set which subtracts the intervals of two datasets. + + +----- + +**Syntax** + +This tool subtracts an entire dataset from another dataset. + +- Any text format is valid. +- If both dataset formats are tabular, you may restrict the subtraction to specific columns **contained in both datasets** and the resulting dataset will include only the columns specified. +- The begin column must be less than or equal to the end column. If it is not, begin column is switched with end column. +- If begin column is specified but end column is not, end column will default to begin_column (and vice versa). +- All blank and comment lines are skipped and not included in the resulting dataset (comment lines are lines beginning with a # character). +- Duplicate lines are eliminated from both dataset prior to subtraction. If any duplicate lines were eliminated from the first dataset, the number is displayed in the resulting history item. + +----- + +**Example** + +If this is the **First dataset**:: + + chr1 4225 19670 + chr10 6 8 + chr1 24417 24420 + chr6_hla_hap2 0 150 + chr2 1 5 + chr10 2 10 + chr1 30 55 + chrY 1 20 + chr1 1225979 42287290 + chr10 7 8 + +and this is the **Second dataset**:: + + chr1 4225 19670 + chr10 6 8 + chr1 24417 24420 + chr6_hla_hap2 0 150 + chr2 1 5 + chr1 30 55 + chrY 1 20 + chr1 1225979 42287290 + +Subtracting the **Second dataset** from the **First dataset** (including all columns) will yield:: + + chr10 7 8 + chr10 2 10 + +Conversely, subtracting the **First dataset** from the **Second dataset** (including all columns) will result in an empty dataset. + +Subtracting the **Second dataset** from the **First dataset** (restricting to columns c1 and c2) will yield:: + + chr10 7 + chr10 2 + + + \ No newline at end of file diff -r 000000000000 -r 8a3448727cec test-data/2.txt --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/2.txt Tue Apr 01 09:12:49 2014 -0400 @@ -0,0 +1,8 @@ +chr1 4225 19670 +chr10 6 8 +chr1 24417 24420 +chr6_hla_hap2 0 150 +chr2 1 5 +chr1 30 55 +chrY 1 20 +chr1 1225979 42287290 diff -r 000000000000 -r 8a3448727cec test-data/eq-removebeginning.dat --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/eq-removebeginning.dat Tue Apr 01 09:12:49 2014 -0400 @@ -0,0 +1,60 @@ +chr11 116124407 116124501 CCDS8374.1_cds_0_0_chr11_116124408_r 0 - +chr11 116206508 116206563 CCDS8377.1_cds_0_0_chr11_116206509_f 0 + +chr11 116211733 116212337 CCDS8378.1_cds_0_0_chr11_116211734_r 0 - +chr11 1812377 1812407 CCDS7726.1_cds_0_0_chr11_1812378_f 0 + +chr12 38440094 38440321 CCDS8736.1_cds_0_0_chr12_38440095_r 0 - +chr13 112381694 112381953 CCDS9526.1_cds_0_0_chr13_112381695_f 0 + +chr14 98710240 98712285 CCDS9949.1_cds_0_0_chr14_98710241_r 0 - +chr15 41486872 41487060 CCDS10096.1_cds_0_0_chr15_41486873_r 0 - +chr15 41673708 41673857 CCDS10097.1_cds_0_0_chr15_41673709_f 0 + +chr15 41679161 41679250 CCDS10098.1_cds_0_0_chr15_41679162_r 0 - +chr15 41826029 41826196 CCDS10101.1_cds_0_0_chr15_41826030_f 0 + +chr16 142908 143003 CCDS10397.1_cds_0_0_chr16_142909_f 0 + +chr16 179963 180135 CCDS10401.1_cds_0_0_chr16_179964_r 0 - +chr16 244413 244681 CCDS10402.1_cds_0_0_chr16_244414_f 0 + +chr16 259268 259383 CCDS10403.1_cds_0_0_chr16_259269_r 0 - +chr18 23786114 23786321 CCDS11891.1_cds_0_0_chr18_23786115_r 0 - +chr18 59406881 59407046 CCDS11985.1_cds_0_0_chr18_59406882_f 0 + +chr18 59455932 59456337 CCDS11986.1_cds_0_0_chr18_59455933_r 0 - +chr18 59600586 59600754 CCDS11988.1_cds_0_0_chr18_59600587_f 0 + +chr19 59068595 59069564 CCDS12866.1_cds_0_0_chr19_59068596_f 0 + +chr19 59236026 59236146 CCDS12872.1_cds_0_0_chr19_59236027_r 0 - +chr19 59297998 59298008 CCDS12877.1_cds_0_0_chr19_59297999_f 0 + +chr19 59302168 59302288 CCDS12878.1_cds_0_0_chr19_59302169_r 0 - +chr2 118288583 118288668 CCDS2120.1_cds_0_0_chr2_118288584_f 0 + +chr2 118394148 118394202 CCDS2121.1_cds_0_0_chr2_118394149_r 0 - +chr2 220190202 220190242 CCDS2441.1_cds_0_0_chr2_220190203_f 0 + +chr2 220229609 220230869 CCDS2443.1_cds_0_0_chr2_220229610_r 0 - +chr20 33330413 33330423 CCDS13249.1_cds_0_0_chr20_33330414_r 0 - +chr20 33513606 33513792 CCDS13255.1_cds_0_0_chr20_33513607_f 0 + +chr20 33579500 33579527 CCDS13256.1_cds_0_0_chr20_33579501_r 0 - +chr20 33593260 33593348 CCDS13257.1_cds_0_0_chr20_33593261_f 0 + +chr21 32707032 32707192 CCDS13614.1_cds_0_0_chr21_32707033_f 0 + +chr21 32869641 32870022 CCDS13615.1_cds_0_0_chr21_32869642_r 0 - +chr21 33321040 33322012 CCDS13620.1_cds_0_0_chr21_33321041_f 0 + +chr21 33744994 33745040 CCDS13625.1_cds_0_0_chr21_33744995_r 0 - +chr22 30120223 30120265 CCDS13897.1_cds_0_0_chr22_30120224_f 0 + +chr22 30160419 30160661 CCDS13898.1_cds_0_0_chr22_30160420_r 0 - +chr22 30665273 30665360 CCDS13901.1_cds_0_0_chr22_30665274_f 0 + +chr22 30939054 30939266 CCDS13903.1_cds_0_0_chr22_30939055_r 0 - +chr5 131424298 131424460 CCDS4149.1_cds_0_0_chr5_131424299_f 0 + +chr5 131556601 131556672 CCDS4151.1_cds_0_0_chr5_131556602_r 0 - +chr5 131621326 131621419 CCDS4152.1_cds_0_0_chr5_131621327_f 0 + +chr5 131847541 131847666 CCDS4155.1_cds_0_0_chr5_131847542_r 0 - +chr6 108299600 108299744 CCDS5061.1_cds_0_0_chr6_108299601_r 0 - +chr6 108594662 108594687 CCDS5063.1_cds_0_0_chr6_108594663_f 0 + +chr6 108640045 108640151 CCDS5064.1_cds_0_0_chr6_108640046_r 0 - +chr6 108722976 108723115 CCDS5067.1_cds_0_0_chr6_108722977_f 0 + +chr7 113660517 113660685 CCDS5760.1_cds_0_0_chr7_113660518_f 0 + +chr7 116512159 116512389 CCDS5771.1_cds_0_0_chr7_116512160_r 0 - +chr7 116714099 116714152 CCDS5773.1_cds_0_0_chr7_116714100_f 0 + +chr7 116945541 116945787 CCDS5774.1_cds_0_0_chr7_116945542_r 0 - +chr8 118881131 118881317 CCDS6324.1_cds_0_0_chr8_118881132_r 0 - +chr9 128764156 128764189 CCDS6914.1_cds_0_0_chr9_128764157_f 0 + +chr9 128787519 128789136 CCDS6915.1_cds_0_0_chr9_128787520_r 0 - +chr9 128882427 128882523 CCDS6917.1_cds_0_0_chr9_128882428_f 0 + +chr9 128937229 128937445 CCDS6919.1_cds_0_0_chr9_128937230_r 0 - +chrX 122745047 122745924 CCDS14606.1_cds_0_0_chrX_122745048_f 0 + +chrX 152648964 152649196 CCDS14733.1_cds_0_0_chrX_152648965_r 0 - +chrX 152691446 152691471 CCDS14735.1_cds_0_0_chrX_152691447_f 0 + +chrX 152694029 152694263 CCDS14736.1_cds_0_0_chrX_152694030_r 0 - diff -r 000000000000 -r 8a3448727cec test-data/eq-showbeginning.dat --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/eq-showbeginning.dat Tue Apr 01 09:12:49 2014 -0400 @@ -0,0 +1,10 @@ +chr1 147962192 147962580 CCDS989.1_cds_0_0_chr1_147962193_r 0 - +chr1 147984545 147984630 CCDS990.1_cds_0_0_chr1_147984546_f 0 + +chr1 148078400 148078582 CCDS993.1_cds_0_0_chr1_148078401_r 0 - +chr1 148185136 148185276 CCDS996.1_cds_0_0_chr1_148185137_f 0 + +chr10 55251623 55253124 CCDS7248.1_cds_0_0_chr10_55251624_r 0 - +chr11 116124407 116124501 CCDS8374.1_cds_0_0_chr11_116124408_r 0 - +chr11 116206508 116206563 CCDS8377.1_cds_0_0_chr11_116206509_f 0 + +chr11 116211733 116212337 CCDS8378.1_cds_0_0_chr11_116211734_r 0 - +chr11 1812377 1812407 CCDS7726.1_cds_0_0_chr11_1812378_f 0 + +chr12 38440094 38440321 CCDS8736.1_cds_0_0_chr12_38440095_r 0 - diff -r 000000000000 -r 8a3448727cec test-data/eq-showbeginning_e.dat --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/eq-showbeginning_e.dat Tue Apr 01 09:12:49 2014 -0400 @@ -0,0 +1,10 @@ +chr1 147962192 147962580 CCDS989.1_cds_0_0_chr1_147962193_r 0 - +chr1 147984545 147984630 CCDS990.1_cds_0_0_chr1_147984546_f 0 + +chr1 148078400 148078582 CCDS993.1_cds_0_0_chr1_148078401_r 0 - +chr1 148185136 148185276 CCDS996.1_cds_0_0_chr1_148185137_f 0 + +chr10 55251623 55253124 CCDS7248.1_cds_0_0_chr10_55251624_r 0 - +chr11 116124407 116124501 CCDS8374.1_cds_0_0_chr11_116124408_r 0 - +chr11 116206508 116206563 CCDS8377.1_cds_0_0_chr11_116206509_f 0 + +chr11 116211733 116212337 CCDS8378.1_cds_0_0_chr11_116211734_r 0 - +chr11 1812377 1812407 CCDS7726.1_cds_0_0_chr11_1812378_f 0 + +chr12 38440094 38440321 CCDS8736.1_cds_0_0_chr12_38440095_r 0 - diff -r 000000000000 -r 8a3448727cec test-data/eq-showtail.dat --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/eq-showtail.dat Tue Apr 01 09:12:49 2014 -0400 @@ -0,0 +1,10 @@ +chr7 116945541 116945787 CCDS5774.1_cds_0_0_chr7_116945542_r 0 - +chr8 118881131 118881317 CCDS6324.1_cds_0_0_chr8_118881132_r 0 - +chr9 128764156 128764189 CCDS6914.1_cds_0_0_chr9_128764157_f 0 + +chr9 128787519 128789136 CCDS6915.1_cds_0_0_chr9_128787520_r 0 - +chr9 128882427 128882523 CCDS6917.1_cds_0_0_chr9_128882428_f 0 + +chr9 128937229 128937445 CCDS6919.1_cds_0_0_chr9_128937230_r 0 - +chrX 122745047 122745924 CCDS14606.1_cds_0_0_chrX_122745048_f 0 + +chrX 152648964 152649196 CCDS14733.1_cds_0_0_chrX_152648965_r 0 - +chrX 152691446 152691471 CCDS14735.1_cds_0_0_chrX_152691447_f 0 + +chrX 152694029 152694263 CCDS14736.1_cds_0_0_chrX_152694030_r 0 - diff -r 000000000000 -r 8a3448727cec test-data/subtract-query-1.dat --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/subtract-query-1.dat Tue Apr 01 09:12:49 2014 -0400 @@ -0,0 +1,2 @@ +chr10 7 8 +chr10 2 10 diff -r 000000000000 -r 8a3448727cec test-data/subtract-query-2.dat --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/subtract-query-2.dat Tue Apr 01 09:12:49 2014 -0400 @@ -0,0 +1,10 @@ +chr11 1812377 1812407 CCDS7726.1_cds_0_0_chr11_1812378_f 0 + +chr11 116124407 116124501 CCDS8374.1_cds_0_0_chr11_116124408_r 0 - +chr1 147962192 147962580 CCDS989.1_cds_0_0_chr1_147962193_r 0 - +chr1 148185136 148185276 CCDS996.1_cds_0_0_chr1_148185137_f 0 + +chr12 38440094 38440321 CCDS8736.1_cds_0_0_chr12_38440095_r 0 - +chr10 55251623 55253124 CCDS7248.1_cds_0_0_chr10_55251624_r 0 - +chr11 116211733 116212337 CCDS8378.1_cds_0_0_chr11_116211734_r 0 - +chr11 116206508 116206563 CCDS8377.1_cds_0_0_chr11_116206509_f 0 + +chr1 148078400 148078582 CCDS993.1_cds_0_0_chr1_148078401_r 0 - +chr1 147984545 147984630 CCDS990.1_cds_0_0_chr1_147984546_f 0 + diff -r 000000000000 -r 8a3448727cec test-data/subtract-query-3.dat --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/subtract-query-3.dat Tue Apr 01 09:12:49 2014 -0400 @@ -0,0 +1,5 @@ +chr1 147984545 147984630 +chr10 55251623 55253124 +chr1 148185136 148185276 +chr1 147962192 147962580 +chr1 148078400 148078582 diff -r 000000000000 -r 8a3448727cec test-data/subtract-query-4.dat --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/subtract-query-4.dat Tue Apr 01 09:12:49 2014 -0400 @@ -0,0 +1,10 @@ +chr11 1812377 1812407 CCDS7726.1_cds_0_0_chr11_1812378_f 0 + +chr11 116124407 116124501 CCDS8374.1_cds_0_0_chr11_116124408_r 0 - +chr1 147962192 147962580 CCDS989.1_cds_0_0_chr1_147962193_r 0 - +chr1 148185136 148185276 CCDS996.1_cds_0_0_chr1_148185137_f 0 + +chr12 38440094 38440321 CCDS8736.1_cds_0_0_chr12_38440095_r 0 - +chr10 55251623 55253124 CCDS7248.1_cds_0_0_chr10_55251624_r 0 - +chr11 116211733 116212337 CCDS8378.1_cds_0_0_chr11_116211734_r 0 - +chr11 116206508 116206563 CCDS8377.1_cds_0_0_chr11_116206509_f 0 + +chr1 148078400 148078582 CCDS993.1_cds_0_0_chr1_148078401_r 0 - +chr1 147984545 147984630 CCDS990.1_cds_0_0_chr1_147984546_f 0 + diff -r 000000000000 -r 8a3448727cec tool_dependencies.xml --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tool_dependencies.xml Tue Apr 01 09:12:49 2014 -0400 @@ -0,0 +1,9 @@ + + + + + + + + +