|
0
|
1 import bisect
|
|
|
2 import csv
|
|
|
3 import os
|
|
|
4 import sys
|
|
|
5 import traceback
|
|
1
|
6 import matplotlib
|
|
|
7 matplotlib.use('Agg')
|
|
0
|
8 from matplotlib import pyplot
|
|
|
9
|
|
|
10 DETAILS = 'D'
|
|
|
11 FINAL_PLOTS = 'F'
|
|
|
12 ORPHANS = 'O'
|
|
|
13 PREVIEW_PLOTS = 'P'
|
|
|
14 SIMPLES = 'S'
|
|
|
15 STATS_GRAPH = 'C'
|
|
|
16 GFF_EXT = 'gff'
|
|
|
17 TABULAR_EXT = 'tabular'
|
|
|
18
|
|
|
19 # Graph settings.
|
|
|
20 COLORS = 'krg'
|
|
|
21 Y_LABEL = 'Peak-pair counts'
|
|
|
22 X_LABEL = 'Peak-pair distance (bp)'
|
|
|
23 TICK_WIDTH = 3
|
|
|
24 ADJUST = [0.140, 0.9, 0.9, 0.1]
|
|
2
|
25 PLOT_FORMAT = 'pdf'
|
|
0
|
26 pyplot.rc('xtick.major', size=10.00)
|
|
|
27 pyplot.rc('ytick.major', size=10.00)
|
|
|
28 pyplot.rc('lines', linewidth=4.00)
|
|
|
29 pyplot.rc('axes', linewidth=3.00)
|
|
1
|
30 pyplot.rc('font', family='Bitstream Vera Sans', size=32.0)
|
|
0
|
31
|
|
|
32
|
|
|
33 class FrequencyDistribution(object):
|
|
|
34
|
|
|
35 def __init__(self, start, end, binsize=10, d=None):
|
|
|
36 self.start = start
|
|
|
37 self.end = end
|
|
|
38 self.dist = d or {}
|
|
|
39 self.binsize = binsize
|
|
|
40
|
|
|
41 def get_bin(self, x):
|
|
|
42 """
|
|
|
43 Returns the bin in which a data point falls
|
|
|
44 """
|
|
|
45 return self.start + (x-self.start) // self.binsize * self.binsize + self.binsize/2.0
|
|
|
46
|
|
|
47 def add(self, x):
|
|
|
48 x = self.get_bin(x)
|
|
|
49 self.dist[x] = self.dist.get(x, 0) + 1
|
|
|
50
|
|
|
51 def graph_series(self):
|
|
|
52 x = []
|
|
|
53 y = []
|
|
|
54 for i in range(self.start, self.end, self.binsize):
|
|
|
55 center = self.get_bin(i)
|
|
|
56 x.append(center)
|
|
|
57 y.append(self.dist.get(center, 0))
|
|
|
58 return x, y
|
|
|
59
|
|
|
60 def mode(self):
|
|
|
61 return max(self.dist.items(), key=lambda data: data[1])[0]
|
|
|
62
|
|
|
63 def size(self):
|
|
|
64 return sum(self.dist.values())
|
|
|
65
|
|
|
66
|
|
|
67 def stop_err(msg):
|
|
|
68 sys.stderr.write(msg)
|
|
|
69 sys.exit(1)
|
|
|
70
|
|
|
71
|
|
|
72 def distance(peak1, peak2):
|
|
|
73 return (peak2[1]+peak2[2])/2 - (peak1[1]+peak1[2])/2
|
|
|
74
|
|
|
75
|
|
|
76 def gff_row(cname, start, end, score, source, type='.', strand='.', phase='.', attrs={}):
|
|
|
77 return (cname, source, type, start, end, score, strand, phase, gff_attrs(attrs))
|
|
|
78
|
|
|
79
|
|
|
80 def gff_attrs(d):
|
|
|
81 if not d:
|
|
|
82 return '.'
|
|
|
83 return ';'.join('%s=%s' % item for item in d.items())
|
|
|
84
|
|
|
85
|
|
|
86 def parse_chromosomes(reader):
|
|
|
87 # This version of cwpair2 accepts only gff format as input.
|
|
|
88 chromosomes = {}
|
|
|
89 reader.next()
|
|
|
90 for line in reader:
|
|
|
91 cname, junk, junk, start, end, value, strand, junk, junk = line
|
|
|
92 start = int(start)
|
|
|
93 end = int(end)
|
|
|
94 value = float(value)
|
|
|
95 if cname not in chromosomes:
|
|
|
96 chromosomes[cname] = []
|
|
|
97 peaks = chromosomes[cname]
|
|
|
98 peaks.append((strand, start, end, value))
|
|
|
99 return chromosomes
|
|
|
100
|
|
|
101
|
|
|
102 def perc95(chromosomes):
|
|
|
103 """
|
|
|
104 Returns the 95th percentile value of the given chromosomes.
|
|
|
105 """
|
|
|
106 values = []
|
|
|
107 for peaks in chromosomes.values():
|
|
|
108 for peak in peaks:
|
|
|
109 values.append(peak[3])
|
|
|
110 values.sort()
|
|
|
111 # Get 95% value
|
|
|
112 return values[int(len(values)*0.95)]
|
|
|
113
|
|
|
114
|
|
|
115 def filter(chromosomes, threshold=0.05):
|
|
|
116 """
|
|
|
117 Filters the peaks to those above a threshold. Threshold < 1.0 is interpreted
|
|
|
118 as a proportion of the maximum, >=1.0 as an absolute value.
|
|
|
119 """
|
|
|
120 if threshold < 1:
|
|
|
121 p95 = perc95(chromosomes)
|
|
|
122 threshold = p95 * threshold
|
|
|
123 # Make the threshold a proportion of the
|
|
|
124 for cname, peaks in chromosomes.items():
|
|
|
125 chromosomes[cname] = [peak for peak in peaks if peak[3] > threshold]
|
|
|
126
|
|
|
127
|
|
|
128 def split_strands(chromosome):
|
|
|
129 watson = [peak for peak in chromosome if peak[0] == '+']
|
|
|
130 crick = [peak for peak in chromosome if peak[0] == '-']
|
|
|
131 return watson, crick
|
|
|
132
|
|
|
133
|
|
|
134 def all_pair_distribution(chromosomes, up_distance, down_distance, binsize):
|
|
|
135 dist = FrequencyDistribution(-up_distance, down_distance, binsize=binsize)
|
|
|
136 for cname, data in chromosomes.items():
|
|
|
137 watson, crick = split_strands(data)
|
|
|
138 crick.sort(key=lambda data: float(data[1]))
|
|
|
139 keys = make_keys(crick)
|
|
|
140 for peak in watson:
|
|
|
141 for cpeak in get_window(crick, peak, up_distance, down_distance, keys):
|
|
|
142 dist.add(distance(peak, cpeak))
|
|
|
143 return dist
|
|
|
144
|
|
|
145
|
|
|
146 def make_keys(crick):
|
|
|
147 return [(data[1] + data[2])//2 for data in crick]
|
|
|
148
|
|
|
149
|
|
|
150 def get_window(crick, peak, up_distance, down_distance, keys=None):
|
|
|
151 """
|
|
|
152 Returns a window of all crick peaks within a distance of a watson peak.
|
|
|
153 crick strand MUST be sorted by distance
|
|
|
154 """
|
|
|
155 strand, start, end, value = peak
|
|
|
156 midpoint = (start + end) // 2
|
|
|
157 lower = midpoint - up_distance
|
|
|
158 upper = midpoint + down_distance
|
|
|
159 keys = keys or make_keys(crick)
|
|
|
160 start_index = bisect.bisect_left(keys, lower)
|
|
|
161 end_index = bisect.bisect_right(keys, upper)
|
|
|
162 return [cpeak for cpeak in crick[start_index:end_index]]
|
|
|
163
|
|
|
164
|
|
|
165 def match_largest(window, peak):
|
|
|
166 if not window:
|
|
|
167 return None
|
|
|
168 return max(window, key=lambda cpeak: cpeak[3])
|
|
|
169
|
|
|
170
|
|
|
171 def match_closest(window, peak):
|
|
|
172 if not window:
|
|
|
173 return None
|
|
|
174
|
|
|
175 def key(cpeak):
|
|
|
176 d = distance(peak, cpeak)
|
|
|
177 # Search negative distances last
|
|
|
178 if d < 0:
|
|
|
179 # And then prefer less negative distances
|
|
|
180 d = 10000 - d
|
|
|
181 return d
|
|
|
182 return min(window, key=key)
|
|
|
183
|
|
|
184
|
|
|
185 def match_mode(window, peak, mode):
|
|
|
186 if not window:
|
|
|
187 return None
|
|
|
188 return min(window, key=lambda cpeak: abs(distance(peak, cpeak)-mode))
|
|
|
189
|
|
|
190 METHODS = {'mode': match_mode, 'closest': match_closest, 'largest': match_largest}
|
|
|
191
|
|
|
192
|
|
|
193 def frequency_plot(freqs, fname, labels=[], title=''):
|
|
|
194 pyplot.clf()
|
|
|
195 pyplot.figure(figsize=(10, 10))
|
|
|
196 for i, freq in enumerate(freqs):
|
|
|
197 x, y = freq.graph_series()
|
|
|
198 pyplot.plot(x, y, '%s-' % COLORS[i])
|
|
|
199 if len(freqs) > 1:
|
|
|
200 pyplot.legend(labels)
|
|
|
201 pyplot.xlim(freq.start, freq.end)
|
|
|
202 pyplot.ylim(ymin=0)
|
|
|
203 pyplot.ylabel(Y_LABEL)
|
|
|
204 pyplot.xlabel(X_LABEL)
|
|
|
205 pyplot.subplots_adjust(left=ADJUST[0], right=ADJUST[1], top=ADJUST[2], bottom=ADJUST[3])
|
|
|
206 # Get the current axes
|
|
|
207 ax = pyplot.gca()
|
|
|
208 for l in ax.get_xticklines() + ax.get_yticklines():
|
|
|
209 l.set_markeredgewidth(TICK_WIDTH)
|
|
|
210 pyplot.savefig(fname)
|
|
|
211
|
|
|
212
|
|
|
213 def create_directories(method):
|
|
|
214 if method == 'all':
|
|
|
215 match_methods = METHODS.keys()
|
|
|
216 else:
|
|
|
217 match_methods = [method]
|
|
|
218 for match_method in match_methods:
|
|
|
219 os.mkdir('%s_%s' % (match_method, DETAILS))
|
|
|
220 os.mkdir('%s_%s' % (match_method, FINAL_PLOTS))
|
|
|
221 os.mkdir('%s_%s' % (match_method, ORPHANS))
|
|
|
222 os.mkdir('%s_%s' % (match_method, PREVIEW_PLOTS))
|
|
|
223 os.mkdir('%s_%s' % (match_method, SIMPLES))
|
|
|
224 os.mkdir('%s_%s' % (match_method, STATS_GRAPH))
|
|
|
225
|
|
|
226
|
|
2
|
227 def process_file(dataset_path, galaxy_hid, method, threshold, up_distance,
|
|
5
|
228 down_distance, binsize, output_files, sort_score):
|
|
0
|
229 if method == 'all':
|
|
|
230 match_methods = METHODS.keys()
|
|
|
231 else:
|
|
|
232 match_methods = [method]
|
|
|
233 statistics = []
|
|
|
234 for match_method in match_methods:
|
|
|
235 stats = perform_process(dataset_path,
|
|
|
236 galaxy_hid,
|
|
|
237 match_method,
|
|
|
238 threshold,
|
|
|
239 up_distance,
|
|
|
240 down_distance,
|
|
|
241 binsize,
|
|
|
242 output_files,
|
|
|
243 sort_score)
|
|
|
244 statistics.append(stats)
|
|
|
245 if output_files == 'all' and method == 'all':
|
|
|
246 frequency_plot([s['dist'] for s in statistics],
|
|
|
247 statistics[0]['graph_path'],
|
|
|
248 labels=METHODS.keys())
|
|
|
249 return statistics
|
|
|
250
|
|
|
251
|
|
|
252 def perform_process(dataset_path, galaxy_hid, method, threshold, up_distance,
|
|
5
|
253 down_distance, binsize, output_files, sort_score):
|
|
0
|
254 output_details = output_files in ["all", "simple_orphan_detail"]
|
|
|
255 output_plots = output_files in ["all"]
|
|
|
256 output_orphans = output_files in ["all", "simple_orphan", "simple_orphan_detail"]
|
|
|
257 # Keep track of statistics for the output file
|
|
|
258 statistics = {}
|
|
|
259 input = csv.reader(open(dataset_path, 'rt'), delimiter='\t')
|
|
|
260 fpath, fname = os.path.split(dataset_path)
|
|
|
261 statistics['fname'] = '%s: data %s' % (method, str(galaxy_hid))
|
|
|
262 statistics['dir'] = fpath
|
|
|
263 if threshold >= 1:
|
|
|
264 filter_string = 'fa%d' % threshold
|
|
|
265 else:
|
|
|
266 filter_string = 'f%d' % (threshold * 100)
|
|
|
267 fname = 'data_%s_%su%dd%db%d' % (galaxy_hid, filter_string, up_distance, down_distance, binsize)
|
|
|
268
|
|
|
269 def make_path(output_type, extension=TABULAR_EXT):
|
|
|
270 # Returns the full path for a certain output.
|
|
|
271 return os.path.join(output_type, '%s_%s.%s' % (output_type, fname, extension))
|
|
|
272
|
|
|
273 def td_writer(output_type, extension=TABULAR_EXT):
|
|
|
274 # Returns a tab-delimited writer for a specified output.
|
|
|
275 output_file_path = make_path(output_type, extension)
|
|
|
276 return csv.writer(open(output_file_path, 'wt'), delimiter='\t')
|
|
|
277
|
|
|
278 try:
|
|
|
279 chromosomes = parse_chromosomes(input)
|
|
|
280 except Exception:
|
|
|
281 stop_err('Unable to parse file "%s".\n%s' % (dataset_path, traceback.format_exc()))
|
|
|
282 if output_details:
|
|
|
283 # Details
|
|
|
284 detailed_output = td_writer('%s_%s' % (method, DETAILS), extension=TABULAR_EXT)
|
|
|
285 detailed_output.writerow(('chrom', 'start', 'end', 'value', 'strand') * 2 + ('midpoint', 'c-w reads sum', 'c-w distance (bp)'))
|
|
|
286 if output_plots:
|
|
|
287 # Final Plot
|
|
2
|
288 final_plot_path = make_path('%s_%s' % (method, FINAL_PLOTS), PLOT_FORMAT)
|
|
0
|
289 if output_orphans:
|
|
|
290 # Orphans
|
|
|
291 orphan_output = td_writer('%s_%s' % (method, ORPHANS), extension=TABULAR_EXT)
|
|
|
292 orphan_output.writerow(('chrom', 'strand', 'start', 'end', 'value'))
|
|
|
293 if output_plots:
|
|
|
294 # Preview Plot
|
|
2
|
295 preview_plot_path = make_path('%s_%s' % (method, PREVIEW_PLOTS), PLOT_FORMAT)
|
|
0
|
296 # Simple
|
|
|
297 simple_output = td_writer('%s_%s' % (method, SIMPLES), extension=GFF_EXT)
|
|
|
298 statistics['stats_path'] = 'statistics.%s' % TABULAR_EXT
|
|
|
299 if output_plots:
|
|
2
|
300 statistics['graph_path'] = make_path('%s_%s' % (method, STATS_GRAPH), PLOT_FORMAT)
|
|
0
|
301 statistics['perc95'] = perc95(chromosomes)
|
|
|
302 if threshold > 0:
|
|
|
303 # Apply filter
|
|
|
304 filter(chromosomes, threshold)
|
|
|
305 if method == 'mode':
|
|
|
306 freq = all_pair_distribution(chromosomes, up_distance, down_distance, binsize)
|
|
|
307 mode = freq.mode()
|
|
|
308 statistics['preview_mode'] = mode
|
|
|
309 if output_plots:
|
|
|
310 frequency_plot([freq], preview_plot_path, title='Preview frequency plot')
|
|
|
311 else:
|
|
|
312 statistics['preview_mode'] = 'NA'
|
|
|
313 dist = FrequencyDistribution(-up_distance, down_distance, binsize=binsize)
|
|
|
314 orphans = 0
|
|
|
315 # x will be used to archive the summary dataset
|
|
|
316 x = []
|
|
|
317 for cname, chromosome in chromosomes.items():
|
|
|
318 # Each peak is (strand, start, end, value)
|
|
|
319 watson, crick = split_strands(chromosome)
|
|
|
320 # Sort by value of each peak
|
|
|
321 watson.sort(key=lambda data: -float(data[3]))
|
|
|
322 # Sort by position to facilitate binary search
|
|
|
323 crick.sort(key=lambda data: float(data[1]))
|
|
|
324 keys = make_keys(crick)
|
|
|
325 for peak in watson:
|
|
|
326 window = get_window(crick, peak, up_distance, down_distance, keys)
|
|
|
327 if method == 'mode':
|
|
|
328 match = match_mode(window, peak, mode)
|
|
|
329 else:
|
|
|
330 match = METHODS[method](window, peak)
|
|
|
331 if match:
|
|
|
332 midpoint = (match[1] + match[2] + peak[1] + peak[2]) // 4
|
|
|
333 d = distance(peak, match)
|
|
|
334 dist.add(d)
|
|
|
335 # Simple output in gff format.
|
|
|
336 x.append(gff_row(cname,
|
|
|
337 source='cwpair',
|
|
|
338 start=midpoint,
|
|
|
339 end=midpoint + 1,
|
|
|
340 score=peak[3] + match[3],
|
|
|
341 attrs={'cw_distance': d}))
|
|
|
342 if output_details:
|
|
|
343 detailed_output.writerow((cname,
|
|
|
344 peak[1],
|
|
|
345 peak[2],
|
|
|
346 peak[3],
|
|
|
347 '+',
|
|
|
348 cname,
|
|
|
349 match[1],
|
|
|
350 match[2],
|
|
|
351 match[3], '-',
|
|
|
352 midpoint,
|
|
|
353 peak[3]+match[3],
|
|
|
354 d))
|
|
|
355 i = bisect.bisect_left(keys, (match[1]+match[2])/2)
|
|
|
356 del crick[i]
|
|
|
357 del keys[i]
|
|
|
358 else:
|
|
|
359 if output_orphans:
|
|
|
360 orphan_output.writerow((cname, peak[0], peak[1], peak[2], peak[3]))
|
|
|
361 # Keep track of orphans for statistics.
|
|
|
362 orphans += 1
|
|
|
363 # Remaining crick peaks are orphans
|
|
|
364 if output_orphans:
|
|
|
365 for cpeak in crick:
|
|
|
366 orphan_output.writerow((cname, cpeak[0], cpeak[1], cpeak[2], cpeak[3]))
|
|
|
367 # Keep track of orphans for statistics.
|
|
|
368 orphans += len(crick)
|
|
|
369 # Sort output by score if specified.
|
|
|
370 if sort_score == "desc":
|
|
|
371 x.sort(key=lambda data: float(data[5]), reverse=True)
|
|
|
372 elif sort_score == "asc":
|
|
|
373 x.sort(key=lambda data: float(data[5]))
|
|
5
|
374 # Writing a summary to gff format file
|
|
0
|
375 for row in x:
|
|
|
376 row_tmp = list(row)
|
|
|
377 # Dataset in tuple cannot be modified in Python, so row will
|
|
|
378 # be converted to list format to add 'chr'.
|
|
|
379 if row_tmp[0] == "999":
|
|
|
380 row_tmp[0] = 'chrM'
|
|
|
381 elif row_tmp[0] == "998":
|
|
|
382 row_tmp[0] = 'chrY'
|
|
|
383 elif row_tmp[0] == "997":
|
|
|
384 row_tmp[0] = 'chrX'
|
|
|
385 else:
|
|
|
386 row_tmp[0] = row_tmp[0]
|
|
|
387 # Print row_tmp.
|
|
|
388 simple_output.writerow(row_tmp)
|
|
|
389 statistics['paired'] = dist.size() * 2
|
|
|
390 statistics['orphans'] = orphans
|
|
|
391 statistics['final_mode'] = dist.mode()
|
|
|
392 if output_plots:
|
|
|
393 frequency_plot([dist], final_plot_path, title='Frequency distribution')
|
|
|
394 statistics['dist'] = dist
|
|
|
395 return statistics
|
