|
0
|
1 #!/usr/bin/env python
|
|
8
|
2 # Jeremy Liu
|
|
|
3 # February 2015
|
|
|
4 # Adapted from Dan Blackenburg's sample data manager
|
|
0
|
5
|
|
|
6 import sys
|
|
|
7 import os
|
|
|
8 import tempfile
|
|
|
9 import shutil
|
|
|
10 import optparse
|
|
|
11 import urllib2
|
|
|
12 #import uuid
|
|
|
13 from ftplib import FTP
|
|
|
14 import tarfile
|
|
|
15 import zipfile
|
|
|
16 import gzip
|
|
|
17 import bz2
|
|
|
18
|
|
|
19 from galaxy.util.json import from_json_string, to_json_string
|
|
|
20
|
|
|
21 CHUNK_SIZE = 2**20 #1mb
|
|
|
22
|
|
|
23 def download_motif_databases( data_manager_dict, params, target_directory, motif_db ):
|
|
|
24
|
|
8
|
25 # Select download URL, file name, data table name, and path using motif_db selector variable
|
|
6
|
26 if motif_db == "pouya":
|
|
8
|
27 BGZ = ['http://compbio.med.harvard.edu/motif-enrichment/pouya_motifs_bed.bgz',
|
|
6
|
28 "pouya_motifs.bed.bgz", "pouya_bgz", "Pouya Encode Motifs (hg19) BGZ"]
|
|
8
|
29 TBI = ['http://compbio.med.harvard.edu/motif-enrichment/pouya_motifs_bed.bgz.tbi',
|
|
6
|
30 "pouya_motifs.bed.bgz.tbi", "pouya_tbi", "Pouya Encode Motifs (hg19) TBI"]
|
|
|
31 elif motif_db == "jaspar":
|
|
8
|
32 BGZ = ['http://compbio.med.harvard.edu/motif-enrichment/jaspar_jolma_motifs.bed.bgz',
|
|
|
33 "jaspar_jolma_motifs.bed.bgz", "jaspar_bgz", "Jaspar and Jolma Motifs (hg19) BGZ"]
|
|
|
34 TBI = ['http://compbio.med.harvard.edu/motif-enrichment/jaspar_jolma_motifs.bed.bgz.tbi',
|
|
6
|
35 "jaspar_jolma_motifs.bed.bgz.tbi", "jaspar_tbi", "Jaspar and Jolma Motifs (hg19) TBI"]
|
|
|
36 elif motif_db == "mouse":
|
|
8
|
37 BGZ = ['http://compbio.med.harvard.edu/motif-enrichment/mm9_motifs_split.bed.bgz',
|
|
|
38 "mm9_motifs_split.bed.bgz", "mouse_bgz", "Mouse Motifs (mm9) BGZ"]
|
|
|
39 TBI = ['http://compbio.med.harvard.edu/motif-enrichment/mm9_motifs_split.bed.bgz.tbi',
|
|
|
40 "mm9_motifs_split.bed.bgz.tbi", "mouse_tbi", "Mouse Motifs (mm9) TBI"]
|
|
6
|
41 else:
|
|
8
|
42 BGZ = ['http://compbio.med.harvard.edu/motif-enrichment/pouya_test_motifs.bed.bgz',
|
|
6
|
43 "pouya_test_motifs.bed.bgz", "test_bgz", "Test Pouya Subset (hg19) BGZ"]
|
|
8
|
44 TBI = ['http://compbio.med.harvard.edu/motif-enrichment/pouya_test_motifs.bed.bgz.tbi',
|
|
6
|
45 "pouya_test_motifs.bed.bgz.tbi", "test_tbi", "Test Pouya Subset (hg19) TBI"]
|
|
|
46
|
|
8
|
47 # Save and add motif bgz file to motif_databases data table
|
|
6
|
48 bgz_reader = urllib2.urlopen( BGZ[0] )
|
|
4
|
49 bgz_data_table_entry = _stream_fasta_to_file( bgz_reader, target_directory, params,
|
|
6
|
50 BGZ[1], BGZ[2], BGZ[3] )
|
|
4
|
51 _add_data_table_entry( data_manager_dict, 'motif_databases', bgz_data_table_entry )
|
|
|
52
|
|
8
|
53 # Save and add motif tbi file to motif_databases data table
|
|
6
|
54 tbi_reader = urllib2.urlopen( TBI[0] )
|
|
4
|
55 tbi_data_table_entry = _stream_fasta_to_file( tbi_reader, target_directory, params,
|
|
6
|
56 TBI[1], TBI[2], TBI[3] )
|
|
4
|
57 _add_data_table_entry( data_manager_dict, 'motif_databases', tbi_data_table_entry )
|
|
0
|
58
|
|
2
|
59 def _add_data_table_entry( data_manager_dict, data_table, data_table_entry ):
|
|
0
|
60 data_manager_dict['data_tables'] = data_manager_dict.get( 'data_tables', {} )
|
|
2
|
61 data_manager_dict['data_tables'][data_table] = data_manager_dict['data_tables'].get( data_table, [] )
|
|
|
62 data_manager_dict['data_tables'][data_table].append( data_table_entry )
|
|
0
|
63 return data_manager_dict
|
|
|
64
|
|
5
|
65 def _stream_fasta_to_file( fasta_stream, target_directory, params,
|
|
|
66 fasta_base_filename, value, name, close_stream=True ):
|
|
0
|
67 fasta_filename = os.path.join( target_directory, fasta_base_filename )
|
|
|
68 fasta_writer = open( fasta_filename, 'wb+' )
|
|
|
69
|
|
|
70 while True:
|
|
|
71 buffer = fasta_stream.read(CHUNK_SIZE)
|
|
|
72 if not buffer:
|
|
|
73 break
|
|
|
74
|
|
|
75 fasta_writer.write(buffer)
|
|
|
76
|
|
|
77 fasta_stream.close()
|
|
|
78 fasta_writer.close()
|
|
|
79
|
|
4
|
80 return dict( value=value, name=name, path=fasta_base_filename )
|
|
0
|
81
|
|
|
82 def main():
|
|
|
83 #Parse Command Line
|
|
|
84 parser = optparse.OptionParser()
|
|
|
85 parser.add_option( '-m', '--motif_db', dest='motif_db', action='store', type="string", default=None, help='motif_db' )
|
|
|
86 (options, args) = parser.parse_args()
|
|
|
87
|
|
|
88 filename = args[0]
|
|
|
89
|
|
|
90 params = from_json_string( open( filename ).read() )
|
|
|
91 target_directory = params[ 'output_data' ][0]['extra_files_path']
|
|
|
92 os.mkdir( target_directory )
|
|
|
93 data_manager_dict = {}
|
|
|
94
|
|
|
95 #Fetch the Motif Database
|
|
1
|
96 download_motif_databases( data_manager_dict, params, target_directory, options.motif_db )
|
|
0
|
97
|
|
|
98 #save info to json file
|
|
|
99 open( filename, 'wb' ).write( to_json_string( data_manager_dict ) )
|
|
|
100
|
|
|
101 if __name__ == "__main__": main()
|
