|
0
|
1 #!/usr/bin/env python
|
|
|
2
|
|
|
3 import sys
|
|
|
4 import os
|
|
|
5 import re
|
|
|
6 import tempfile
|
|
|
7 import subprocess
|
|
|
8 import fileinput
|
|
|
9 import shutil
|
|
|
10 import optparse
|
|
|
11 import urllib2
|
|
|
12 from ftplib import FTP
|
|
|
13 import tarfile
|
|
|
14
|
|
|
15 from galaxy.util.json import from_json_string, to_json_string
|
|
|
16
|
|
|
17 def stop_err(msg):
|
|
|
18 sys.stderr.write(msg)
|
|
|
19 sys.exit(1)
|
|
|
20
|
|
|
21
|
|
|
22 def fetch_databases(jar_path,genome_list=None):
|
|
|
23 snpDBs = dict()
|
|
|
24 (snpEff_dir,snpEff_jar) = os.path.split(jar_path)
|
|
|
25 databases_path = 'databases.out'
|
|
|
26 databases_output = open(databases_path,'w')
|
|
|
27 args = [ 'java','-jar', ]
|
|
|
28 args.append( snpEff_jar )
|
|
|
29 args.append( 'databases' )
|
|
|
30 # tmp_stderr = tempfile.NamedTemporaryFile( prefix = "tmp-data-manager-snpEff-stderr" )
|
|
|
31 # databases_output = open(databases_path)
|
|
|
32 # proc = subprocess.Popen( args=args, shell=False, cwd=snpEff_dir, stdout=databases_output.fileno(), stderr=tmp_stderr.fileno() )
|
|
|
33 proc = subprocess.Popen( args=args, shell=False, cwd=snpEff_dir, stdout=databases_output.fileno() )
|
|
|
34 return_code = proc.wait()
|
|
|
35 if return_code:
|
|
|
36 sys.exit( return_code )
|
|
|
37 databases_output.close()
|
|
|
38 try:
|
|
|
39 fh = open(databases_path,'r')
|
|
|
40 for i,line in enumerate(fh):
|
|
|
41 fields = line.split('\t')
|
|
|
42 if len(fields) >= 2:
|
|
|
43 genome_version = fields[0].strip()
|
|
|
44 if genome_list and genome_version not in genome_list:
|
|
|
45 continue
|
|
|
46 if genome_version.startswith("Genome") or genome_version.startswith("-"):
|
|
|
47 continue
|
|
|
48 description = fields[1].strip()
|
|
|
49 snpDBs[genome_version] = description;
|
|
|
50 except Exception, e:
|
|
|
51 stop_err( 'Error parsing %s %s\n' % (config,str( e )) )
|
|
|
52 else:
|
|
|
53 fh.close()
|
|
|
54 return snpDBs
|
|
|
55
|
|
|
56 def getOrganismNames(jar_path,genomes,organisms) :
|
|
|
57 genome_list = genomes.split(',')
|
|
|
58 organism_list = organisms.split(',') if organisms else []
|
|
|
59 if len(genome_list) != len(organism_list):
|
|
|
60 descriptions = []
|
|
|
61 snpDBdict = fetch_databases(jar_path,genome_list=genome_list);
|
|
|
62 for genome in snpDBdict:
|
|
|
63 descriptions.append(snpDBdict[genome] if genome in snpDBdict else genome)
|
|
|
64 return ','.join(descriptions)
|
|
|
65 return organisms
|
|
|
66
|
|
|
67 """
|
|
|
68 # Download human database 'hg19'
|
|
|
69 java -jar snpEff.jar download -v hg19
|
|
|
70
|
|
|
71 <command>java -jar \$SNPEFF_JAR_PATH/snpEff.jar download -c \$JAVA_JAR_PATH/snpEff.config $genomeVersion > $logfile </command>
|
|
|
72
|
|
|
73 snpEffectPredictor.bin
|
|
|
74 regulation_HeLa-S3.bin
|
|
|
75 regulation_pattern = 'regulation_(.+).bin'
|
|
|
76 """
|
|
|
77 def download_database(data_manager_dict, target_directory, jar_path,config,genome_version,organism):
|
|
|
78 ## get data_dir from config
|
|
|
79 ##---
|
|
|
80 ## Databases are stored here
|
|
|
81 ## E.g.: Information for 'hg19' is stored in data_dir/hg19/
|
|
|
82 ##
|
|
|
83 ## Note: Since version 2.1 you can use tilde ('~') as first character to refer to your home directory
|
|
|
84 ##---
|
|
|
85 #data_dir = ~/snpEff/data/
|
|
|
86 data_dir = target_directory
|
|
|
87 (snpEff_dir,snpEff_jar) = os.path.split(jar_path)
|
|
|
88 args = [ 'java','-jar' ]
|
|
|
89 args.append( jar_path )
|
|
|
90 args.append( 'download' )
|
|
|
91 args.append( '-c' )
|
|
|
92 args.append( config )
|
|
|
93 args.append( '-dataDir' )
|
|
|
94 args.append( data_dir )
|
|
|
95 args.append( '-v' )
|
|
|
96 args.append( genome_version )
|
|
|
97 proc = subprocess.Popen( args=args, shell=False, cwd=snpEff_dir )
|
|
|
98 return_code = proc.wait()
|
|
|
99 if return_code:
|
|
|
100 sys.exit( return_code )
|
|
|
101 ## search data_dir/genome_version for files
|
|
|
102 regulation_pattern = 'regulation_(.+).bin'
|
|
|
103 # annotation files that are included in snpEff by a flag
|
|
|
104 annotations_dict = {'nextProt.bin' : '-nextprot','motif.bin': '-motif'}
|
|
|
105 genome_path = os.path.join(data_dir,genome_version)
|
|
|
106 if os.path.isdir(genome_path):
|
|
|
107 for root, dirs, files in os.walk(genome_path):
|
|
|
108 for fname in files:
|
|
|
109 if fname.startswith('snpEffectPredictor'):
|
|
|
110 # if snpEffectPredictor.bin download succeeded
|
|
|
111 name = genome_version + (' : ' + organism if organism else '')
|
|
|
112 data_table_entry = dict(value=genome_version, name=name, path=data_dir)
|
|
|
113 _add_data_table_entry( data_manager_dict, 'snpeff_genomedb', data_table_entry )
|
|
|
114 else:
|
|
|
115 m = re.match(regulation_pattern,fname)
|
|
|
116 if m:
|
|
|
117 name = m.groups()[0]
|
|
|
118 data_table_entry = dict(genome=genome_version,value=name, name=name)
|
|
|
119 _add_data_table_entry( data_manager_dict, 'snpeff_regulationdb', data_table_entry )
|
|
|
120 elif fname in annotations_dict:
|
|
|
121 value = annotations_dict[fname]
|
|
|
122 name = value.lstrip('-')
|
|
|
123 data_table_entry = dict(genome=genome_version,value=value, name=name)
|
|
|
124 _add_data_table_entry( data_manager_dict, 'snpeff_annotations', data_table_entry )
|
|
|
125 return data_manager_dict
|
|
|
126
|
|
|
127 def _add_data_table_entry( data_manager_dict, data_table, data_table_entry ):
|
|
|
128 data_manager_dict['data_tables'] = data_manager_dict.get( 'data_tables', {} )
|
|
|
129 data_manager_dict['data_tables'][data_table] = data_manager_dict['data_tables'].get( data_table, [] )
|
|
|
130 data_manager_dict['data_tables'][data_table].append( data_table_entry )
|
|
|
131 return data_manager_dict
|
|
|
132
|
|
|
133 def main():
|
|
|
134 #Parse Command Line
|
|
|
135 parser = optparse.OptionParser()
|
|
|
136 parser.add_option( '-j', '--jar_path', dest='jar_path', action='store', type="string", default=None, help='snpEff.jar path' )
|
|
|
137 parser.add_option( '-c', '--config', dest='config', action='store', type="string", default=None, help='snpEff.config path' )
|
|
|
138 parser.add_option( '-g', '--genome_version', dest='genome_version', action='store', type="string", default=None, help='genome_version' )
|
|
|
139 parser.add_option( '-o', '--organism', dest='organism', action='store', type="string", default=None, help='organism name' )
|
|
|
140 (options, args) = parser.parse_args()
|
|
|
141
|
|
|
142 filename = args[0]
|
|
|
143
|
|
|
144 params = from_json_string( open( filename ).read() )
|
|
|
145 target_directory = params[ 'output_data' ][0]['extra_files_path']
|
|
|
146 os.mkdir( target_directory )
|
|
|
147 data_manager_dict = {}
|
|
|
148
|
|
|
149
|
|
|
150 #Create SnpEff Reference Data
|
|
|
151 for genome_version, organism in zip(options.genome_version.split(','), getOrganismNames(options.jar_path,options.genome_version,options.organism).split(',')):
|
|
|
152 download_database( data_manager_dict, target_directory, options.jar_path, options.config, genome_version, organism )
|
|
|
153
|
|
|
154 #save info to json file
|
|
|
155 open( filename, 'wb' ).write( to_json_string( data_manager_dict ) )
|
|
|
156
|
|
|
157 if __name__ == "__main__": main()
|
|
|
158
|
