Mercurial > repos > jjohnson > data_manager_snpeff
annotate data_manager/data_manager_snpEff_download.py @ 6:a329eda0cdff draft default tip
Uploaded
| author | jjohnson |
|---|---|
| date | Wed, 09 Dec 2015 13:49:55 -0500 |
| parents | 78bcf4ac437c |
| children |
| rev | line source |
|---|---|
| 0 | 1 #!/usr/bin/env python |
| 2 | |
| 3 import sys | |
| 4 import os | |
| 5 import re | |
| 6 import tempfile | |
| 7 import subprocess | |
| 8 import fileinput | |
| 9 import shutil | |
| 10 import optparse | |
| 11 import urllib2 | |
|
5
78bcf4ac437c
Use tool_data_table with key and version columns added to allow for multiple versions in a .loc file
Jim Johnson <jj@umn.edu>
parents:
3
diff
changeset
|
12 import gzip |
| 0 | 13 from ftplib import FTP |
| 14 import tarfile | |
| 15 | |
| 16 from galaxy.util.json import from_json_string, to_json_string | |
| 17 | |
| 18 def stop_err(msg): | |
| 19 sys.stderr.write(msg) | |
| 20 sys.exit(1) | |
| 21 | |
| 22 | |
| 23 def fetch_databases(jar_path,genome_list=None): | |
| 24 snpDBs = dict() | |
| 25 (snpEff_dir,snpEff_jar) = os.path.split(jar_path) | |
| 26 databases_path = 'databases.out' | |
| 27 databases_output = open(databases_path,'w') | |
| 28 args = [ 'java','-jar', ] | |
| 29 args.append( snpEff_jar ) | |
| 30 args.append( 'databases' ) | |
| 31 # tmp_stderr = tempfile.NamedTemporaryFile( prefix = "tmp-data-manager-snpEff-stderr" ) | |
| 32 # databases_output = open(databases_path) | |
| 33 # proc = subprocess.Popen( args=args, shell=False, cwd=snpEff_dir, stdout=databases_output.fileno(), stderr=tmp_stderr.fileno() ) | |
| 34 proc = subprocess.Popen( args=args, shell=False, cwd=snpEff_dir, stdout=databases_output.fileno() ) | |
| 35 return_code = proc.wait() | |
| 36 if return_code: | |
| 37 sys.exit( return_code ) | |
| 38 databases_output.close() | |
| 39 try: | |
| 40 fh = open(databases_path,'r') | |
| 41 for i,line in enumerate(fh): | |
| 42 fields = line.split('\t') | |
| 43 if len(fields) >= 2: | |
| 44 genome_version = fields[0].strip() | |
| 45 if genome_list and genome_version not in genome_list: | |
| 46 continue | |
| 47 if genome_version.startswith("Genome") or genome_version.startswith("-"): | |
| 48 continue | |
| 49 description = fields[1].strip() | |
| 50 snpDBs[genome_version] = description; | |
| 51 except Exception, e: | |
| 52 stop_err( 'Error parsing %s %s\n' % (config,str( e )) ) | |
| 53 else: | |
| 54 fh.close() | |
| 55 return snpDBs | |
| 56 | |
| 57 def getOrganismNames(jar_path,genomes,organisms) : | |
| 58 genome_list = genomes.split(',') | |
| 59 organism_list = organisms.split(',') if organisms else [] | |
| 60 if len(genome_list) != len(organism_list): | |
| 61 descriptions = [] | |
| 62 snpDBdict = fetch_databases(jar_path,genome_list=genome_list); | |
| 63 for genome in snpDBdict: | |
| 64 descriptions.append(snpDBdict[genome] if genome in snpDBdict else genome) | |
| 65 return ','.join(descriptions) | |
| 66 return organisms | |
| 67 | |
|
5
78bcf4ac437c
Use tool_data_table with key and version columns added to allow for multiple versions in a .loc file
Jim Johnson <jj@umn.edu>
parents:
3
diff
changeset
|
68 def getSnpeffVersion(jar_path): |
|
78bcf4ac437c
Use tool_data_table with key and version columns added to allow for multiple versions in a .loc file
Jim Johnson <jj@umn.edu>
parents:
3
diff
changeset
|
69 snpeff_version = 'SnpEff ?.?' |
|
78bcf4ac437c
Use tool_data_table with key and version columns added to allow for multiple versions in a .loc file
Jim Johnson <jj@umn.edu>
parents:
3
diff
changeset
|
70 (snpEff_dir,snpEff_jar) = os.path.split(jar_path) |
|
78bcf4ac437c
Use tool_data_table with key and version columns added to allow for multiple versions in a .loc file
Jim Johnson <jj@umn.edu>
parents:
3
diff
changeset
|
71 stderr_path = 'snpeff.err' |
|
78bcf4ac437c
Use tool_data_table with key and version columns added to allow for multiple versions in a .loc file
Jim Johnson <jj@umn.edu>
parents:
3
diff
changeset
|
72 stderr_fh = open(stderr_path,'w') |
|
78bcf4ac437c
Use tool_data_table with key and version columns added to allow for multiple versions in a .loc file
Jim Johnson <jj@umn.edu>
parents:
3
diff
changeset
|
73 args = [ 'java','-jar', ] |
|
78bcf4ac437c
Use tool_data_table with key and version columns added to allow for multiple versions in a .loc file
Jim Johnson <jj@umn.edu>
parents:
3
diff
changeset
|
74 args.append( snpEff_jar ) |
|
78bcf4ac437c
Use tool_data_table with key and version columns added to allow for multiple versions in a .loc file
Jim Johnson <jj@umn.edu>
parents:
3
diff
changeset
|
75 args.append( '-h' ) |
|
78bcf4ac437c
Use tool_data_table with key and version columns added to allow for multiple versions in a .loc file
Jim Johnson <jj@umn.edu>
parents:
3
diff
changeset
|
76 proc = subprocess.Popen( args=args, shell=False, cwd=snpEff_dir, stderr=stderr_fh.fileno() ) |
|
78bcf4ac437c
Use tool_data_table with key and version columns added to allow for multiple versions in a .loc file
Jim Johnson <jj@umn.edu>
parents:
3
diff
changeset
|
77 return_code = proc.wait() |
|
78bcf4ac437c
Use tool_data_table with key and version columns added to allow for multiple versions in a .loc file
Jim Johnson <jj@umn.edu>
parents:
3
diff
changeset
|
78 if return_code != 255: |
|
78bcf4ac437c
Use tool_data_table with key and version columns added to allow for multiple versions in a .loc file
Jim Johnson <jj@umn.edu>
parents:
3
diff
changeset
|
79 sys.exit( return_code ) |
|
78bcf4ac437c
Use tool_data_table with key and version columns added to allow for multiple versions in a .loc file
Jim Johnson <jj@umn.edu>
parents:
3
diff
changeset
|
80 stderr_fh.close() |
|
78bcf4ac437c
Use tool_data_table with key and version columns added to allow for multiple versions in a .loc file
Jim Johnson <jj@umn.edu>
parents:
3
diff
changeset
|
81 fh = open(stderr_path,'r') |
|
78bcf4ac437c
Use tool_data_table with key and version columns added to allow for multiple versions in a .loc file
Jim Johnson <jj@umn.edu>
parents:
3
diff
changeset
|
82 for line in fh: |
|
78bcf4ac437c
Use tool_data_table with key and version columns added to allow for multiple versions in a .loc file
Jim Johnson <jj@umn.edu>
parents:
3
diff
changeset
|
83 m = re.match('^[Ss]npEff version (SnpEff)\s*(\d+\.\d+).*$',line) |
|
78bcf4ac437c
Use tool_data_table with key and version columns added to allow for multiple versions in a .loc file
Jim Johnson <jj@umn.edu>
parents:
3
diff
changeset
|
84 if m: |
|
78bcf4ac437c
Use tool_data_table with key and version columns added to allow for multiple versions in a .loc file
Jim Johnson <jj@umn.edu>
parents:
3
diff
changeset
|
85 snpeff_version = m.groups()[0] + m.groups()[1] |
|
78bcf4ac437c
Use tool_data_table with key and version columns added to allow for multiple versions in a .loc file
Jim Johnson <jj@umn.edu>
parents:
3
diff
changeset
|
86 break |
|
78bcf4ac437c
Use tool_data_table with key and version columns added to allow for multiple versions in a .loc file
Jim Johnson <jj@umn.edu>
parents:
3
diff
changeset
|
87 fh.close() |
|
78bcf4ac437c
Use tool_data_table with key and version columns added to allow for multiple versions in a .loc file
Jim Johnson <jj@umn.edu>
parents:
3
diff
changeset
|
88 return snpeff_version |
|
78bcf4ac437c
Use tool_data_table with key and version columns added to allow for multiple versions in a .loc file
Jim Johnson <jj@umn.edu>
parents:
3
diff
changeset
|
89 |
|
78bcf4ac437c
Use tool_data_table with key and version columns added to allow for multiple versions in a .loc file
Jim Johnson <jj@umn.edu>
parents:
3
diff
changeset
|
90 # Starting with SnpEff 4.1 the .bin files contain the SnpEff version: |
|
78bcf4ac437c
Use tool_data_table with key and version columns added to allow for multiple versions in a .loc file
Jim Johnson <jj@umn.edu>
parents:
3
diff
changeset
|
91 # Example - the first 3 line of GRCh37.75/snpEffectPredictor.bin (uncompressed): |
|
78bcf4ac437c
Use tool_data_table with key and version columns added to allow for multiple versions in a .loc file
Jim Johnson <jj@umn.edu>
parents:
3
diff
changeset
|
92 """ |
|
78bcf4ac437c
Use tool_data_table with key and version columns added to allow for multiple versions in a .loc file
Jim Johnson <jj@umn.edu>
parents:
3
diff
changeset
|
93 SnpEff 4.1 |
|
78bcf4ac437c
Use tool_data_table with key and version columns added to allow for multiple versions in a .loc file
Jim Johnson <jj@umn.edu>
parents:
3
diff
changeset
|
94 CHROMOSOME 2 1 0 179197 GL000219.1 false |
|
78bcf4ac437c
Use tool_data_table with key and version columns added to allow for multiple versions in a .loc file
Jim Johnson <jj@umn.edu>
parents:
3
diff
changeset
|
95 CHROMOSOME 3 1 0 81347269 HSCHR17_1 false |
|
78bcf4ac437c
Use tool_data_table with key and version columns added to allow for multiple versions in a .loc file
Jim Johnson <jj@umn.edu>
parents:
3
diff
changeset
|
96 """ |
|
78bcf4ac437c
Use tool_data_table with key and version columns added to allow for multiple versions in a .loc file
Jim Johnson <jj@umn.edu>
parents:
3
diff
changeset
|
97 def getSnpeffVersionFromFile(path): |
|
78bcf4ac437c
Use tool_data_table with key and version columns added to allow for multiple versions in a .loc file
Jim Johnson <jj@umn.edu>
parents:
3
diff
changeset
|
98 snpeff_version = None |
|
78bcf4ac437c
Use tool_data_table with key and version columns added to allow for multiple versions in a .loc file
Jim Johnson <jj@umn.edu>
parents:
3
diff
changeset
|
99 try: |
|
78bcf4ac437c
Use tool_data_table with key and version columns added to allow for multiple versions in a .loc file
Jim Johnson <jj@umn.edu>
parents:
3
diff
changeset
|
100 fh = gzip.open(path, 'rb') |
|
78bcf4ac437c
Use tool_data_table with key and version columns added to allow for multiple versions in a .loc file
Jim Johnson <jj@umn.edu>
parents:
3
diff
changeset
|
101 buf = fh.read(100) |
|
78bcf4ac437c
Use tool_data_table with key and version columns added to allow for multiple versions in a .loc file
Jim Johnson <jj@umn.edu>
parents:
3
diff
changeset
|
102 lines = buf.splitlines() |
|
78bcf4ac437c
Use tool_data_table with key and version columns added to allow for multiple versions in a .loc file
Jim Johnson <jj@umn.edu>
parents:
3
diff
changeset
|
103 m = re.match('^(SnpEff)\s+(\d+\.\d+).*$',lines[0].strip()) |
|
78bcf4ac437c
Use tool_data_table with key and version columns added to allow for multiple versions in a .loc file
Jim Johnson <jj@umn.edu>
parents:
3
diff
changeset
|
104 if m: |
|
78bcf4ac437c
Use tool_data_table with key and version columns added to allow for multiple versions in a .loc file
Jim Johnson <jj@umn.edu>
parents:
3
diff
changeset
|
105 snpeff_version = m.groups()[0] + m.groups()[1] |
|
78bcf4ac437c
Use tool_data_table with key and version columns added to allow for multiple versions in a .loc file
Jim Johnson <jj@umn.edu>
parents:
3
diff
changeset
|
106 fh.close() |
|
78bcf4ac437c
Use tool_data_table with key and version columns added to allow for multiple versions in a .loc file
Jim Johnson <jj@umn.edu>
parents:
3
diff
changeset
|
107 except Exception, e: |
|
78bcf4ac437c
Use tool_data_table with key and version columns added to allow for multiple versions in a .loc file
Jim Johnson <jj@umn.edu>
parents:
3
diff
changeset
|
108 stop_err( 'Error parsing SnpEff version from: %s %s\n' % (path,str( e )) ) |
|
78bcf4ac437c
Use tool_data_table with key and version columns added to allow for multiple versions in a .loc file
Jim Johnson <jj@umn.edu>
parents:
3
diff
changeset
|
109 return snpeff_version |
|
78bcf4ac437c
Use tool_data_table with key and version columns added to allow for multiple versions in a .loc file
Jim Johnson <jj@umn.edu>
parents:
3
diff
changeset
|
110 |
| 0 | 111 """ |
| 112 # Download human database 'hg19' | |
| 113 java -jar snpEff.jar download -v hg19 | |
| 114 | |
| 115 <command>java -jar \$SNPEFF_JAR_PATH/snpEff.jar download -c \$JAVA_JAR_PATH/snpEff.config $genomeVersion > $logfile </command> | |
| 116 | |
| 117 snpEffectPredictor.bin | |
| 118 regulation_HeLa-S3.bin | |
| 119 regulation_pattern = 'regulation_(.+).bin' | |
| 120 """ | |
|
5
78bcf4ac437c
Use tool_data_table with key and version columns added to allow for multiple versions in a .loc file
Jim Johnson <jj@umn.edu>
parents:
3
diff
changeset
|
121 def download_database(data_manager_dict, target_directory, jar_path, config, genome_version, organism): |
| 0 | 122 ## get data_dir from config |
| 123 ##--- | |
| 124 ## Databases are stored here | |
| 125 ## E.g.: Information for 'hg19' is stored in data_dir/hg19/ | |
| 126 ## | |
| 127 ## Note: Since version 2.1 you can use tilde ('~') as first character to refer to your home directory | |
| 128 ##--- | |
| 129 #data_dir = ~/snpEff/data/ | |
| 130 data_dir = target_directory | |
| 131 (snpEff_dir,snpEff_jar) = os.path.split(jar_path) | |
| 132 args = [ 'java','-jar' ] | |
| 133 args.append( jar_path ) | |
| 134 args.append( 'download' ) | |
| 135 args.append( '-c' ) | |
| 136 args.append( config ) | |
| 137 args.append( '-dataDir' ) | |
| 138 args.append( data_dir ) | |
| 139 args.append( '-v' ) | |
| 140 args.append( genome_version ) | |
| 141 proc = subprocess.Popen( args=args, shell=False, cwd=snpEff_dir ) | |
| 142 return_code = proc.wait() | |
| 143 if return_code: | |
| 144 sys.exit( return_code ) | |
| 145 ## search data_dir/genome_version for files | |
| 146 regulation_pattern = 'regulation_(.+).bin' | |
| 147 # annotation files that are included in snpEff by a flag | |
| 148 annotations_dict = {'nextProt.bin' : '-nextprot','motif.bin': '-motif'} | |
| 149 genome_path = os.path.join(data_dir,genome_version) | |
|
5
78bcf4ac437c
Use tool_data_table with key and version columns added to allow for multiple versions in a .loc file
Jim Johnson <jj@umn.edu>
parents:
3
diff
changeset
|
150 snpeff_version = getSnpeffVersion(jar_path) |
|
78bcf4ac437c
Use tool_data_table with key and version columns added to allow for multiple versions in a .loc file
Jim Johnson <jj@umn.edu>
parents:
3
diff
changeset
|
151 key = snpeff_version + '_' + genome_version |
| 0 | 152 if os.path.isdir(genome_path): |
| 153 for root, dirs, files in os.walk(genome_path): | |
| 154 for fname in files: | |
| 155 if fname.startswith('snpEffectPredictor'): | |
| 156 # if snpEffectPredictor.bin download succeeded | |
| 157 name = genome_version + (' : ' + organism if organism else '') | |
|
5
78bcf4ac437c
Use tool_data_table with key and version columns added to allow for multiple versions in a .loc file
Jim Johnson <jj@umn.edu>
parents:
3
diff
changeset
|
158 # version = getSnpeffVersionFromFile(os.path.join(root,fname)) |
|
78bcf4ac437c
Use tool_data_table with key and version columns added to allow for multiple versions in a .loc file
Jim Johnson <jj@umn.edu>
parents:
3
diff
changeset
|
159 data_table_entry = dict(key=key,version=snpeff_version,value=genome_version, name=name, path=data_dir) |
|
78bcf4ac437c
Use tool_data_table with key and version columns added to allow for multiple versions in a .loc file
Jim Johnson <jj@umn.edu>
parents:
3
diff
changeset
|
160 _add_data_table_entry( data_manager_dict, 'snpeffv_genomedb', data_table_entry ) |
| 0 | 161 else: |
| 162 m = re.match(regulation_pattern,fname) | |
| 163 if m: | |
| 164 name = m.groups()[0] | |
|
5
78bcf4ac437c
Use tool_data_table with key and version columns added to allow for multiple versions in a .loc file
Jim Johnson <jj@umn.edu>
parents:
3
diff
changeset
|
165 data_table_entry = dict(key=key,version=snpeff_version,genome=genome_version,value=name, name=name) |
|
78bcf4ac437c
Use tool_data_table with key and version columns added to allow for multiple versions in a .loc file
Jim Johnson <jj@umn.edu>
parents:
3
diff
changeset
|
166 _add_data_table_entry( data_manager_dict, 'snpeffv_regulationdb', data_table_entry ) |
| 0 | 167 elif fname in annotations_dict: |
| 168 value = annotations_dict[fname] | |
| 169 name = value.lstrip('-') | |
|
5
78bcf4ac437c
Use tool_data_table with key and version columns added to allow for multiple versions in a .loc file
Jim Johnson <jj@umn.edu>
parents:
3
diff
changeset
|
170 data_table_entry = dict(key=key,version=snpeff_version,genome=genome_version,value=value, name=name) |
|
78bcf4ac437c
Use tool_data_table with key and version columns added to allow for multiple versions in a .loc file
Jim Johnson <jj@umn.edu>
parents:
3
diff
changeset
|
171 _add_data_table_entry( data_manager_dict, 'snpeffv_annotations', data_table_entry ) |
| 0 | 172 return data_manager_dict |
| 173 | |
| 174 def _add_data_table_entry( data_manager_dict, data_table, data_table_entry ): | |
| 175 data_manager_dict['data_tables'] = data_manager_dict.get( 'data_tables', {} ) | |
| 176 data_manager_dict['data_tables'][data_table] = data_manager_dict['data_tables'].get( data_table, [] ) | |
| 177 data_manager_dict['data_tables'][data_table].append( data_table_entry ) | |
| 178 return data_manager_dict | |
| 179 | |
| 180 def main(): | |
| 181 #Parse Command Line | |
| 182 parser = optparse.OptionParser() | |
| 183 parser.add_option( '-j', '--jar_path', dest='jar_path', action='store', type="string", default=None, help='snpEff.jar path' ) | |
| 184 parser.add_option( '-c', '--config', dest='config', action='store', type="string", default=None, help='snpEff.config path' ) | |
| 185 parser.add_option( '-g', '--genome_version', dest='genome_version', action='store', type="string", default=None, help='genome_version' ) | |
| 186 parser.add_option( '-o', '--organism', dest='organism', action='store', type="string", default=None, help='organism name' ) | |
| 187 (options, args) = parser.parse_args() | |
| 188 | |
| 189 filename = args[0] | |
| 190 | |
| 191 params = from_json_string( open( filename ).read() ) | |
| 192 target_directory = params[ 'output_data' ][0]['extra_files_path'] | |
| 193 os.mkdir( target_directory ) | |
| 194 data_manager_dict = {} | |
| 195 | |
| 196 | |
| 197 #Create SnpEff Reference Data | |
| 198 for genome_version, organism in zip(options.genome_version.split(','), getOrganismNames(options.jar_path,options.genome_version,options.organism).split(',')): | |
| 199 download_database( data_manager_dict, target_directory, options.jar_path, options.config, genome_version, organism ) | |
| 200 | |
| 201 #save info to json file | |
| 202 open( filename, 'wb' ).write( to_json_string( data_manager_dict ) ) | |
| 203 | |
| 204 if __name__ == "__main__": main() | |
| 205 |