Mercurial > repos > jjohnson > data_manager_snpeff
annotate data_manager/data_manager_snpEff_download.py @ 3:c6769a700e55
Update loc data to separate version 4 databases
| author | Jim Johnson <jj@umn.edu> |
|---|---|
| date | Tue, 21 Oct 2014 14:04:33 -0500 |
| parents | bc50aa3b328a |
| children | 78bcf4ac437c |
| rev | line source |
|---|---|
| 0 | 1 #!/usr/bin/env python |
| 2 | |
| 3 import sys | |
| 4 import os | |
| 5 import re | |
| 6 import tempfile | |
| 7 import subprocess | |
| 8 import fileinput | |
| 9 import shutil | |
| 10 import optparse | |
| 11 import urllib2 | |
| 12 from ftplib import FTP | |
| 13 import tarfile | |
| 14 | |
| 15 from galaxy.util.json import from_json_string, to_json_string | |
| 16 | |
| 17 def stop_err(msg): | |
| 18 sys.stderr.write(msg) | |
| 19 sys.exit(1) | |
| 20 | |
| 21 | |
| 22 def fetch_databases(jar_path,genome_list=None): | |
| 23 snpDBs = dict() | |
| 24 (snpEff_dir,snpEff_jar) = os.path.split(jar_path) | |
| 25 databases_path = 'databases.out' | |
| 26 databases_output = open(databases_path,'w') | |
| 27 args = [ 'java','-jar', ] | |
| 28 args.append( snpEff_jar ) | |
| 29 args.append( 'databases' ) | |
| 30 # tmp_stderr = tempfile.NamedTemporaryFile( prefix = "tmp-data-manager-snpEff-stderr" ) | |
| 31 # databases_output = open(databases_path) | |
| 32 # proc = subprocess.Popen( args=args, shell=False, cwd=snpEff_dir, stdout=databases_output.fileno(), stderr=tmp_stderr.fileno() ) | |
| 33 proc = subprocess.Popen( args=args, shell=False, cwd=snpEff_dir, stdout=databases_output.fileno() ) | |
| 34 return_code = proc.wait() | |
| 35 if return_code: | |
| 36 sys.exit( return_code ) | |
| 37 databases_output.close() | |
| 38 try: | |
| 39 fh = open(databases_path,'r') | |
| 40 for i,line in enumerate(fh): | |
| 41 fields = line.split('\t') | |
| 42 if len(fields) >= 2: | |
| 43 genome_version = fields[0].strip() | |
| 44 if genome_list and genome_version not in genome_list: | |
| 45 continue | |
| 46 if genome_version.startswith("Genome") or genome_version.startswith("-"): | |
| 47 continue | |
| 48 description = fields[1].strip() | |
| 49 snpDBs[genome_version] = description; | |
| 50 except Exception, e: | |
| 51 stop_err( 'Error parsing %s %s\n' % (config,str( e )) ) | |
| 52 else: | |
| 53 fh.close() | |
| 54 return snpDBs | |
| 55 | |
| 56 def getOrganismNames(jar_path,genomes,organisms) : | |
| 57 genome_list = genomes.split(',') | |
| 58 organism_list = organisms.split(',') if organisms else [] | |
| 59 if len(genome_list) != len(organism_list): | |
| 60 descriptions = [] | |
| 61 snpDBdict = fetch_databases(jar_path,genome_list=genome_list); | |
| 62 for genome in snpDBdict: | |
| 63 descriptions.append(snpDBdict[genome] if genome in snpDBdict else genome) | |
| 64 return ','.join(descriptions) | |
| 65 return organisms | |
| 66 | |
| 67 """ | |
| 68 # Download human database 'hg19' | |
| 69 java -jar snpEff.jar download -v hg19 | |
| 70 | |
| 71 <command>java -jar \$SNPEFF_JAR_PATH/snpEff.jar download -c \$JAVA_JAR_PATH/snpEff.config $genomeVersion > $logfile </command> | |
| 72 | |
| 73 snpEffectPredictor.bin | |
| 74 regulation_HeLa-S3.bin | |
| 75 regulation_pattern = 'regulation_(.+).bin' | |
| 76 """ | |
| 77 def download_database(data_manager_dict, target_directory, jar_path,config,genome_version,organism): | |
| 78 ## get data_dir from config | |
| 79 ##--- | |
| 80 ## Databases are stored here | |
| 81 ## E.g.: Information for 'hg19' is stored in data_dir/hg19/ | |
| 82 ## | |
| 83 ## Note: Since version 2.1 you can use tilde ('~') as first character to refer to your home directory | |
| 84 ##--- | |
| 85 #data_dir = ~/snpEff/data/ | |
| 86 data_dir = target_directory | |
| 87 (snpEff_dir,snpEff_jar) = os.path.split(jar_path) | |
| 88 args = [ 'java','-jar' ] | |
| 89 args.append( jar_path ) | |
| 90 args.append( 'download' ) | |
| 91 args.append( '-c' ) | |
| 92 args.append( config ) | |
| 93 args.append( '-dataDir' ) | |
| 94 args.append( data_dir ) | |
| 95 args.append( '-v' ) | |
| 96 args.append( genome_version ) | |
| 97 proc = subprocess.Popen( args=args, shell=False, cwd=snpEff_dir ) | |
| 98 return_code = proc.wait() | |
| 99 if return_code: | |
| 100 sys.exit( return_code ) | |
| 101 ## search data_dir/genome_version for files | |
| 102 regulation_pattern = 'regulation_(.+).bin' | |
| 103 # annotation files that are included in snpEff by a flag | |
| 104 annotations_dict = {'nextProt.bin' : '-nextprot','motif.bin': '-motif'} | |
| 105 genome_path = os.path.join(data_dir,genome_version) | |
| 106 if os.path.isdir(genome_path): | |
| 107 for root, dirs, files in os.walk(genome_path): | |
| 108 for fname in files: | |
| 109 if fname.startswith('snpEffectPredictor'): | |
| 110 # if snpEffectPredictor.bin download succeeded | |
| 111 name = genome_version + (' : ' + organism if organism else '') | |
| 112 data_table_entry = dict(value=genome_version, name=name, path=data_dir) | |
|
3
c6769a700e55
Update loc data to separate version 4 databases
Jim Johnson <jj@umn.edu>
parents:
0
diff
changeset
|
113 _add_data_table_entry( data_manager_dict, 'snpeff4_genomedb', data_table_entry ) |
| 0 | 114 else: |
| 115 m = re.match(regulation_pattern,fname) | |
| 116 if m: | |
| 117 name = m.groups()[0] | |
| 118 data_table_entry = dict(genome=genome_version,value=name, name=name) | |
|
3
c6769a700e55
Update loc data to separate version 4 databases
Jim Johnson <jj@umn.edu>
parents:
0
diff
changeset
|
119 _add_data_table_entry( data_manager_dict, 'snpeff4_regulationdb', data_table_entry ) |
| 0 | 120 elif fname in annotations_dict: |
| 121 value = annotations_dict[fname] | |
| 122 name = value.lstrip('-') | |
| 123 data_table_entry = dict(genome=genome_version,value=value, name=name) | |
|
3
c6769a700e55
Update loc data to separate version 4 databases
Jim Johnson <jj@umn.edu>
parents:
0
diff
changeset
|
124 _add_data_table_entry( data_manager_dict, 'snpeff4_annotations', data_table_entry ) |
| 0 | 125 return data_manager_dict |
| 126 | |
| 127 def _add_data_table_entry( data_manager_dict, data_table, data_table_entry ): | |
| 128 data_manager_dict['data_tables'] = data_manager_dict.get( 'data_tables', {} ) | |
| 129 data_manager_dict['data_tables'][data_table] = data_manager_dict['data_tables'].get( data_table, [] ) | |
| 130 data_manager_dict['data_tables'][data_table].append( data_table_entry ) | |
| 131 return data_manager_dict | |
| 132 | |
| 133 def main(): | |
| 134 #Parse Command Line | |
| 135 parser = optparse.OptionParser() | |
| 136 parser.add_option( '-j', '--jar_path', dest='jar_path', action='store', type="string", default=None, help='snpEff.jar path' ) | |
| 137 parser.add_option( '-c', '--config', dest='config', action='store', type="string", default=None, help='snpEff.config path' ) | |
| 138 parser.add_option( '-g', '--genome_version', dest='genome_version', action='store', type="string", default=None, help='genome_version' ) | |
| 139 parser.add_option( '-o', '--organism', dest='organism', action='store', type="string", default=None, help='organism name' ) | |
| 140 (options, args) = parser.parse_args() | |
| 141 | |
| 142 filename = args[0] | |
| 143 | |
| 144 params = from_json_string( open( filename ).read() ) | |
| 145 target_directory = params[ 'output_data' ][0]['extra_files_path'] | |
| 146 os.mkdir( target_directory ) | |
| 147 data_manager_dict = {} | |
| 148 | |
| 149 | |
| 150 #Create SnpEff Reference Data | |
| 151 for genome_version, organism in zip(options.genome_version.split(','), getOrganismNames(options.jar_path,options.genome_version,options.organism).split(',')): | |
| 152 download_database( data_manager_dict, target_directory, options.jar_path, options.config, genome_version, organism ) | |
| 153 | |
| 154 #save info to json file | |
| 155 open( filename, 'wb' ).write( to_json_string( data_manager_dict ) ) | |
| 156 | |
| 157 if __name__ == "__main__": main() | |
| 158 |