Mercurial > repos > iuc > data_manager_gemini_database_downloader
changeset 0:a47cd0bacb67 draft
Uploaded
| author | iuc |
|---|---|
| date | Sat, 16 Aug 2014 11:03:57 -0400 |
| parents | |
| children | 50405cb4f8cd |
| files | data_manager/data_manager_gemini_download.py data_manager/data_manager_gemini_download.xml data_manager_conf.xml tool-data/gemini_databases.loc.sample tool_data_table_conf.xml.sample tool_dependencies.xml |
| diffstat | 6 files changed, 256 insertions(+), 0 deletions(-) [+] |
line wrap: on
line diff
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/data_manager/data_manager_gemini_download.py Sat Aug 16 11:03:57 2014 -0400 @@ -0,0 +1,158 @@ +#!/usr/bin/env python + +import sys +import os +import re +import tempfile +import subprocess +import fileinput +import shutil +import optparse +import urllib2 +from ftplib import FTP +import tarfile + +from galaxy.util.json import from_json_string, to_json_string + +def stop_err(msg): + sys.stderr.write(msg) + sys.exit(1) + + +def fetch_databases(jar_path,genome_list=None): + snpDBs = dict() + (snpEff_dir,snpEff_jar) = os.path.split(jar_path) + databases_path = 'databases.out' + databases_output = open(databases_path,'w') + args = [ 'java','-jar', ] + args.append( snpEff_jar ) + args.append( 'databases' ) + # tmp_stderr = tempfile.NamedTemporaryFile( prefix = "tmp-data-manager-snpEff-stderr" ) + # databases_output = open(databases_path) + # proc = subprocess.Popen( args=args, shell=False, cwd=snpEff_dir, stdout=databases_output.fileno(), stderr=tmp_stderr.fileno() ) + proc = subprocess.Popen( args=args, shell=False, cwd=snpEff_dir, stdout=databases_output.fileno() ) + return_code = proc.wait() + if return_code: + sys.exit( return_code ) + databases_output.close() + try: + fh = open(databases_path,'r') + for i,line in enumerate(fh): + fields = line.split('\t') + if len(fields) >= 2: + genome_version = fields[0].strip() + if genome_list and genome_version not in genome_list: + continue + if genome_version.startswith("Genome") or genome_version.startswith("-"): + continue + description = fields[1].strip() + snpDBs[genome_version] = description; + except Exception, e: + stop_err( 'Error parsing %s %s\n' % (config,str( e )) ) + else: + fh.close() + return snpDBs + +def getOrganismNames(jar_path,genomes,organisms) : + genome_list = genomes.split(',') + organism_list = organisms.split(',') if organisms else [] + if len(genome_list) != len(organism_list): + descriptions = [] + snpDBdict = fetch_databases(jar_path,genome_list=genome_list); + for genome in snpDBdict: + descriptions.append(snpDBdict[genome] if genome in snpDBdict else genome) + return ','.join(descriptions) + return organisms + +""" +# Download human database 'hg19' +java -jar snpEff.jar download -v hg19 + + <command>java -jar \$SNPEFF_JAR_PATH/snpEff.jar download -c \$JAVA_JAR_PATH/snpEff.config $genomeVersion > $logfile </command> + +snpEffectPredictor.bin +regulation_HeLa-S3.bin +regulation_pattern = 'regulation_(.+).bin' +""" +def download_database(data_manager_dict, target_directory, jar_path,config,genome_version,organism): + ## get data_dir from config + ##--- + ## Databases are stored here + ## E.g.: Information for 'hg19' is stored in data_dir/hg19/ + ## + ## Note: Since version 2.1 you can use tilde ('~') as first character to refer to your home directory + ##--- + #data_dir = ~/snpEff/data/ + data_dir = target_directory + (snpEff_dir,snpEff_jar) = os.path.split(jar_path) + args = [ 'java','-jar' ] + args.append( jar_path ) + args.append( 'download' ) + args.append( '-c' ) + args.append( config ) + args.append( '-dataDir' ) + args.append( data_dir ) + args.append( '-v' ) + args.append( genome_version ) + proc = subprocess.Popen( args=args, shell=False, cwd=snpEff_dir ) + return_code = proc.wait() + if return_code: + sys.exit( return_code ) + ## search data_dir/genome_version for files + regulation_pattern = 'regulation_(.+).bin' + # annotation files that are included in snpEff by a flag + annotations_dict = {'nextProt.bin' : '-nextprot','motif.bin': '-motif'} + genome_path = os.path.join(data_dir,genome_version) + if os.path.isdir(genome_path): + for root, dirs, files in os.walk(genome_path): + for fname in files: + if fname.startswith('snpEffectPredictor'): + # if snpEffectPredictor.bin download succeeded + name = genome_version + (' : ' + organism if organism else '') + data_table_entry = dict(value=genome_version, name=name, path=data_dir) + _add_data_table_entry( data_manager_dict, 'snpeff_genomedb', data_table_entry ) + else: + m = re.match(regulation_pattern,fname) + if m: + name = m.groups()[0] + data_table_entry = dict(genome=genome_version,value=name, name=name) + _add_data_table_entry( data_manager_dict, 'snpeff_regulationdb', data_table_entry ) + elif fname in annotations_dict: + value = annotations_dict[fname] + name = value.lstrip('-') + data_table_entry = dict(genome=genome_version,value=value, name=name) + _add_data_table_entry( data_manager_dict, 'snpeff_annotations', data_table_entry ) + return data_manager_dict + +def _add_data_table_entry( data_manager_dict, data_table, data_table_entry ): + data_manager_dict['data_tables'] = data_manager_dict.get( 'data_tables', {} ) + data_manager_dict['data_tables'][data_table] = data_manager_dict['data_tables'].get( data_table, [] ) + data_manager_dict['data_tables'][data_table].append( data_table_entry ) + return data_manager_dict + +def main(): + #Parse Command Line + parser = optparse.OptionParser() + parser.add_option( '-j', '--jar_path', dest='jar_path', action='store', type="string", default=None, help='snpEff.jar path' ) + parser.add_option( '-c', '--config', dest='config', action='store', type="string", default=None, help='snpEff.config path' ) + parser.add_option( '-g', '--genome_version', dest='genome_version', action='store', type="string", default=None, help='genome_version' ) + parser.add_option( '-o', '--organism', dest='organism', action='store', type="string", default=None, help='organism name' ) + (options, args) = parser.parse_args() + + filename = args[0] + + params = from_json_string( open( filename ).read() ) + target_directory = params[ 'output_data' ][0]['extra_files_path'] + os.mkdir( target_directory ) + data_manager_dict = {} + + + #Create SnpEff Reference Data + for genome_version, organism in zip(options.genome_version.split(','), getOrganismNames(options.jar_path,options.genome_version,options.organism).split(',')): + download_database( data_manager_dict, target_directory, options.jar_path, options.config, genome_version, organism ) + + #save info to json file + open( filename, 'wb' ).write( to_json_string( data_manager_dict ) ) + +if __name__ == "__main__": main() +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/data_manager/data_manager_gemini_download.xml Sat Aug 16 11:03:57 2014 -0400 @@ -0,0 +1,62 @@ +<tool id="data_manager_gemini_download" name="GEMINI Download" version="0.9.1" tool_type="manage_data"> + <description>Download a new database</description> + <requirements> + <requirement type="package" version="0.9.1">gemini</requirement> + </requirements> + <command interpreter="python"> + #from galaxy.util.json import from_json_string, to_json_string + + ##\$GEMINI_ROOT_DIR/gemini/gemini/install-data.py /home/bag/test/gdata/ $gerp_bp $cadd_score + + #set $params = from_json_string( open( '$out_file' ).read() ) + #set $target_directory = $params[ 'output_data' ][0]['extra_files_path'] + mkdir $target_directory + #set $data_manager_dict = { 'data_tables': {'data_table': [ {'value': 'bar', 'name': 'foo', 'path': '$target_directory'] } } + + data_manager_dict['data_tables'] = data_manager_dict.get( 'data_tables', {} ) + data_manager_dict['data_tables'][data_table] = data_manager_dict['data_tables'].get( data_table, [] ) + data_manager_dict['data_tables'][data_table].append( data_table_entry ) + + + #silent open( '$out_file', 'wb' ).write( to_json_string( $data_manager_dict ) ) + +## data_manager_snpEff_download.py --jar_path \$SNPEFF_JAR_PATH/snpEff.jar --config \$SNPEFF_JAR_PATH/snpEff.config +## #if $snpDb.genomeSrc == "named": +## --genome_version "${snpDb.genome_version}" +## #else +## --genome_version "${snpDb.genome_databases.fields.value}" +## --organism "${snpDb.genome_databases.fields.name}" +## #end if +## "$out_file" + </command> + <inputs> + + <param name="cadd" type="boolean" truevalue="--extra cadd_score" falsevalue="" checked="True" + label="Download CADD scores for GEMINI database annotation" help="(--extra cadd_score)"/> + + <param name="gerp_bp" type="boolean" truevalue="--extra cadd_score" falsevalue="" checked="True" + label="Download GERP for GEMINI database annotation" help="(--extra gerp_bp)"/> + + </inputs> + <outputs> + <data name="out_file" format="data_manager_json" label="${tool.name}"/> + </outputs> + <stdio> + <exit_code range=":-1" level="fatal" description="Error: Cannot open file" /> + <exit_code range="1:" level="fatal" description="Error" /> + </stdio> + <tests> + </tests> + <help> + +This tool downloads the GEMINI databases. + +For details about this tool, please go to http://http://gemini.readthedocs.org + + </help> + <citations> + <citation type="doi">10.1371/journal.pcbi.1003153</citation> + <yield /> + </citations> +</tool> +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/data_manager_conf.xml Sat Aug 16 11:03:57 2014 -0400 @@ -0,0 +1,20 @@ +<?xml version="1.0"?> +<data_managers> + <data_manager tool_file="data_manager/data_manager_gemini_download.xml" id="data_manager_gemini_download" > + <data_table name="gemini_database"> <!-- Defines a Data Table to be modified. --> + <output> <!-- Handle the output of the Data Manager Tool --> + <column name="value" /> <!-- columns that are going to be specified by the Data Manager Tool --> + <column name="name" /> <!-- columns that are going to be specified by the Data Manager Tool --> + <column name="path" output_ref="out_file" > + <move type="directory" relativize_symlinks="True"> + <target base="${GALAXY_DATA_MANAGER_DATA_PATH}">gemini/data</target> + </move> + <value_translation>${GALAXY_DATA_MANAGER_DATA_PATH}/gemini/data</value_translation> + <value_translation type="function">abspath</value_translation> + </column> + </output> + </data_table> + </data_manager> +</data_managers> + +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tool-data/gemini_databases.loc.sample Sat Aug 16 11:03:57 2014 -0400 @@ -0,0 +1,3 @@ +## GEMINI databases +#Version Description Path +#08_08_2014 Database (08-08-2014) /data/0/gemini/08_08_2014/
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tool_data_table_conf.xml.sample Sat Aug 16 11:03:57 2014 -0400 @@ -0,0 +1,7 @@ +<tables> + <table name="gemini_databases" comment_char="#"> + <columns>value, name</columns> + <file path="tool-data/gemini_databases.loc" /> + </table> +</tables> +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tool_dependencies.xml Sat Aug 16 11:03:57 2014 -0400 @@ -0,0 +1,6 @@ +<?xml version="1.0"?> +<tool_dependency> + <package name="gemini" version="0.9.1"> + <repository changeset_revision="610ebefc0db1" name="package_gemini_0_9_1" owner="iuc" toolshed="https://testtoolshed.g2.bx.psu.edu" /> + </package> +</tool_dependency>