Mercurial > repos > iuc > data_manager_gemini_database_downloader

--- a/data_manager/data_manager_gemini_download.py	Sat Aug 16 11:03:57 2014 -0400
+++ b/data_manager/data_manager_gemini_download.py	Sat Aug 16 12:33:49 2014 -0400
@@ -2,157 +2,19 @@

 import sys
 import os
-import re
-import tempfile
-import subprocess
-import fileinput
-import shutil
-import optparse
-import urllib2
-from ftplib import FTP
-import tarfile
-
-from galaxy.util.json import from_json_string, to_json_string
-
-def stop_err(msg):
-    sys.stderr.write(msg)
-    sys.exit(1)
-
-
-def fetch_databases(jar_path,genome_list=None):
-    snpDBs = dict()
-    (snpEff_dir,snpEff_jar) = os.path.split(jar_path)
-    databases_path = 'databases.out'
-    databases_output = open(databases_path,'w')
-    args = [ 'java','-jar', ]
-    args.append( snpEff_jar )
-    args.append( 'databases' )
-    # tmp_stderr = tempfile.NamedTemporaryFile( prefix = "tmp-data-manager-snpEff-stderr" )
-    # databases_output = open(databases_path)
-    # proc = subprocess.Popen( args=args, shell=False, cwd=snpEff_dir, stdout=databases_output.fileno(), stderr=tmp_stderr.fileno() )
-    proc = subprocess.Popen( args=args, shell=False, cwd=snpEff_dir, stdout=databases_output.fileno() )
-    return_code = proc.wait()
-    if return_code:
-        sys.exit( return_code )
-    databases_output.close()
-    try:
-        fh = open(databases_path,'r')
-        for i,line in enumerate(fh):
-            fields = line.split('\t')
-            if len(fields) >= 2:
-                genome_version = fields[0].strip()
-                if genome_list and genome_version not in genome_list:
-                    continue
-                if genome_version.startswith("Genome") or genome_version.startswith("-"):
-                    continue
-                description = fields[1].strip()
-                snpDBs[genome_version] = description;
-    except Exception, e:
-        stop_err( 'Error parsing %s %s\n' % (config,str( e )) )
-    else:
-        fh.close()
-    return snpDBs
-
-def getOrganismNames(jar_path,genomes,organisms) :
-    genome_list = genomes.split(',')
-    organism_list = organisms.split(',') if organisms else []
-    if len(genome_list) != len(organism_list):
-        descriptions = []
-        snpDBdict = fetch_databases(jar_path,genome_list=genome_list);
-        for genome in snpDBdict:
-            descriptions.append(snpDBdict[genome] if genome in snpDBdict else genome)
-        return ','.join(descriptions)
-    return organisms
-
-"""
-# Download human database 'hg19'
-java -jar snpEff.jar download -v hg19
-
-        <command>java -jar \$SNPEFF_JAR_PATH/snpEff.jar download -c \$JAVA_JAR_PATH/snpEff.config $genomeVersion > $logfile </command>
-
-snpEffectPredictor.bin
-regulation_HeLa-S3.bin
-regulation_pattern = 'regulation_(.+).bin'
-"""
-def download_database(data_manager_dict, target_directory, jar_path,config,genome_version,organism):
-    ## get data_dir from config
-    ##---
-    ## Databases are stored here
-    ## E.g.: Information for 'hg19' is stored in data_dir/hg19/
-    ##
-    ## Note: Since version 2.1 you can use tilde ('~') as first character to refer to your home directory
-    ##---
-    #data_dir = ~/snpEff/data/
-    data_dir = target_directory
-    (snpEff_dir,snpEff_jar) = os.path.split(jar_path)
-    args = [ 'java','-jar' ]
-    args.append( jar_path )
-    args.append( 'download' )
-    args.append( '-c' )
-    args.append( config )
-    args.append( '-dataDir' )
-    args.append( data_dir )
-    args.append( '-v' )
-    args.append( genome_version )
-    proc = subprocess.Popen( args=args, shell=False, cwd=snpEff_dir )
-    return_code = proc.wait()
-    if return_code:
-        sys.exit( return_code )
-    ## search data_dir/genome_version for files
-    regulation_pattern = 'regulation_(.+).bin'
-    #  annotation files that are included in snpEff by a flag
-    annotations_dict = {'nextProt.bin' : '-nextprot','motif.bin': '-motif'}
-    genome_path = os.path.join(data_dir,genome_version)
-    if os.path.isdir(genome_path):
-        for root, dirs, files in os.walk(genome_path):
-            for fname in files:
-                if fname.startswith('snpEffectPredictor'):
-                    # if snpEffectPredictor.bin download succeeded
-                    name = genome_version + (' : ' + organism if organism else '')
-                    data_table_entry = dict(value=genome_version, name=name, path=data_dir)
-                    _add_data_table_entry( data_manager_dict, 'snpeff_genomedb', data_table_entry )
-                else:
-                    m = re.match(regulation_pattern,fname)
-                    if m:
-                        name = m.groups()[0]
-                        data_table_entry = dict(genome=genome_version,value=name, name=name)
-                        _add_data_table_entry( data_manager_dict, 'snpeff_regulationdb', data_table_entry )
-                    elif fname in annotations_dict:
-                        value = annotations_dict[fname]
-                        name = value.lstrip('-')
-                        data_table_entry = dict(genome=genome_version,value=value, name=name)
-                        _add_data_table_entry( data_manager_dict, 'snpeff_annotations', data_table_entry )
-    return data_manager_dict
-
-def _add_data_table_entry( data_manager_dict, data_table, data_table_entry ):
-    data_manager_dict['data_tables'] = data_manager_dict.get( 'data_tables', {} )
-    data_manager_dict['data_tables'][data_table] = data_manager_dict['data_tables'].get( data_table, [] )
-    data_manager_dict['data_tables'][data_table].append( data_table_entry )
-    return data_manager_dict
+import json

 def main():
-    #Parse Command Line
-    parser = optparse.OptionParser()
-    parser.add_option( '-j', '--jar_path', dest='jar_path', action='store', type="string", default=None, help='snpEff.jar path' )
-    parser.add_option( '-c', '--config', dest='config', action='store', type="string", default=None, help='snpEff.config path' )
-    parser.add_option( '-g', '--genome_version', dest='genome_version', action='store', type="string", default=None, help='genome_version' )
-    parser.add_option( '-o', '--organism', dest='organism', action='store', type="string", default=None, help='organism name' )
-    (options, args) = parser.parse_args()

-    filename = args[0]
-
-    params = from_json_string( open( filename ).read() )
+    params = json.loads( open( sys.argv[1] ).read() )
     target_directory = params[ 'output_data' ][0]['extra_files_path']
     os.mkdir( target_directory )
-    data_manager_dict = {}
-
+    data_manager_dict = { 'data_tables': {'gemini_databases': [ {'value': sys.argv[2], 'name': sys.argv[3], 'path': target_directory }] } }

-    #Create SnpEff Reference Data
-    for genome_version, organism in zip(options.genome_version.split(','), getOrganismNames(options.jar_path,options.genome_version,options.organism).split(',')):
-        download_database( data_manager_dict, target_directory, options.jar_path, options.config, genome_version, organism )
+     #save info to json file
+    with open( sys.argv[1], 'wb' ) as out:
+        out.write( json.dumps( data_manager_dict ) )

-    #save info to json file
-    open( filename, 'wb' ).write( to_json_string( data_manager_dict ) )
+if __name__ == "__main__":
+    main()

-if __name__ == "__main__": main()
-
--- a/data_manager/data_manager_gemini_download.xml	Sat Aug 16 11:03:57 2014 -0400
+++ b/data_manager/data_manager_gemini_download.xml	Sat Aug 16 12:33:49 2014 -0400
@@ -7,18 +7,9 @@
         #from galaxy.util.json import from_json_string, to_json_string

         ##\$GEMINI_ROOT_DIR/gemini/gemini/install-data.py /home/bag/test/gdata/ $gerp_bp $cadd_score
-
-        #set $params = from_json_string( open( '$out_file' ).read() )
-        #set $target_directory = $params[ 'output_data' ][0]['extra_files_path']
-        mkdir $target_directory
-        #set $data_manager_dict = { 'data_tables': {'data_table': [ {'value': 'bar', 'name': 'foo', 'path': '$target_directory'] } }
+        data_manager_gemini_download.py "$out_file" "foo" "bar"

-    data_manager_dict['data_tables'] = data_manager_dict.get( 'data_tables', {} )
-    data_manager_dict['data_tables'][data_table] = data_manager_dict['data_tables'].get( data_table, [] )
-    data_manager_dict['data_tables'][data_table].append( data_table_entry )
-
-
-        #silent open( '$out_file', 'wb' ).write( to_json_string( $data_manager_dict ) )
+        ##silent open( "${out_file}", 'wb' ).write( to_json_string( $data_manager_dict ) )

 ##        data_manager_snpEff_download.py --jar_path \$SNPEFF_JAR_PATH/snpEff.jar --config \$SNPEFF_JAR_PATH/snpEff.config
 ##        #if $snpDb.genomeSrc == "named":
@@ -42,8 +33,8 @@
            <data name="out_file" format="data_manager_json" label="${tool.name}"/>
     </outputs>
     <stdio>
-        <exit_code range=":-1"  level="fatal"   description="Error: Cannot open file" />
-        <exit_code range="1:"  level="fatal"   description="Error" />
+        <exit_code range=":-1"  level="fatal" description="Error: Cannot open file" />
+        <exit_code range="1:"  level="fatal" description="Error" />
     </stdio>
     <tests>
     </tests>
--- a/data_manager_conf.xml	Sat Aug 16 11:03:57 2014 -0400
+++ b/data_manager_conf.xml	Sat Aug 16 12:33:49 2014 -0400
@@ -1,7 +1,7 @@
 <?xml version="1.0"?>
 <data_managers>
   <data_manager tool_file="data_manager/data_manager_gemini_download.xml" id="data_manager_gemini_download" >
-    <data_table name="gemini_database">  <!-- Defines a Data Table to be modified. -->
+    <data_table name="gemini_databases">  <!-- Defines a Data Table to be modified. -->
       <output> <!-- Handle the output of the Data Manager Tool -->
         <column name="value" /> <!-- columns that are going to be specified by the Data Manager Tool -->
         <column name="name" />  <!-- columns that are going to be specified by the Data Manager Tool -->