Mercurial > repos > jjohnson > data_manager_snpeff

changeset 0:bc50aa3b328a

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Uploaded
author jjohnson
date Thu, 26 Jun 2014 09:13:02 -0400
parents
children 8ac7bbaac976
files data_manager/data_manager_snpEff_databases.py data_manager/data_manager_snpEff_databases.xml data_manager/data_manager_snpEff_download.py data_manager/data_manager_snpEff_download.xml data_manager_conf.xml repository_dependencies.xml tool-data/snpeff_annotations.loc.sample tool-data/snpeff_databases.loc.sample tool-data/snpeff_genomedb.loc.sample tool-data/snpeff_regulationdb.loc.sample tool_data_table_conf.xml.sample tool_dependencies.xml
diffstat 12 files changed, 443 insertions(+), 0 deletions(-) [+]
line wrap: on
line diff
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/data_manager/data_manager_snpEff_databases.py	Thu Jun 26 09:13:02 2014 -0400
@@ -0,0 +1,82 @@
+#!/usr/bin/env python
+
+import sys
+import os
+import re
+import tempfile
+import subprocess
+import fileinput
+import shutil
+import optparse
+import urllib2
+from ftplib import FTP
+import tarfile
+
+from galaxy.util.json import from_json_string, to_json_string
+
+def stop_err(msg):
+    sys.stderr.write(msg)
+    sys.exit(1)
+
+def fetch_databases(data_manager_dict, target_directory, jar_path):
+    (snpEff_dir,snpEff_jar) = os.path.split(jar_path)
+    if not os.path.exists(target_directory):
+        os.makedirs(target_directory)
+    databases_path = os.path.join( target_directory, 'databases.out' )
+    databases_output = open(databases_path,'w')
+    args = [ 'java','-jar', ]
+    args.append( snpEff_jar )
+    args.append( 'databases' )
+    # tmp_stderr = tempfile.NamedTemporaryFile( prefix = "tmp-data-manager-snpEff-stderr" )
+    # databases_output = open(databases_path)
+    # proc = subprocess.Popen( args=args, shell=False, cwd=snpEff_dir, stdout=databases_output.fileno(), stderr=tmp_stderr.fileno() )
+    proc = subprocess.Popen( args=args, shell=False, cwd=snpEff_dir, stdout=databases_output.fileno() )
+    return_code = proc.wait()
+    if return_code:
+        sys.exit( return_code )
+    databases_output.close()
+    try:
+        data_manager_dict['data_tables'] = data_manager_dict.get( 'data_tables', {} )
+        data_manager_dict['data_tables']['snpeff_databases'] = data_manager_dict['data_tables'].get( 'snpeff_databases', [] )
+        data_table_entries = []
+        fh = open(databases_path,'r')
+        for i,line in enumerate(fh):
+            fields = line.split('\t')
+            if len(fields) >= 2:
+                genome_version = fields[0].strip()
+                if genome_version.startswith("Genome") or genome_version.startswith("-"):
+                    continue
+                #snpeff test genome
+                if genome_version == '30c2c903' or fields[1].strip() == 'TestCase' or fields[1].strip().startswith('Test_'):
+                    continue
+                description = fields[1].strip() + ' : ' + genome_version
+                data_table_entries.append(dict(value=genome_version, name=description))
+        data_manager_dict['data_tables']['snpeff_databases'] = data_table_entries
+    except Exception, e:
+        stop_err( 'Error parsing %s %s\n' % (config,str( e )) )
+    else:
+        fh.close()
+    return data_manager_dict
+
+def main():
+    #Parse Command Line
+    parser = optparse.OptionParser()
+    parser.add_option( '-j', '--jar_path', dest='jar_path', action='store', type="string", default=None, help='snpEff.jar path' )
+    (options, args) = parser.parse_args()
+
+    filename = args[0]
+
+    params = from_json_string( open( filename ).read() )
+    target_directory = params[ 'output_data' ][0]['extra_files_path']
+    os.mkdir( target_directory )
+    data_manager_dict = {}
+
+
+    #Create Defuse Reference Data
+    data_manager_dict = fetch_databases( data_manager_dict, target_directory, options.jar_path)
+
+    #save info to json file
+    open( filename, 'wb' ).write( to_json_string( data_manager_dict ) )
+
+if __name__ == "__main__": main()
+
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/data_manager/data_manager_snpEff_databases.xml	Thu Jun 26 09:13:02 2014 -0400
@@ -0,0 +1,41 @@
+<tool id="data_manager_snpeff_databases" name="SnpEff Databases" version="3.6" tool_type="manage_data">
+	<description>Read the list of available snpEff databases</description>
+	<requirements>
+		<requirement type="package" version="3.6">snpEff</requirement>
+	</requirements>
+	<command interpreter="python">
+        data_manager_snpEff_databases.py --jar_path \$SNPEFF_JAR_PATH/snpEff.jar "$out_file"
+        </command>
+	<inputs>
+	</inputs>
+	<outputs>
+           <data name="out_file" format="data_manager_json"/>
+	</outputs>
+        <stdio>
+          <exit_code range=":-1"  level="fatal"   description="Error: Cannot open file" />
+          <exit_code range="1:"  level="fatal"   description="Error" />
+        </stdio>
+        <tests>
+            <test>
+                <output name="out_file">
+                    <assert_contents>
+                        <!-- Check that a genome was added -->
+                        <has_text text="GRCh37.72" />
+                    </assert_contents>
+                </output>
+            </test>
+        </tests>
+	<help>
+
+This tool updatess the list of SnpEff databases for the SnpEff Download data manager.
+It should only need to be run once for a snpEff version, 
+since it populates the SnpEff Download data manager from the snpEff config file.
+
+For information about snpEff:    http://snpEff.sourceforge.net
+
+Please cite:
+"A program for annotating and predicting the effects of single nucleotide polymorphisms, SnpEff: SNPs in the genome of Drosophila melanogaster strain w1118; iso-2; iso-3.", Cingolani P, Platts A, Wang le L, Coon M, Nguyen T, Wang L, Land SJ, Lu X, Ruden DM. Fly (Austin). 2012 Apr-Jun;6(2):80-92. PMID: 22728672 [PubMed - in process]
+
+	</help>
+</tool>
+
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/data_manager/data_manager_snpEff_download.py	Thu Jun 26 09:13:02 2014 -0400
@@ -0,0 +1,158 @@
+#!/usr/bin/env python
+
+import sys
+import os
+import re
+import tempfile
+import subprocess
+import fileinput
+import shutil
+import optparse
+import urllib2
+from ftplib import FTP
+import tarfile
+
+from galaxy.util.json import from_json_string, to_json_string
+
+def stop_err(msg):
+    sys.stderr.write(msg)
+    sys.exit(1)
+
+
+def fetch_databases(jar_path,genome_list=None):
+    snpDBs = dict()
+    (snpEff_dir,snpEff_jar) = os.path.split(jar_path)
+    databases_path = 'databases.out' 
+    databases_output = open(databases_path,'w')
+    args = [ 'java','-jar', ]
+    args.append( snpEff_jar )
+    args.append( 'databases' )
+    # tmp_stderr = tempfile.NamedTemporaryFile( prefix = "tmp-data-manager-snpEff-stderr" )
+    # databases_output = open(databases_path)
+    # proc = subprocess.Popen( args=args, shell=False, cwd=snpEff_dir, stdout=databases_output.fileno(), stderr=tmp_stderr.fileno() )
+    proc = subprocess.Popen( args=args, shell=False, cwd=snpEff_dir, stdout=databases_output.fileno() )
+    return_code = proc.wait()
+    if return_code:
+        sys.exit( return_code )
+    databases_output.close()
+    try:
+        fh = open(databases_path,'r')
+        for i,line in enumerate(fh):
+            fields = line.split('\t')
+            if len(fields) >= 2:
+                genome_version = fields[0].strip()
+                if genome_list and genome_version not in genome_list:
+                    continue
+                if genome_version.startswith("Genome") or genome_version.startswith("-"):
+                    continue
+                description = fields[1].strip()
+                snpDBs[genome_version] = description;
+    except Exception, e:
+        stop_err( 'Error parsing %s %s\n' % (config,str( e )) )
+    else:
+        fh.close()
+    return snpDBs
+
+def getOrganismNames(jar_path,genomes,organisms) :
+    genome_list = genomes.split(',')
+    organism_list = organisms.split(',') if organisms else []
+    if len(genome_list) != len(organism_list):
+        descriptions = []
+        snpDBdict = fetch_databases(jar_path,genome_list=genome_list); 
+        for genome in snpDBdict:
+            descriptions.append(snpDBdict[genome] if genome in snpDBdict else genome)
+        return ','.join(descriptions)
+    return organisms    
+
+"""
+# Download human database 'hg19'
+java -jar snpEff.jar download -v hg19
+
+        <command>java -jar \$SNPEFF_JAR_PATH/snpEff.jar download -c \$JAVA_JAR_PATH/snpEff.config $genomeVersion > $logfile </command>
+
+snpEffectPredictor.bin
+regulation_HeLa-S3.bin
+regulation_pattern = 'regulation_(.+).bin'
+"""
+def download_database(data_manager_dict, target_directory, jar_path,config,genome_version,organism):
+    ## get data_dir from config 
+    ##---
+    ## Databases are stored here
+    ## E.g.: Information for 'hg19' is stored in data_dir/hg19/
+    ##
+    ## Note: Since version 2.1 you can use tilde ('~') as first character to refer to your home directory
+    ##---
+    #data_dir = ~/snpEff/data/
+    data_dir = target_directory
+    (snpEff_dir,snpEff_jar) = os.path.split(jar_path)
+    args = [ 'java','-jar' ]
+    args.append( jar_path )
+    args.append( 'download' )
+    args.append( '-c' )
+    args.append( config )
+    args.append( '-dataDir' )
+    args.append( data_dir )
+    args.append( '-v' )
+    args.append( genome_version )
+    proc = subprocess.Popen( args=args, shell=False, cwd=snpEff_dir )
+    return_code = proc.wait()
+    if return_code:
+        sys.exit( return_code )
+    ## search data_dir/genome_version for files
+    regulation_pattern = 'regulation_(.+).bin'
+    #  annotation files that are included in snpEff by a flag
+    annotations_dict = {'nextProt.bin' : '-nextprot','motif.bin': '-motif'}
+    genome_path = os.path.join(data_dir,genome_version)
+    if os.path.isdir(genome_path):
+        for root, dirs, files in os.walk(genome_path):
+            for fname in files:
+                if fname.startswith('snpEffectPredictor'):
+                    # if snpEffectPredictor.bin download succeeded
+                    name = genome_version + (' : ' + organism if organism else '') 
+                    data_table_entry = dict(value=genome_version, name=name, path=data_dir)
+                    _add_data_table_entry( data_manager_dict, 'snpeff_genomedb', data_table_entry )
+                else:
+                    m = re.match(regulation_pattern,fname)
+                    if m:
+                        name = m.groups()[0]
+                        data_table_entry = dict(genome=genome_version,value=name, name=name)
+                        _add_data_table_entry( data_manager_dict, 'snpeff_regulationdb', data_table_entry )
+                    elif fname in annotations_dict:
+                        value = annotations_dict[fname]
+                        name = value.lstrip('-')
+                        data_table_entry = dict(genome=genome_version,value=value, name=name)
+                        _add_data_table_entry( data_manager_dict, 'snpeff_annotations', data_table_entry )
+    return data_manager_dict
+
+def _add_data_table_entry( data_manager_dict, data_table, data_table_entry ):
+    data_manager_dict['data_tables'] = data_manager_dict.get( 'data_tables', {} )
+    data_manager_dict['data_tables'][data_table] = data_manager_dict['data_tables'].get( data_table, [] )
+    data_manager_dict['data_tables'][data_table].append( data_table_entry )
+    return data_manager_dict
+
+def main():
+    #Parse Command Line
+    parser = optparse.OptionParser()
+    parser.add_option( '-j', '--jar_path', dest='jar_path', action='store', type="string", default=None, help='snpEff.jar path' )
+    parser.add_option( '-c', '--config', dest='config', action='store', type="string", default=None, help='snpEff.config path' )
+    parser.add_option( '-g', '--genome_version', dest='genome_version', action='store', type="string", default=None, help='genome_version' )
+    parser.add_option( '-o', '--organism', dest='organism', action='store', type="string", default=None, help='organism name' )
+    (options, args) = parser.parse_args()
+
+    filename = args[0]
+
+    params = from_json_string( open( filename ).read() )
+    target_directory = params[ 'output_data' ][0]['extra_files_path']
+    os.mkdir( target_directory )
+    data_manager_dict = {}
+
+
+    #Create SnpEff Reference Data
+    for genome_version, organism in zip(options.genome_version.split(','), getOrganismNames(options.jar_path,options.genome_version,options.organism).split(',')):
+        download_database( data_manager_dict, target_directory, options.jar_path, options.config, genome_version, organism )
+
+    #save info to json file
+    open( filename, 'wb' ).write( to_json_string( data_manager_dict ) )
+
+if __name__ == "__main__": main()
+
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/data_manager/data_manager_snpEff_download.xml	Thu Jun 26 09:13:02 2014 -0400
@@ -0,0 +1,71 @@
+<tool id="data_manager_snpeff_download" name="SnpEff Download" version="3.6" tool_type="manage_data">
+    <description>Download a new database</description>
+    <requirements>
+        <requirement type="package" version="3.6">snpEff</requirement>
+    </requirements>
+    <command interpreter="python">
+        data_manager_snpEff_download.py --jar_path \$SNPEFF_JAR_PATH/snpEff.jar --config \$SNPEFF_JAR_PATH/snpEff.config 
+        #if $snpDb.genomeSrc == "named":
+          --genome_version "${snpDb.genome_version}"
+        #else
+          --genome_version "${snpDb.genome_databases.fields.value}"
+          --organism "${snpDb.genome_databases.fields.name}"
+        #end if
+        "$out_file"
+        </command>
+    <inputs>
+        <conditional name="snpDb">
+            <param name="genomeSrc" type="select" label="Select Genome Version">
+                <help>The list is very large and may take some time to display on your browser</help>
+                <option value="named">By Genome Version Name</option>
+                <option value="cached">Select from list</option>
+            </param>
+            <when value="named">
+                <param name="genome_version" type="text" size="40" value="" label="Snpff Genome Version Name (e.g. GRCh37.74)">
+                    <help>http://sourceforge.net/projects/snpeff/files/databases/</help>
+                    <validator type="regex" message="A genome version name is required">\S+</validator>
+                </param>
+            </when>
+            <when value="cached">
+                <param name="genome_databases" type="select" display="checkboxes" multiple="true" label="Genome Version">
+                    <options from_data_table="snpeff_databases">
+                        <filter type="sort_by" column="0" />
+                    </options>
+                </param>
+            </when>
+        </conditional>
+    </inputs>
+
+    <outputs>
+           <data name="out_file" format="data_manager_json" label="${tool.name}"/>
+    </outputs>
+    <stdio>
+        <exit_code range=":-1"  level="fatal"   description="Error: Cannot open file" />
+        <exit_code range="1:"  level="fatal"   description="Error" />
+    </stdio>
+    <tests>
+        <test>
+            <param name="genomeSrc"  value="named"/>
+            <param name="genome_version" value="GRCh37.71"/>
+            <output name="out_file">
+                <assert_contents>
+                    <!-- Check that a genome was added -->
+                    <has_text text="GRCh37.71" />
+                    <has_text text="snpeff_regulationdb" />
+                    <has_text text="snpeff_annotations" />
+                </assert_contents>
+            </output>
+        </test>
+    </tests>
+    <help>
+
+This tool downloads a SnpEff database.
+
+For details about this tool, please go to http://snpEff.sourceforge.net
+
+Please cite:
+"A program for annotating and predicting the effects of single nucleotide polymorphisms, SnpEff: SNPs in the genome of Drosophila melanogaster strain w1118; iso-2; iso-3.", Cingolani P, Platts A, Wang le L, Coon M, Nguyen T, Wang L, Land SJ, Lu X, Ruden DM. Fly (Austin). 2012 Apr-Jun;6(2):80-92. PMID: 22728672 [PubMed - in process]
+
+    </help>
+</tool>
+
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/data_manager_conf.xml	Thu Jun 26 09:13:02 2014 -0400
@@ -0,0 +1,42 @@
+<?xml version="1.0"?>
+<data_managers>
+  <data_manager tool_file="data_manager/data_manager_snpEff_databases.xml" id="data_manager_snpeff_databases" >
+    <data_table name="snpeff_databases">  <!-- Defines a Data Table to be modified. -->
+      <output> <!-- Handle the output of the Data Manager Tool -->
+        <column name="value" /> <!-- columns that are going to be specified by the Data Manager Tool -->
+        <column name="name" /> <!-- columns that are going to be specified by the Data Manager Tool -->
+      </output>
+    </data_table>
+  </data_manager>
+  <data_manager tool_file="data_manager/data_manager_snpEff_download.xml" id="data_manager_snpeff_download" >
+    <data_table name="snpeff_genomedb">  <!-- Defines a Data Table to be modified. -->
+      <output> <!-- Handle the output of the Data Manager Tool -->
+        <column name="value" /> <!-- columns that are going to be specified by the Data Manager Tool -->
+        <column name="name" />  <!-- columns that are going to be specified by the Data Manager Tool -->
+        <column name="path" output_ref="out_file" >
+          <move type="directory" relativize_symlinks="True">
+            <target base="${GALAXY_DATA_MANAGER_DATA_PATH}">snpEff/data</target>
+          </move>
+          <value_translation>${GALAXY_DATA_MANAGER_DATA_PATH}/snpEff/data</value_translation>
+          <value_translation type="function">abspath</value_translation>
+        </column>
+      </output>
+    </data_table>
+    <data_table name="snpeff_regulationdb">  <!-- Defines a Data Table to be modified. -->
+      <output> <!-- Handle the output of the Data Manager Tool -->
+        <column name="genome" /> <!-- columns that are going to be specified by the Data Manager Tool -->
+        <column name="value" /> <!-- columns that are going to be specified by the Data Manager Tool -->
+        <column name="name" />  <!-- columns that are going to be specified by the Data Manager Tool -->
+      </output>
+    </data_table>
+    <data_table name="snpeff_annotations">  <!-- Defines a Data Table to be modified. -->
+      <output> <!-- Handle the output of the Data Manager Tool -->
+        <column name="genome" /> <!-- columns that are going to be specified by the Data Manager Tool -->
+        <column name="value" /> <!-- columns that are going to be specified by the Data Manager Tool -->
+        <column name="name" />  <!-- columns that are going to be specified by the Data Manager Tool -->
+      </output>
+    </data_table>
+  </data_manager>
+</data_managers>
+
+
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/repository_dependencies.xml	Thu Jun 26 09:13:02 2014 -0400
@@ -0,0 +1,4 @@
+<?xml version="1.0"?>
+<repositories description="This requires the SnpEff datatype definitions.">
+    <repository name="snpeff_datatypes" owner="jjohnson" changeset_revision="cd8f8c54bf9a" toolshed="http://testtoolshed.g2.bx.psu.edu/" />
+</repositories>
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tool-data/snpeff_annotations.loc.sample	Thu Jun 26 09:13:02 2014 -0400
@@ -0,0 +1,5 @@
+## Regulation Databases for SnpEff 
+## These are from the list on: http://snpeff.sourceforge.net/download.html
+#genome	annotation_name description
+#GRCh37.71	nextprot	nextprot
+#GRCh37.71	motif	motif
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tool-data/snpeff_databases.loc.sample	Thu Jun 26 09:13:02 2014 -0400
@@ -0,0 +1,5 @@
+## Available Databases for SnpEff 
+## These are from the list on: http://snpeff.sourceforge.net/download.html
+## the Description field in this sample is "Genome : Version" 
+#Version	Description
+#GRCh37.68	Homo sapiens : GRCh37.68
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tool-data/snpeff_genomedb.loc.sample	Thu Jun 26 09:13:02 2014 -0400
@@ -0,0 +1,5 @@
+## Downloaded Databases for SnpEff 
+## These are from the list on: http://snpeff.sourceforge.net/download.html
+## the Description field in this sample is "Genome : Version" 
+#Version        Description	data_dir path
+#GRCh37.68      Homo sapiens : GRCh37.68	/home/galaxy/snpEff/data
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tool-data/snpeff_regulationdb.loc.sample	Thu Jun 26 09:13:02 2014 -0400
@@ -0,0 +1,4 @@
+## Regulation Databases for SnpEff 
+## These are from the list on: http://snpeff.sourceforge.net/download.html
+#genome	regulation_name description
+#GRCh37.70	CD4	CD4
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tool_data_table_conf.xml.sample	Thu Jun 26 09:13:02 2014 -0400
@@ -0,0 +1,19 @@
+<tables>
+    <table name="snpeff_databases" comment_char="#">
+        <columns>value, name</columns>
+        <file path="tool-data/snpeff_databases.loc" />
+    </table>
+    <table name="snpeff_genomedb" comment_char="#">
+        <columns>value, name, path</columns>
+        <file path="tool-data/snpeff_genomedb.loc" />
+    </table>
+    <table name="snpeff_regulationdb" comment_char="#">
+        <columns>genome, value, name</columns>
+        <file path="tool-data/snpeff_regulationdb.loc" />
+    </table>
+    <table name="snpeff_annotations" comment_char="#">
+        <columns>genome, value, name</columns>
+        <file path="tool-data/snpeff_annotations.loc" />
+    </table>
+</tables>
+
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tool_dependencies.xml	Thu Jun 26 09:13:02 2014 -0400
@@ -0,0 +1,7 @@
+<?xml version="1.0"?>
+<tool_dependency>
+    <package name="snpEff" version="3.6">
+        <repository name="package_snpeff_3_6" owner="jjohnson" changeset_revision="849895b37940" toolshed="http://testtoolshed.g2.bx.psu.edu/" />
+    </package>
+</tool_dependency>
+