Mercurial > repos > sh477 > data_manager_vep_cache_downloader
changeset 2:612026ea6db5 draft
Uploaded first real version
| author | sh477 |
|---|---|
| date | Tue, 22 Feb 2022 16:26:44 +0000 |
| parents | b0c0733f58da |
| children | a7f871f7d3df |
| files | .shed.yml data_manager/data_manager_vep_cache_download.py data_manager/data_manager_vep_cache_download.xml data_manager_conf.xml data_manager_conf.xml.sample data_manager_conf.xml.test test-data/dbkeys.loc test-data/from_test-meta.data_manager.json test-data/vep_versioned_caches.loc tool-data/dbkeys.loc.sample tool-data/vep_versioned_caches.loc.sample |
| diffstat | 11 files changed, 170 insertions(+), 0 deletions(-) [+] |
line wrap: on
line diff
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/.shed.yml Tue Feb 22 16:26:44 2022 +0000 @@ -0,0 +1,11 @@ +categories: +- Data Managers +description: Download and install annotation cache files for Ensembl VEP +long_description: | + This tool downloads given versions of VEP cache annotation files and makes + them available to Ensembl VEP in Galaxy via the "vep_versioned_caches" data + table. +name: data_manager_vep_cache_downloader +owner: sh477 +remote_repository_url: +type: unrestricted \ No newline at end of file
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/data_manager/data_manager_vep_cache_download.py Tue Feb 22 16:26:44 2022 +0000 @@ -0,0 +1,54 @@ +#!/usr/bin/env python + +import datetime +import json +import os +import re +from urllib.request import urlretrieve +import sys +import tarfile + + +def main(): + # Read in given out_file and create target directory for file download + with open(sys.argv[1]) as fh: + params = json.load(fh) + target_directory = params['output_data'][0]['extra_files_path'] + os.mkdir(target_directory) + + # Process parameters for metadata and file download + url = params['param_dict']['url'].rstrip("/") + "/" + params['param_dict']['file_name'].lstrip("/") + m = re.search(r"_([^_]*?)_vep_(\d+?)_", params['param_dict']['file_name']) + version = str(m.group(1)) + cache_type = m.group(0) if m.group(0) == "merged" or m.group(0) == "refseq" else "default" + + # Download and extract given cache archive, remove archive afterwards + final_file, headers = urlretrieve(url, os.path.join(target_directory, params['param_dict']['file_name'])) + tar = tarfile.open(final_file, "r:gz") + tar.extractall(target_directory) + tar.close() + os.remove(final_file) + + # Construct metadata for the new data table entry + data_manager_dict = { + 'data_tables': { + 'vep_cache_databases': [ + { + 'value': params['param_dict']['file_name'].strip(".tar.gz"), + 'dbkey': params['param_dict']['dbkey'], + 'version': version, + 'cachetype': ctype, + 'name': params['param_dict']['display_name'], + 'path': './%s' % params['param_dict']['file_name'].strip(".tar.gz") + } + ] + } + } + + # Save metadata to out_file + with open(sys.argv[1], 'w') as fh: + json.dump(data_manager_dict, fh, sort_keys=True) + + +if __name__ == "__main__": + main()
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/data_manager/data_manager_vep_cache_download.xml Tue Feb 22 16:26:44 2022 +0000 @@ -0,0 +1,42 @@ +<tool id="data_manager_vep_cache_downloader" name="Download and install VEP cache" version="0.1" tool_type="manage_data"> + <description>the cache files required by VEP</description> + <requirements> + <requirement type="package" version="3.9">python</requirement> + </requirements> + <command detect_errors="exit_code"> + python '$__tool_directory__/data_manager_vep_cache_download.py' '$out_file' + </command> + <inputs> + <param name="dbkey" type="genomebuild" + label="DBKEY of genome that the VEP cache data is for" + help="" /> + <param name="url" type="text" value="http://ftp.ensembl.org/pub/release-105/variation/indexed_vep_cache/" + label="FTP root url for VEP cache files" help=""/> + <param name="file_name" type="text" label="File name of cache file to be downloaded from root url." help="E.g. homo_sapiens_vep_105_GRCh38.tar.gz"/> + <param name="display_name" type="text" label="Display name used in data-selection dropdowns." help="E.g. Homo sapiens hg38 (V105)"/> + </inputs> + <outputs> + <data name="out_file" format="data_manager_json"/> + </outputs> + <tests> + <test> + <param name="dbkey" value="ce11"/> + <param name="url" value="http://ftp.ensembl.org/pub/release-105/variation/indexed_vep_cache/"/> + <param name="file_name" value="caenorhabditis_elegans_vep_105_WBcel235.tar.gz"/> + <param name="display_name" value="C. elegans c11 (V105)"/> + <output name="out_file" file="from_test-meta.data_manager.json"/> + </test> + </tests> + <help> +This tool downloads given versions of VEP cache annotation files and makes them available to Ensembl VEP in Galaxy via the +"vep_versioned_caches" data table. You should use the indexed version of the cache files and it is strongly recommended to +use the cache files which version number matches the VEP version number. Note that for most genomes there are three versions +of cache data available: default, refseq and merged (combining the former two). Choose the one suitable for your usage. + +A general introduction to the VEP cache and download links can be found on the official website: +https://www.ensembl.org/info/docs/tools/vep/script/vep_cache.html + </help> + <citations> + <citation type="doi">10.1186/s13059-016-0974-4</citation> + </citations> +</tool>
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/data_manager_conf.xml Tue Feb 22 16:26:44 2022 +0000 @@ -0,0 +1,21 @@ +<?xml version="1.0"?> +<data_managers> + <data_manager tool_file="data_manager/data_manager_vep_cache_download.xml" id="data_manager_vep_cache_download" > + <data_table name="vep_versioned_caches"> <!-- Defines a Data Table to be modified. --> + <output> <!-- Handle the output of the Data Manager Tool --> + <column name="value" /> <!-- columns that are going to be specified by the Data Manager Tool --> + <column name="dbkey" /> <!-- columns that are going to be specified by the Data Manager Tool --> + <column name="version" /> <!-- columns that are going to be specified by the Data Manager Tool --> + <column name="cachetype" /> <!-- columns that are going to be specified by the Data Manager Tool --> + <column name="name" /> <!-- columns that are going to be specified by the Data Manager Tool --> + <column name="path" output_ref="out_file" > + <move type="directory" relativize_symlinks="True"> + <target base="${GALAXY_DATA_MANAGER_DATA_PATH}">vep/${version}/${dbkey}/${cachetype}</target> + </move> + <value_translation>${GALAXY_DATA_MANAGER_DATA_PATH}/vep/${version}/${dbkey}/${cachetype}/</value_translation> + <value_translation type="function">abspath</value_translation> + </column> + </output> + </data_table> + </data_manager> +</data_managers> \ No newline at end of file
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/data_manager_conf.xml.sample Tue Feb 22 16:26:44 2022 +0000 @@ -0,0 +1,12 @@ +<tables> + <!-- Table of installed versioned vep cache data --> + <table name="vep_versioned_caches" comment_char="#"> + <columns>value, dbkey, version, cachetype, name, path</columns> + <file path="tool-data/vep_versioned_caches.loc" /> + </table> + <!-- Locations of dbkeys and len files under genome directory --> + <table name="__dbkeys__" comment_char="#"> + <columns>value, name, len_path</columns> + <file path="tool-data/dbkeys.loc" /> + </table> +</tables> \ No newline at end of file
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/data_manager_conf.xml.test Tue Feb 22 16:26:44 2022 +0000 @@ -0,0 +1,12 @@ +<tables> + <!-- Table of installed versioned vep cache data --> + <table name="versioned vep cache" comment_char="#"> + <columns>value, dbkey, version, cachetype, name, path</columns> + <file path="${__HERE__}/test-data/vep_versioned_caches.loc" /> + </table> + <!-- Locations of dbkeys and len files under genome directory --> + <table name="__dbkeys__" comment_char="#"> + <columns>value, name, len_path</columns> + <file path="${__HERE__}/test-data/dbkeys.loc" /> + </table> +</tables> \ No newline at end of file
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/dbkeys.loc Tue Feb 22 16:26:44 2022 +0000 @@ -0,0 +1,3 @@ +#<dbkey> <display_name> <len_file_path> +hg38 Human hg38 a_path +ce11 C. elegans ce11 a_path
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/from_test-meta.data_manager.json Tue Feb 22 16:26:44 2022 +0000 @@ -0,0 +1,1 @@ +{"data_tables": {"vep_cache_databases": [{"value": "caenorhabditis_elegans_vep_105_WBcel235", "dbkey": "ce11", "version": "105", "cachetype": "default", "name": "C. elegans c11 (V105)", "path": "./caenorhabditis_elegans_vep_105_WBcel235"}]}} \ No newline at end of file
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/vep_versioned_caches.loc Tue Feb 22 16:26:44 2022 +0000 @@ -0,0 +1,2 @@ +#<value> <dbkey> <version> <cachetype> <name> <path> +#
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tool-data/dbkeys.loc.sample Tue Feb 22 16:26:44 2022 +0000 @@ -0,0 +1,1 @@ +#<dbkey> <display_name> <len_file_path>
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tool-data/vep_versioned_caches.loc.sample Tue Feb 22 16:26:44 2022 +0000 @@ -0,0 +1,11 @@ +#This file describes vep cache data and its metadata available on the server. +#The data table has the format (white space characters are TAB characters): +# +#<value> <dbkey> <version> <cachetype> <name> <path> +# +#So, vep_versioned_caches.loc tables could look like this: +# +#homo_sapiens_vep_105_GRCh38 hg38 105 default Homo sapiens hg38 (V105) /path/to/vep_versioned_caches/105/hg38/default +#homo_sapiens_refseq_vep_105_GRCh38 hg38 105 refseq Homo sapiens hg38 refseq (V105) /path/to/vep_versioned_caches/105/hg38/refseq +#homo_sapiens_merged_vep_105_GRCh38 hg38 105 merged Homo sapiens hg38 merged (V105) /path/to/vep_versioned_caches/105/hg38/merged +# \ No newline at end of file