Mercurial > repos > iuc > data_manager_vsnp_excel

changeset 0:bd6e2a8755e6 draft default tip

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
"planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_vsnp_excel commit 28873701c097a136c789024151e2fb59fa1091cd"
author iuc
date Fri, 08 May 2020 21:58:50 +0000
parents
children
files data_manager/vsnp_excel_fetcher.py data_manager/vsnp_excel_fetcher.xml data_manager_conf.xml test-data/AF2122.fa test-data/all_fasta.loc test-data/vsnp_excel.json test-data/vsnp_excel.loc tool-data/all_fasta.loc.sample tool-data/vsnp_excel.loc.sample tool_data_table_conf.xml.sample tool_data_table_conf.xml.test
diffstat 10 files changed, 268 insertions(+), 0 deletions(-) [+]
line wrap: on
line diff
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/data_manager/vsnp_excel_fetcher.py	Fri May 08 21:58:50 2020 +0000
@@ -0,0 +1,66 @@
+import argparse
+import json
+import os
+import sys
+try:
+    # For Python 3.0 and later
+    from urllib.request import Request, urlopen
+except ImportError:
+    # Fall back to Python 2 imports
+    from urllib2 import Request, urlopen
+
+
+def url_download(url, workdir):
+    file_path = os.path.abspath(os.path.join(workdir, os.path.basename(url)))
+    src = None
+    dst = None
+    try:
+        req = Request(url)
+        src = urlopen(req)
+        with open(file_path, 'wb') as dst:
+            while True:
+                chunk = src.read(2**10)
+                if chunk:
+                    dst.write(chunk)
+                else:
+                    break
+    except Exception as e:
+        sys.exit(str(e))
+    finally:
+        if src:
+            src.close()
+    return file_path
+
+
+def download(dbkey, name, url, out_file):
+
+    with open(out_file) as fh:
+        params = json.loads(fh.read())
+
+    workdir = params['output_data'][0]['extra_files_path']
+    os.makedirs(workdir)
+    file_path = url_download(url, workdir)
+    entry_name = os.path.basename(file_path)
+
+    data_manager_json = {"data_tables": {}}
+    data_manager_entry = {}
+    data_manager_entry['value'] = dbkey
+    data_manager_entry['name'] = entry_name
+    data_manager_entry['path'] = file_path
+    data_manager_entry['description'] = "Excel file for %s" % name
+    data_manager_json["data_tables"]["vsnp_excel"] = data_manager_entry
+
+    with open(out_file, 'w') as fh:
+        fh.write(json.dumps(data_manager_json, sort_keys=True))
+
+
+parser = argparse.ArgumentParser()
+
+parser.add_argument('--dbkey', dest='dbkey', help='Genome reference dbkey')
+parser.add_argument('--name', dest='name', help='Reference display name')
+parser.add_argument('--url', dest='url', help='URL to download Excel file')
+parser.add_argument('--out_file', dest='out_file', help='JSON output file')
+
+args = parser.parse_args()
+
+download(args.dbkey, args.name, args.url, args.out_file)
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/data_manager/vsnp_excel_fetcher.xml	Fri May 08 21:58:50 2020 +0000
@@ -0,0 +1,103 @@
+<?xml version="1.0"?>
+<tool id="vsnp_excel_fetcher" name="vSNP Excel data manager" tool_type="manage_data" profile="18.09" version="0.0.8">
+    <description>Download vSNP Excel files</description>
+    <requirements>
+        <requirement type="package" version="3.7">python</requirement>
+    </requirements>
+    <command detect_errors="exit_code"><![CDATA[
+    python '$__tool_directory__/vsnp_excel_fetcher.py'
+    --dbkey '${all_fasta_source.fields.dbkey}'
+    --name '${all_fasta_source.fields.name}'
+    --out_file '$out_file'
+    --url '$url'
+    ]]>
+    </command>
+    <inputs>
+        <param name="all_fasta_source" type="select" label="FASTA reference">
+            <options from_data_table="all_fasta"/>
+        </param>
+        <param name="url" type="text" value="" label="URL to download the Excel file associated with the selected FASTA reference" optional="False" />
+    </inputs>
+    <outputs>
+        <data name="out_file" format="data_manager_json" />
+    </outputs>
+    <tests>
+        <test>
+            <param name="all_fasta_source" value="AF2122"/>
+            <param name="url" value="https://github.com/USDA-VS/vSNP_reference_options/raw/master/Mycobacterium_AF2122/Mbovis_define_filter.xlsx"/>
+            <output name="out_file" value="vsnp_excel.json" compare="contains"/>
+        </test>
+    </tests>
+    <help><![CDATA[
+This tool fetches a vSNP Excel file associated with each supported genome reference to populate the vsnp_excel data table.  The dbkey and name fields
+in the vsnp data table are inherited from the *all_fasta* data table, so no user entry is necessary.  These public vSNP Excel files are available in GitHub
+at https://github.com/USDA-VS/vSNP_reference_options.
+
+ * **Mycobacterium bovis AF2122/97**
+
+   * **Excel file**    https://github.com/USDA-VS/vSNP_reference_options/raw/master/Mycobacterium_AF2122/Mbovis_define_filter.xlsx
+
+ * **Brucella abortus bv. 1 str. 9-941**
+
+   * **Excel file**    https://github.com/USDA-VS/vSNP_reference_options/raw/master/Brucella_abortus1/Bab1_define_filter.xlsx
+
+ * **Brucella abortus strain BER**
+
+   * **Excel file**    https://github.com/USDA-VS/vSNP_reference_options/raw/master/Brucella_abortus3/Bab3_define_filter.xlsx
+
+ * **Brucella canis ATCC 23365**
+
+   * **Excel file**    https://github.com/USDA-VS/vSNP_reference_options/raw/master/Brucella_canis/Bcanis_define_filter.xlsx
+
+ * **Brucella ceti TE10759-12**
+
+   * **Excel file**    https://github.com/USDA-VS/vSNP_reference_options/raw/master/Brucella_ceti2/Bceti2_define_filter.xlsx
+
+ * **Brucella melitensis bv. 1 str. 16M**
+
+   * **Excel file**    https://github.com/USDA-VS/vSNP_reference_options/raw/master/Brucella_melitensis-bv1/Bmelbv1_define_filter.xlsx
+
+ * **Brucella melitensis bv. 3 str. Ether**
+
+   * **Excel file**    https://github.com/USDA-VS/vSNP_reference_options/raw/master/Brucella_melitensis-bv3/Bmelbv3_define_filter.xlsx
+
+ * **Brucella melitensis BwIM_SOM_36b**
+
+   * **Excel file**    https://github.com/USDA-VS/vSNP_reference_options/raw/master/Brucella_melitensis-bv1b/Bmelbv1b_define_filter.xlsx
+
+ * **Brucella melitensis ATCC 23457**
+
+   * **Excel file**    https://github.com/USDA-VS/vSNP_reference_options/raw/master/Brucella_melitensis-bv2/Bmelbv2_define_filter.xlsx
+
+ * **Brucella ovis ATCC 25840**
+
+   * **Excel file**    https://github.com/USDA-VS/vSNP_reference_options/raw/master/Brucella_ovis/B_ovis_define_filter.xlsx
+
+ * **Brucella suis 1330**
+
+   * **Excel file**    https://github.com/USDA-VS/vSNP_reference_options/raw/master/Brucella_suis1/Bsuis1_define_filter.xlsx
+
+ * **Mycobacterium tuberculosis H37Rv**
+
+   * **Excel file**    https://github.com/USDA-VS/vSNP_reference_options/raw/master/Mycobacterium_H37/H37_define_filter.xlsx
+
+ * **Mycobacterium avium subsp. paratuberculosis strain Telford**
+
+   * **Excel file**    https://github.com/USDA-VS/vSNP_reference_options/raw/master/para-CP033688/CP033688_define_filter.xlsx
+
+ * **Mycobacterium avium subsp. paratuberculosis K-10**
+
+   * **Excel file**    https://github.com/USDA-VS/vSNP_reference_options/raw/master/para-NC002944/NC_002944_define_filter.xlsx
+
+ * **Brucella suis ATCC 23445**
+
+   * **Excel file**    https://github.com/USDA-VS/vSNP_reference_options/raw/master/Brucella_suis2/Bsuis2_define_filter.xlsx
+
+ * **Brucella suis bv. 3 str. 686**
+
+   * **Excel file**    https://github.com/USDA-VS/vSNP_reference_options/raw/master/Brucella_suis3/Bsuis3_define_filter.xlsx
+
+    ]]></help>
+    <citations>
+    </citations>
+</tool>
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/data_manager_conf.xml	Fri May 08 21:58:50 2020 +0000
@@ -0,0 +1,21 @@
+<?xml version="1.0"?>
+<data_managers>
+    <data_manager tool_file="data_manager/vsnp_excel_fetcher.xml" id="vsnp_excel_fetcher">
+        <data_table name="vsnp_excel">
+            <output>
+                <column name="value" />
+                <column name="name" />
+                <column name="path" output_ref="out_file">
+                    <move type="file" relativize_symlinks="True">
+                        <source>${path}</source>
+			<target base="${GALAXY_DATA_MANAGER_DATA_PATH}">vsnp/${value}/excel/${name}</target>
+                    </move>
+		    <value_translation>${GALAXY_DATA_MANAGER_DATA_PATH}/vsnp/${value}/excel/${name}</value_translation>
+                    <value_translation type="function">abspath</value_translation>
+                </column>
+                <column name="description" />
+            </output>
+        </data_table>
+    </data_manager>
+</data_managers>
+
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/test-data/AF2122.fa	Fri May 08 21:58:50 2020 +0000
@@ -0,0 +1,11 @@
+>NC_002945.4 Mycobacterium bovis AF2122/97 genome assembly, chromosome: Mycobacterium_bovis_AF2122/97
+TTGACCGATGACCCCGGTTCAGGCTTCACCACAGTGTGGAACGCGGTCGTCTCCGAACTTAACGGCGACC
+CTAAGGTTGACGACGGACCCAGCAGTGATGCTAATCTCAGCGCTCCGCTGACCCCTCAGCAAAGGGCTTG
+GCTCAATCTCGTCCAGCCATTGACCATCGTCGAGGGGTTTGCTCTGTTATCCGTGCCGAGCAGCTTTGTC
+CAAAACGAAATCGAGCGCCATCTGCGGGCCCCGATTACCGACGCTCTCAGCCGCCGACTCGGACATCAGA
+TCCAACTCGGGGTCCGCATCGCTCCGCCGGCGACCGACGAAGCCGACGACACTACCGTGCCGCCTTCCGA
+AAATCCTGCTACCACATCGCCAGACACCACAACCGACAACGACGAGATTGATGACAGCGCTGCGGCACGG
+GGCGATAACCAGCACAGTTGGCCAAGTTACTTCACCGAGCGCCCGCGCAATACCGATTCCGCTACCGCTG
+GCGTAACCAGCCTTAACCGTCGCTACACCTTTGATACGTTCGTTATCGGCGCCTCCAACCGGTTCGCGCA
+CGCCGCCGCCTTGGCGATCGCAGAAGCACCCGCCCGCGCTTACAACCCCCTGTTCATCTGGGGCGAGTCC
+GGTCTCGGCAAGACACACCTGCTACACGCGGCAGGCAACTATGCCCAACGGTTGTTCCCGGGAATGCGGG
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/test-data/all_fasta.loc	Fri May 08 21:58:50 2020 +0000
@@ -0,0 +1,19 @@
+#This file lists the locations and dbkeys of all the fasta files
+#under the "genome" directory (a directory that contains a directory
+#for each build). The script extract_fasta.py will generate the file
+#all_fasta.loc. This file has the format (white space characters are
+#TAB characters):
+#
+#<unique_build_id>	<dbkey>		<display_name>	<file_path>
+#
+#So, all_fasta.loc could look something like this:
+#
+#apiMel3	apiMel3	Honeybee (Apis mellifera): apiMel3		/path/to/genome/apiMel3/apiMel3.fa
+#hg19canon	hg19		Human (Homo sapiens): hg19 Canonical		/path/to/genome/hg19/hg19canon.fa
+#hg19full	hg19		Human (Homo sapiens): hg19 Full			/path/to/genome/hg19/hg19full.fa
+#
+#Your all_fasta.loc file should contain an entry for each individual
+#fasta file. So there will be multiple fasta files for each build,
+#such as with hg19 above.
+#
+AF2122	AF2122	AF2122	${__HERE__}/AF2122.fa
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/test-data/vsnp_excel.json	Fri May 08 21:58:50 2020 +0000
@@ -0,0 +1,1 @@
+{"data_tables": {"vsnp_excel": {"description": "Excel file for AF2122", "name": "Mbovis_define_filter.xlsx", "path":
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tool-data/all_fasta.loc.sample	Fri May 08 21:58:50 2020 +0000
@@ -0,0 +1,18 @@
+#This file lists the locations and dbkeys of all the fasta files
+#under the "genome" directory (a directory that contains a directory
+#for each build). The script extract_fasta.py will generate the file
+#all_fasta.loc. This file has the format (white space characters are
+#TAB characters):
+#
+#<unique_build_id>	<dbkey>		<display_name>	<file_path>
+#
+#So, all_fasta.loc could look something like this:
+#
+#apiMel3	apiMel3	Honeybee (Apis mellifera): apiMel3		/path/to/genome/apiMel3/apiMel3.fa
+#hg19canon	hg19		Human (Homo sapiens): hg19 Canonical		/path/to/genome/hg19/hg19canon.fa
+#hg19full	hg19		Human (Homo sapiens): hg19 Full			/path/to/genome/hg19/hg19full.fa
+#
+#Your all_fasta.loc file should contain an entry for each individual
+#fasta file. So there will be multiple fasta files for each build,
+#such as with hg19 above.
+#
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tool-data/vsnp_excel.loc.sample	Fri May 08 21:58:50 2020 +0000
@@ -0,0 +1,4 @@
+## vSNP Excel files
+#Value	Name	Path	Description
+#AF2122	Mbovis_define_filter.xlsx	vsnp/AF2122/Mbovis_define_filter.xlsx	Excel file for Mycobacterium bovis AF2122/97
+#NC_006932	Bab1_define_filter.xlsx	vsnp/NC_006932/Bab1_define_filter.xlsx	Excel file for Brucella abortus bv. 1 str. 9-941
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tool_data_table_conf.xml.sample	Fri May 08 21:58:50 2020 +0000
@@ -0,0 +1,13 @@
+<tables>
+    <!-- Locations of all fasta files under genome directory -->
+    <table name="all_fasta" comment_char="#">
+        <columns>value, dbkey, name, path</columns>
+        <file path="tool-data/all_fasta.loc" />
+    </table>
+    <!-- Location of excel files for vsnp_excel version 0.0.8 -->
+    <table name="vsnp_excel" comment_char="#">
+        <columns>value, name, path, description</columns>
+        <file path="tool-data/vsnp_excel.loc" />
+    </table>
+</tables>
+
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tool_data_table_conf.xml.test	Fri May 08 21:58:50 2020 +0000
@@ -0,0 +1,12 @@
+<tables>
+    <!-- Locations of all fasta files under genome directory -->
+    <table name="all_fasta" comment_char="#">
+        <columns>value, dbkey, name, path</columns>
+        <file path="${__HERE__}/test-data/all_fasta.loc" />
+    </table>
+    <!-- Location of excel files for vsnp_excel version 0.0.8 -->
+    <table name="vsnp_excel" comment_char="#">
+        <columns>value, name, path, description</columns>
+        <file path="${__HERE__}/test-data/vsnp_excel.loc" />
+    </table>
+</tables>