Mercurial > repos > matthias > data_manager_mitos

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/data_manager/data_manager.py	Thu May 09 16:20:33 2019 -0400
@@ -0,0 +1,90 @@
+import argparse
+import json
+import os
+import shutil
+import sys
+import tarfile
+try:
+    # For Python 3.0 and later
+    from urllib.request import Request, urlopen
+except ImportError:
+    # Fall back to Python 2 imports
+    from urllib2 import Request, urlopen
+
+ZENODO = {
+    "mitos":"2683856",
+    "mitos2": "2672835"
+}
+
+NAMES = {
+    "mitos1-refdata": "RefSeq39 + MiTFi tRNA models",
+    "refseq39": "RefSeq39 (equivalent to MITOS1 data)",
+    "refseq63m": "RefSeq63 Metazoa",
+    "refseq63f": "RefSeq63 Fungi",
+    "refseq63o": "RefSeq63 Opisthokonta",
+    "refseq81m": "RefSeq81 Metazoa",
+    "refseq81f": "RefSeq81 Fungi",
+    "refseq81o": "RefSeq81 Opisthokonta"
+}
+
+def url_download(tpe, db, workdir):
+    """
+    download http://ab.inf.uni-tuebingen.de/data/software/megan6/download/FNAME
+    to workdir
+    and unzip
+
+    return the name of the resulting dir
+    """
+    tarfname = os.path.join(workdir, db + ".tar.bz")
+    if not os.path.exists(workdir):
+        os.makedirs(workdir)
+    src = None
+    dst = None
+    try:
+        req = Request("https://zenodo.org/record/{zenodoid}/files/{db}.tar.bz2?download=1".format(zenodoid=ZENODO[tpe], db=db))
+        src = urlopen(req)
+        with open(tarfname, 'wb') as dst:
+            while True:
+                chunk = src.read(2**10)
+                if chunk:
+                    dst.write(chunk)
+                else:
+                    break
+    finally:
+        if src:
+            src.close()
+    with tarfile.open(tarfname, "r:bz2") as tar:
+        tar.extractall(workdir)
+    os.remove(tarfname)
+    unzipped = os.listdir(workdir)
+    assert len(unzipped) == 1
+    return unzipped[0]
+
+def main(tpe, db, outjson):
+
+    workdir = os.getcwd()
+
+    path = url_download(tpe, db, workdir)
+
+    data_manager_entry = {}
+    data_manager_entry['value'] = db
+    data_manager_entry['name'] = NAMES[db]
+    data_manager_entry['type'] = tpe
+    data_manager_entry['path'] = path
+    data_manager_json = dict(data_tables=dict(mitos=data_manager_entry))
+
+    params = json.loads(open(outjson).read())
+    target_directory = params['output_data'][0]['extra_files_path']
+    os.mkdir(target_directory)
+    output_path = os.path.abspath(os.path.join(os.getcwd(), 'mitos'))
+    shutil.move(os.path.join(workdir, path), target_directory)
+    file(outjson, 'w').write(json.dumps(data_manager_json))
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Create data manager json.')
+    parser.add_argument('--out', action='store', help='JSON filename')
+    parser.add_argument('--type', action='store', help='mitos version')
+    parser.add_argument('--db', action='store', help='db name')
+    args = parser.parse_args()
+
+    main(args.type, args.db, args.out)
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/data_manager/mitos_fetcher.xml	Thu May 09 16:20:33 2019 -0400
@@ -0,0 +1,52 @@
+<?xml version="1.0"?>
+<tool id="mitos_fetcher_data_manager" name="MITOS" tool_type="manage_data" version="0.0.1">
+    <description>reference data downloader</description>
+    <command detect_errors="exit_code">
+    <![CDATA[
+    python '$__tool_directory__/data_manager.py'
+    --out '${out_file}'
+    --type '$type_cond.type_select'
+    --db '$type_cond.database'
+    ]]>
+    </command>
+    <inputs>
+        <conditional name="type_cond">
+            <param name="type_select" type="select" label="MITOS version">
+                <option value="mitos">MITOS</option>
+                <option value="mitos2">MITOS2</option>
+	    </param>
+            <when value="mitos">
+                <param name="database" type="select" label="Refernce data version">
+                    <option value="mitos1-refdata">RefSeq39 + MiTFi tRNA models</option>
+                </param>
+            </when>
+            <when value="mitos2">
+                <param name="database" type="select" label="Refernce data version">
+                    <option value="refseq39">RefSeq39 (equivalent to MITOS1 data)</option>
+                    <option value="refseq63m">RefSeq63 Metazoa</option>
+                    <option value="refseq63f">RefSeq63 Fungi</option>
+                    <option value="refseq63o">RefSeq63 Opisthokonta</option>
+                    <option value="refseq81m">RefSeq81 Metazoa</option>
+                    <option value="refseq81f">RefSeq81 Fungi</option>
+                    <option value="refseq81o">RefSeq81 Opisthokonta</option>
+                </param>
+            </when>
+        </conditional>
+    </inputs>
+    <outputs>
+        <data name="out_file" format="data_manager_json" />
+    </outputs>
+    <tests>
+        <test>
+            <conditional name="type_cond">
+                <param name="type_select" value="mitos"/>
+                <param name="database" value="mitos1-refdata"/>
+            </conditional>
+            <output name="out_file" file="mitos_refseq39.json"/>
+        </test>
+    </tests>
+    <help>
+TODO
+    </help>
+</tool>
+
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/data_manager_conf.xml	Thu May 09 16:20:33 2019 -0400
@@ -0,0 +1,20 @@
+<?xml version="1.0"?>
+<data_managers>
+    <data_manager tool_file="data_manager/mitos_fetcher.xml" id="mitos_fetcher" version="0.0.1">
+        <data_table name="mitos">
+            <output>
+                <column name="value" />
+                <column name="name" />
+                <column name="type" />
+                <column name="path" output_ref="out_file">
+                    <move type="directory" relativize_symlinks="True">
+                        <source>${path}</source>
+                        <target base="${GALAXY_DATA_MANAGER_DATA_PATH}">mitos/${path}</target>
+                    </move>
+                    <value_translation>${GALAXY_DATA_MANAGER_DATA_PATH}/mitos/${path}</value_translation>
+                    <value_translation type="function">abspath</value_translation>
+                </column>
+            </output>
+        </data_table>
+    </data_manager>
+</data_managers>
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/test-data/mitos_refseq39.json	Thu May 09 16:20:33 2019 -0400
@@ -0,0 +1,1 @@
+{"data_tables": {"mitos": {"path": "mitos1-refdata", "type": "mitos", "name": "RefSeq39 + MiTFi tRNA models", "value": "mitos1-refdata"}}}
\ No newline at end of file
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tool-data/mitos.loc.sample	Thu May 09 16:20:33 2019 -0400
@@ -0,0 +1,8 @@
+# consists of 4 tab separated columns:
+# - value identifier
+# - name the name shown in the tool form
+# - type mitos/mitos2
+# - path the directory containing the reference data
+#
+# example:
+# mitos1-refdata	RefSeq39 + MiTFi tRNA models	mitos	/tmp/tmpSp4v2Y/tmpR7DRNf/tmp53wbgs/database/data_manager_tool-dataWhw0Ns/mitos/mitos1-refdata
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tool_data_table_conf.xml.sample	Thu May 09 16:20:33 2019 -0400
@@ -0,0 +1,8 @@
+<?xml version="1.0"?>
+<tables>
+    <!-- Locations of data downloaded for the megan tools -->
+    <table name="mitos" comment_char="#">
+        <columns>value, name, type, path</columns>
+        <file path="tool-data/mitos.loc" />
+    </table>
+</tables>
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tool_data_table_conf.xml.test	Thu May 09 16:20:33 2019 -0400
@@ -0,0 +1,7 @@
+<tables>
+    <!-- Locations of indexes in the Bowtie2 mapper format -->
+    <table name="mitos" comment_char="#">
+        <columns>value, name, type, path</columns>
+        <file path="${__HERE__}/test-data/mitos.loc" />
+    </table>
+</tables>