Mercurial > repos > oinizan > data_manager_frogs
changeset 0:493202ba3af5 draft
"planemo upload for repository https://github.com/geraldinepascal/FROGS-wrappers/ commit 56a8d94cd18a5b89f83ccfa9cdc7c80a28d044d4-dirty"
| author | oinizan |
|---|---|
| date | Wed, 18 Aug 2021 15:24:49 +0000 |
| parents | |
| children | 04845601de8c |
| files | data_manager/FROGS_data_manager.py data_manager/FROGS_data_manager.xml data_manager_conf.xml tool-data/frogs_db.loc.sample tool_data_table_conf.xml.sample tool_dependencies.xml |
| diffstat | 6 files changed, 323 insertions(+), 0 deletions(-) [+] |
line wrap: on
line diff
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/data_manager/FROGS_data_manager.py Wed Aug 18 15:24:49 2021 +0000 @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# import json +import argparse +import os +# import sys +import tarfile +import time +import urllib + +from galaxy.util.json import from_json_string, to_json_string + +import requests + +# GALAXY_database=~/galaxy/galaxy-20.09/database +# FROGS_data_manager.py --database=frogs_db_data --all_dbs=false \ +# --date=0 --amplicons=16S --bases=SILVA --filters=Pintail100 \ +# --only_last_versions=true \ +# --tool_data=/home/maria/galaxy/galaxy-20.09/tool-data \ +# --output $GALAXY_database/objects/e/7/7/dataset_e7766c39-8f36-450c-adf5-3e4ee8d5c562.dat + + +def get_args(): + parser = argparse.ArgumentParser() + parser.add_argument("-d", "--database") + parser.add_argument("--all_dbs") + parser.add_argument("--date") + parser.add_argument("--amplicons") + parser.add_argument("--bases") + parser.add_argument("--filters") + parser.add_argument("--only_last_versions") + parser.add_argument("--tool_data") + parser.add_argument("-o", "--output") + args = parser.parse_args() + + return args + + +def _add_data_table_entry(data_manager_dict, data_table_entry, data_table): + data_manager_dict['data_tables'] = data_manager_dict.get('data_tables', {}) + data_manager_dict['data_tables'][data_table] = data_manager_dict['data_tables'].get(data_table, []) + data_manager_dict['data_tables'][data_table].append(data_table_entry) + return data_manager_dict + + +def keep_only_last_version(db_index): + db_dict = dict() + for line in db_index: + db_type = "_".join(line[1:4]) if line[3] != "" else "_".join(line[1:3]) + if db_type not in db_dict: + db_dict[db_type] = line + return list(db_dict.values()) + + +def frogs_sources(data_manager_dict, target_directory): + + # variables + amplicons_list = [] + bases_list = [] + filters_list = [] + if args.all_dbs == "false": + amplicons_list = [amplicon.lower().strip() for amplicon in args.amplicons.split(",") if amplicon != ""] + bases_list = [base.lower().strip() for base in args.bases.split(",") if base != ""] + filters_list = [filter.lower().strip() for filter in args.filters.split(",") if filter != ""] + bottom_date = int(args.date) + tool_data_path = args.tool_data + + # get frogs database index + frogs_db_index_link = "http://genoweb.toulouse.inra.fr/frogs_databanks/assignation/FROGS_databases.tsv" + with requests.Session() as s: + download = s.get(frogs_db_index_link) + decoded_content = download.content.decode('utf-8') + db_index = decoded_content.splitlines() + db_index = [line.split("\t") for line in db_index[1:]] + db_index = [[line[0], line[1].lower(), line[2].lower(), line[3].lower()] + line[4:] for line in db_index] + + # filter databases + if args.all_dbs == "false": + # filter by amplicons + if len(amplicons_list) != 0: + db_index = [line for line in db_index if any([amplicon in amplicons_list for amplicon in line[1].split(',')])] + # filter by base + if len(bases_list) != 0: + db_index = [line for line in db_index if line[2] in bases_list] + # filter by filters + if len(filters_list) != 0: + db_index = [line for line in db_index if line[3] in filters_list] + # filter by date + if bottom_date != 0: + db_index = [line for line in db_index if int(line[0]) >= bottom_date] + if args.only_last_versions == "true": + # keep only last version + db_index = keep_only_last_version(db_index) + + # get frogs dbs + os.chdir(target_directory) + dir_name = "frogs_db_" + time.strftime("%Y%m%d") + os.mkdir(dir_name) + dbs = set([]) + for line in db_index: + value = line[5] + name = value.replace("_", " ") if "_" not in line[4] else value.replace(line[4], "").replace("_", " ") + line[4] + link = line[6] + name_dir = "".join([line[6].replace(".tar.gz", "").split("/")[-1]]) + file_path = tool_data_path + "/frogs_db/" + name_dir + if not os.path.exists(file_path): # if the file is not already in frogs_db directory + + # download frogs db + dl_file = urllib.request.URLopener() + dl_file.retrieve(link, "tmp.tar.gz") + + # unzip frogs db + with tarfile.open("tmp.tar.gz") as tar: + tar.extractall(dir_name) + tar.close() + os.remove('tmp.tar.gz') + + # get fasta file path + tmp = set(os.listdir(dir_name)) + new_db = dir_name + "/" + "".join(tmp.difference(dbs)) + files = os.listdir(new_db) + fasta = "".join([file for file in files if file.endswith('.fasta')]) + path = new_db + '/' + fasta + dbs = os.listdir(dir_name) + # release = value + "_" + time.strftime("%Y-%m-%d") + # date = time.strftime("%Y%m%d") + path = os.path.join(target_directory, path) + + data_table_entry = dict(name=name, value=value, path=path) + _add_data_table_entry(data_manager_dict, data_table_entry, "frogs_db") + +# def HVL_sources(data_manager_dict,target_directory): +# HVL_dir = "http://genoweb.toulouse.inra.fr/frogs_databanks/HVL/ITS/UNITE_s_7.1_20112016" +# os.chdir(target_directory) +# for link in [HVL_dir + "/Unite_s_7.1_20112016_ITS1.fasta",HVL_dir + "/Unite_s_7.1_20112016_ITS2.fasta"]: +# file_name=link.split("/")[-1].replace('.fasta',"_"+time.strftime("%Y-%m-%d")+".fasta") +# dl_file = urllib.URLopener() +# dl_file.retrieve(link,file_name) + +# #get fasta file path +# path = os.path.join(target_directory,file_name) +# if link.endswith('ITS1.fasta'): +# name = "UNITE 7.1 ITS1 " + time.strftime("%Y-%m-%d") +# elif link.endswith('ITS2.fasta'): +# name = "UNITE 7.1 ITS2 " + time.strftime("%Y-%m-%d") +# value=file_name.replace('.fasta','') + +# data_table_entry = dict(name = name, value = value, path=path) +# _add_data_table_entry(data_manager_dict, data_table_entry, "frogs_HVL_db") + + +def main(): + + # get args from command line + global args + args = get_args() + + # Extract json file params + data_manager_dict = {} + filename = args.output + params = from_json_string(open(filename).read()) + target_directory = params['output_data'][0]['extra_files_path'] + os.mkdir(target_directory) + + # if args.database=="frogs_db_data": + frogs_sources(data_manager_dict, target_directory) + # elif args.database=="HVL_db_data": + # HVL_sources(data_manager_dict,target_directory) + + # save info to json file + open(filename, 'wt').write(to_json_string(data_manager_dict)) + + +if __name__ == "__main__": + main()
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/data_manager/FROGS_data_manager.xml Wed Aug 18 15:24:49 2021 +0000 @@ -0,0 +1,64 @@ +<tool id="FROGS_data_manager" name="FROGS Data manager" version="3.2.2" tool_type="manage_data"> + <requirements> + </requirements> + <stdio> + <exit_code range="1:" /> + </stdio> + <command><![CDATA[ + $__tool_directory__/FROGS_data_manager.py + --database="frogs_db_data" + + --all_dbs="$db_type.db" + #if $db_type.db=="false" + --date="$db_type.date" + --amplicons="$db_type.amplicons" + --bases="$db_type.bases" + --filters="$db_type.filters" + #end if + --only_last_versions="$only_last_versions" + + --tool_data="$__tool_data_path__" + --output "$output" + ]]></command> + <inputs> + <conditional name="db_type"> + <param name="db" type="boolean" checked="true" truevalue="true" falsevalue="false" label="Download all databases"/> + <when value="true"/> + <when value="false"> + <param name="date" value="0" type="text" label="Download only most recent databases" help="Select databases uploaded later than this date. Please enter a date at the following format: YYYYMMDD, else leave 0."> + <validator type="regex" message="Please enter a date at the following format: YYYYMMDD, leave 0 for no date">0|[1-2]{1}[0-9]{3}[0-1]{1}[0-9]{1}([0-2]{1}[0-9]{1}|3[0-1]{1})</validator> + </param> + <param name="amplicons" type="text" label='Select amplicon specific databases' help='Write amplicons names separated by ","; example: "COI,ITS,16S" or "23S"'/> + <param name="bases" type="text" label='Select database by name' help='Write base names separated by ","; example: "SILVA,PR2,MIDAS" or "BOLD"'/> + <param name="filters" type="text" label='Select database on filter name' help='Write filter names separated by ",";example: "Pintail100,Fungi"'/> + </when> + </conditional> + <param name="only_last_versions" type="boolean" checked="true" label="Download only the last version of selected database"/> + </inputs> + <outputs> + <data name="output" format="data_manager_json"/> + </outputs> + <help> + + @HELP_LOGO@ + + FROGS datamanager allows to download preformated databases for FROGS Affiliation OTU tool. + + All databases are available at http://genoweb.toulouse.inra.fr/frogs_databanks/assignation/. + + This tools is based on the `FROGS_databases.tsv.txt <http://genoweb.toulouse.inra.fr/frogs_databanks/assignation/FROGS_databases.tsv.txt>`_, that we update each time a new database is formatted. + + .. image:: static/images/FROGS_db.png + :height: 157 + :width: 961 + + You may download all databases, but you may (should) filter whished database on different criteria: + + - on a date, to download only last formated databases + - on an amplicon type + - on a base name + - eventually on a filtered name, this may be the case for example, for 16S SILVA database, for which we provide reference sequence with pintail score above a threshold + + Last option allow you to download only last version of previously selected databases, indeed we provide reference database since 2016 with for example, around 1 version of SILVA per year. + </help> +</tool> \ No newline at end of file
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/data_manager_conf.xml Wed Aug 18 15:24:49 2021 +0000 @@ -0,0 +1,19 @@ +<?xml version="1.0"?> +<data_managers> + <data_manager tool_file="data_manager/FROGS_data_manager.xml" id="frogs_data_manager"> + <data_table name="frogs_db"> + <output> + <column name="value" /> + <column name="name" /> + <column name="path" output_ref="output"> + <move type="directory" relativize_symlinks="False"> + <source>#echo "/".join(str($path).split('/')[:-1])#</source> + <target base="${GALAXY_DATA_MANAGER_DATA_PATH}">frogs_db/#echo str($path).split('/')[-2]#/</target> + </move> + <value_translation>${GALAXY_DATA_MANAGER_DATA_PATH}/frogs_db/#echo "/".join(str($path).split('/')[-2:])#</value_translation> + <value_translation type="function">abspath</value_translation> + </column> + </output> + </data_table> + </data_manager> +</data_managers>
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tool-data/frogs_db.loc.sample Wed Aug 18 15:24:49 2021 +0000 @@ -0,0 +1,53 @@ +# Copyright (C) 2014 INRA +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# +# +#This is a sample file that enables tools FROGS_affiliations_OTU to use taxonomy database for +#taxonomy affiliation. You will need to create or download Blast+ index and train your database +#for RDP classifier these data files. +#download link : http://genoweb.toulouse.inra.fr/frogs_databanks/assignation +#Finally you will need to create frogs_db.loc file similar to this one in your galaxy +#tool-data directory.The frogs_db.loc file has this format (longer white space characters are +#TAB characters): +# +#<unique_database_name> <file_path> +# +#First column will be the visible name in galaxy. +#So, for example, if you had 16S silva 128 indexed stored in +#/galaxy_databanks/16S/silva_128/ +#then the frogs_db.loc entry would look like this: +# +#silva 128 16S /galaxy_databanks/16S/silva_128/silva_128_16S.fasta +# +#and your /galaxy_databanks/16S/silva_128/ directory +#would contain index files: +# +#-rw-r--r-- 1 mbernard FROGS 8097966 5 déc. 16:56 bergeyTrainingTree.xml +#-rw-r--r-- 1 mbernard FROGS 1572981589 5 déc. 16:56 genus_wordConditionalProbList.txt +#-rw-r--r-- 1 mbernard FROGS 1654 5 déc. 16:56 LICENCE.txt +#-rw-r--r-- 1 mbernard FROGS 1072228 5 déc. 16:56 logWordPrior.txt +#-rw-r--r-- 1 mbernard FROGS 940834335 5 déc. 16:56 silva_128_16S.fasta +#-rw-r--r-- 1 mbernard FROGS 152606489 5 déc. 16:56 silva_128_16S.fasta.nhr +#-rw-r--r-- 1 mbernard FROGS 6918588 5 déc. 16:56 silva_128_16S.fasta.nin +#-rw-r--r-- 1 mbernard FROGS 205320030 5 déc. 16:56 silva_128_16S.fasta.nsq +#-rw-r--r-- 1 mbernard FROGS 281 5 déc. 16:56 silva_128_16S.fasta.properties +#-rw-r--r-- 1 mbernard FROGS 3420464 5 déc. 16:56 silva_128_16S.tax +#-rw-r--r-- 1 mbernard FROGS 964048 5 déc. 16:57 wordConditionalProbIndexArr.txt +# +# +#<name> <name> <file_path> +# +# EXAMPLE FOR TEST : +#ITS1_test ITS1_test ${__HERE__}/frogs_db_data/ITS1.rdp.fasta
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tool_data_table_conf.xml.sample Wed Aug 18 15:24:49 2021 +0000 @@ -0,0 +1,8 @@ +<?xml version="1.0"?> +<tables> + <!-- Location of frogs database files --> + <table name="frogs_db" comment_char="#"> + <columns>value, name, path</columns> + <file path="tool-data/frogs_db.loc" /> + </table> +</tables>