Mercurial > repos > mingchen0919 > elastic_fastq_dump
changeset 1:1aeef205e648 draft
planemo upload
author | mingchen0919 |
---|---|
date | Sun, 01 Apr 2018 16:48:53 -0400 |
parents | 28ce9fcb06c7 |
children | 3e6452f224b2 |
files | LICENSE README.md elastic_tool.Rmd elastic_tool.xml elastic_tool_macros.xml elastic_tool_wrappers_macros.xml |
diffstat | 6 files changed, 305 insertions(+), 229 deletions(-) [+] |
line wrap: on
line diff
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/LICENSE Sun Apr 01 16:48:53 2018 -0400 @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2018 Staton Laboratory + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE.
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/README.md Sun Apr 01 16:48:53 2018 -0400 @@ -0,0 +1,63 @@ +# Elastic Galaxy Tools + +* [What are Elastic Galaxy Tools](#what-are-elastic-galaxy-tools?) +* [How to use Elastic Galaxy Tools](#how-to-use-elastic-galaxy-tools?) +* [How to develop an Elastic Galaxy Tool](#how-to-develop-an-elastic-galaxy-tool?) + + [Publish a Galaxy Tool](#publish-a-galaxy-tool) + + + + +## What are Elastic Galaxy Tools? + +**Elastic Galaxy tools** is a type of Galaxy tools that have elastic user interfaces. Users can adjust the +number of input fields a tool can have, just like using a command line tool. When using a command line tool, +users can choose which (non-required) options to use. I hope Galaxy users can do the same thing, but without +actually writing any command line scripts. + + +## How to use Elastic Galaxy Tools? + +Although any (Elastic) Galaxy tool is a wrapper of a command line tool which enables performing data analysis +through web interfaces without actually writing any command line script, understanding how to read command line +manuals will help you use Elastic Galaxy tools. + +Running a command line tool is very simple. You start with the tool name, then you specify options (also called flags) +that the tool can recognize. The tool manual should tell you which options the tool can recognize. An option may or may +not have a value followed. Some options are required and some are optional. You have to use all required options to +successfully run a command line tool. However, you can use as many optional options as you want in a call of a command +line tool. + +## How to develop an Elastic Galaxy Tool? + +### Publish a Galaxy tool + +We can use the tool **planemo** to easily publish our Galaxy tools to Tool Shed or test Tool Shed. +More details can be found **[here](http://planemo.readthedocs.io/en/latest/publishing.html)**. + +* Publish to test Tool Shed + +```bash +planemo shed_create --shed_target testtoolshed +``` + +* Update tool repository in test Tool Shed + +```bash +planemo shed_update --check_diff --shed_target testtoolshed +``` + +* Publish to Tool Shed + +```bash +planemo shed_create --shed_target toolshed +``` + +* Update tool repository in Tool Shed + +```bash +planemo shed_update --check_diff --shed_target toolshed +``` + + +
--- a/elastic_tool.Rmd Mon Mar 26 00:09:35 2018 -0400 +++ b/elastic_tool.Rmd Sun Apr 01 16:48:53 2018 -0400 @@ -1,5 +1,5 @@ --- -title: 'Dynamic tool' +title: 'Tool Report' output: html_document --- @@ -18,14 +18,13 @@ ## User input -```{r, echo=FALSE} -##-------- build script files ----------- - -# build script file: script.sh +```{r, 'display user input'} +# get user input and save it into a data frame. df = read.table(paste0(Sys.getenv('REPORT_FILES_PATH'), '/options_and_arguments.txt'), sep = '|', header = TRUE) -# prepend A_TOOL_OUTPUT_PATH +# if the input type is 'path_relative_to_a_tool', prepend A_TOOL_OUTPUT_PATH to the value to make +# the value a full path. if (nrow(df[df$type == 'path_relative_to_a_tool', ]) > 0) { for (i in 1:nrow(df[df$type == 'path_relative_to_a_tool', ])) { root_path = readLines(df[df$type == 'path_relative_to_a_tool', ][i, 'path_type'])[1] @@ -35,13 +34,16 @@ } } -# get tool name +## display user input as a table +knitr::kable(df) +``` + + +```{r, 'build script', echo=FALSE} +##-------- build script files ----------- + +# get tool name, the first line of the script is always the tool name. tool_name = df[df$type == 'tool_name', 'value'] -# build script for displaying help messages (this probably should be tool specific) -write(paste0(tool_name, ' -h'), - file = paste0(Sys.getenv('REPORT_FILES_PATH'), '/help.sh')) - -knitr::kable(df) # if the number of option/argument pairs is larger than 0, build script file df2 = df[df$type != 'tool_name', ] @@ -51,7 +53,7 @@ write(paste0(tool_name, ' \\'), paste0(Sys.getenv('REPORT_FILES_PATH'), '/script.sh')) df2 = df[df$type != 'tool_name', ] - write(paste(' ', df2$flag, df2$value, '\\', sep = ' '), + write(paste(' ', df2$option, df2$value, '\\', sep = ' '), file = paste0(Sys.getenv('REPORT_FILES_PATH'), '/script.sh'), append = TRUE ) # remember that after writing option/argument lines to the script.sh, the last line @@ -66,17 +68,13 @@ write(paste0(tool_name, ' -h'), file = paste0(Sys.getenv('REPORT_FILES_PATH'), '/script.sh')) } - ``` -```{bash, echo=FALSE} -## code to open help documentation -sh ${REPORT_FILES_PATH}/help.sh > ${TOOL_HELP_DOC} -``` +```{r, 'create paths if they do not exist', echo=FALSE} +## if the input type is 'path_relative_to_this_tool', that means +## we need to create a directory or file path. - -```{r, echo=FALSE} # create paths before running the job script df_paths = df[df$type == 'path_relative_to_this_tool', ] @@ -103,15 +101,14 @@ ``` -```{bash, echo=FALSE} -# run job script -# it's important to run the job within the REPORT_FILES_PATH -cd ${REPORT_FILES_PATH} && sh script.sh +```{bash, 'run jobs', echo=FALSE} +# run job script, always use absolute path. +# we want to run all jobs within the working path. +sh ${REPORT_FILES_PATH}/script.sh ``` - -```{bash, results='asis', echo=FALSE} +```{bash, 'display script', results='asis', echo=FALSE} echo '## Job script' echo '' echo '' @@ -121,16 +118,10 @@ ``` -```{r, echo=FALSE} -## obtain REPORT_FILES_PAHT and save it to a galaxy output. -database_root = paste(head(strsplit(Sys.getenv('TOOL_LOG'), '/')[[1]], -1), collapse = '/') -tool_output_dir_id = tail(strsplit(Sys.getenv('REPORT_FILES_PATH'), '/')[[1]], 1) -tool_output_dir = paste0(database_root, '/', tool_output_dir_id) -write(tool_output_dir, Sys.getenv('TOOL_OUTPUT_DIR')) -``` +```{r, 'display output directory contents', results='asis', echo=FALSE} +## after the job is done, we list all files from the output directory. +## full relative path to the output directory needs to be displayed. - -```{r, results='asis', echo=FALSE} cat('##All output files') cat('\n\n') all_files = list.files(path = Sys.getenv('REPORT_FILES_PATH'), @@ -143,3 +134,14 @@ cat('\n') ``` + +```{r, 'save output directory of this tool', echo=FALSE} +## each elastic tool has a galaxy history output which contains the REPORT_FILES_PATH of this tool +## so that other tools can reference the outputs from this tool. + +## obtain REPORT_FILES_PAHT and save it to a galaxy output. +database_root = paste(head(strsplit(Sys.getenv('TOOL_LOG'), '/')[[1]], -1), collapse = '/') +tool_output_dir_id = tail(strsplit(Sys.getenv('REPORT_FILES_PATH'), '/')[[1]], 1) +tool_output_dir = paste0(database_root, '/', tool_output_dir_id) +write(tool_output_dir, Sys.getenv('TOOL_OUTPUT_DIR')) +```
--- a/elastic_tool.xml Mon Mar 26 00:09:35 2018 -0400 +++ b/elastic_tool.xml Sun Apr 01 16:48:53 2018 -0400 @@ -1,26 +1,154 @@ -<?xml version="1.0" encoding="UTF-8"?> -<tool id="elastic_fastq_dump" name="elastic fastq-dump" version="1.0.0"> - <macros> - <import>elastic_tool_wrappers_macros.xml</import> - </macros> - <description/> - <requirements> - <expand macro="rmarkdown_requirements"/> - </requirements> - <expand macro="stdio"/> - <expand macro="shell_command"/> - <inputs> - <expand macro="tool_name"/> - <expand macro="option_argument_single_dataset"/> - <expand macro="option_argument_multiple_datasets"/> - <expand macro="option_argument_path_relative_to_a_tool"/> - <expand macro="option_argument_path_relative_to_this_tool"/> - <expand macro="option_argument_string_float_integer"/> - </inputs> - <outputs> - <expand macro="output_set"/> - </outputs> - <citations> - <expand macro="citations"/> - </citations> +<tool id="elastic_fastq_dump" name="elastic fastq-dump" version="1.0.1"> + <macros> + <import>elastic_tool_macros.xml</import> + </macros> + <expand macro="tool_description"/> + <requirements> + <expand macro="elastic_tool_requirements"/> + </requirements> + <stdio> + <regex match="XXX" source="stderr" level="warning" + description="Check the tool log output file for more details."/> + </stdio> + <command><![CDATA[ + + ############ first, create a directory to store all files + mkdir -p $report.files_path && + + + ############ save the tool installation directory to an environment variable + export TOOL_INSTALL_DIR='${__tool_directory__}' && + + + ######################### + ## + ## save user input option/argument pairs into a file + ## + ######################### + + ## first line will be the header + echo 'type|option|value|path_type' > $report.files_path/options_and_arguments.txt && + + ############ tool_name + ##-t '$tool_name' + echo "tool_name|not available|${tool_name}|not available" >> $report.files_path/options_and_arguments.txt && + + #################################################### + ## loop through repeats to get option/argument pairs + #################################################### + + ############ option_argument_single_dataset + #for i in $option_argument_single_dataset_repeat: + #set $item = 'single_dataset|' + str($i.option) + "|" + str($i.value) + "|" + echo '$item' >> $report.files_path/options_and_arguments.txt && + #end for + + ############ option_argument_multiple_datasets + #for i in $option_argument_multiple_datasets_repeat: + #set $item = 'multiple_datasets|' + str($i.option) + "|" + str($i.value).replace(',', str($i.delimiter)) + "|" + echo '$item' >> $report.files_path/options_and_arguments.txt && + #end for + + + ############ option_argument_path_relative_to_a_tool + #for i in $option_argument_path_relative_to_a_tool_repeat: + #set $item = 'path_relative_to_a_tool|' + str($i.option) + "|" + str($i.value) + "|" + str($i.a_tool_output_dir) + echo '$item' >> $report.files_path/options_and_arguments.txt && + #end for + + ############ option_argument_path_relative_to_this_tool + #for i in $option_argument_path_relative_to_this_tool_repeat: + #set $item = 'path_relative_to_this_tool|' + str($i.option) + "|" + str($i.value) + "|" + str($i.path_type) + echo '$item' >> $report.files_path/options_and_arguments.txt && + #end for + + ############ option_argument_string_float_integer + #for i in $option_argument_string_float_integer_repeat: + #set $item = 'value|' + str($i.option) + "|" + str($i.value) + "|" + echo '$item' >> $report.files_path/options_and_arguments.txt && + #end for + + ############ output_set + ## '$report' + ## '$report.files_path' + ## '$tool_output_dir' + ## '$tool_log' + ####################### + export REPORT='$report' && + export REPORT_FILES_PATH='$report.files_path' && + export TOOL_OUTPUT_DIR='$tool_output_dir' && + export TOOL_LOG='$tool_log' && + + + ############ run render R script to render R markdowns + Rscript '${__tool_directory__}/elastic_tool_render.R' + + ]]></command> + <inputs> + <param type="select" name="tool_name" multiple="false" label="Tool name"> + <expand macro="tool_options"/> + </param> + <repeat name="option_argument_single_dataset_repeat" title="INPUT DATA FROM HISTORY: single data set" min="0" + default="0"> + <param type="text" name="option" optional="true" label="option"/> + <param name="value" type="data" optional="false" multiple="false" label="value"/> + </repeat> + <repeat name="option_argument_multiple_datasets_repeat" title="INPUT DATA FROM HISTORY: multiple data sets" + min="0" default="0"> + <param type="text" name="option" optional="true" label="option"/> + <param name="value" type="data" optional="false" multiple="true" label="value"/> + <param type="select" name="delimiter" multiple="false" label="file delimiter"> + <option value=" " selected="false">space</option> + <option value="," selected="false">comma</option> + </param> + </repeat> + <repeat name="option_argument_path_relative_to_a_tool_repeat" + title="INPUT DATA PATH: relative to a tool output directory" min="0" default="0"> + <param type="text" name="option" optional="true" label="option"/> + <param name="a_tool_output_dir" type="data" optional="false" multiple="false" + label="a tool output directory"/> + <param type="text" name="value" optional="true" label="value"/> + </repeat> + <repeat name="option_argument_string_float_integer_repeat" title="INPUT VALUE: string/float/integer" min="0" + default="0"> + <param type="text" name="option" label="option"/> + <param type="text" name="value" label="value"/> + </repeat> + <repeat name="option_argument_path_relative_to_this_tool_repeat" + title="OUTPUT PATH: relative to this tool output directory" min="0" default="0"> + <param type="text" name="option" optional="true" label="option"/> + <param type="text" name="value" optional="false" label="value"/> + <param type="select" name="path_type" multiple="false" label="path type"> + <option value="file_path" selected="false">file path</option> + <option value="dir_path" selected="false">directory path</option> + </param> + </repeat> + </inputs> + <outputs> + <data format="html" name="report" label="${tool.name} report ${on_string}"/> + <data format="txt" name="tool_output_dir" label="${tool.name} output directory ${on_string}"/> + <data format="txt" name="tool_log" label="${tool.name} log ${on_string}"/> + </outputs> + <citations> + <citation type="bibtex"><![CDATA[ + @article{allaire2016rmarkdown, + title={rmarkdown: Dynamic Documents for R, 2016}, + author={Allaire, J and Cheng, Joe and Xie, Yihui and McPherson, Jonathan and Chang, Winston and Allen, Jeff + and Wickham, Hadley and Atkins, Aron and Hyndman, Rob}, + journal={R package version 0.9}, + volume={6}, + year={2016} + } + ]]></citation> + <citation type="bibtex"><![CDATA[ + @book{xie2015elastic, + title={Dynamic Documents with R and knitr}, + author={Xie, Yihui}, + volume={29}, + year={2015}, + publisher={CRC Press} + } + ]]></citation> + <expand macro="citations"/> + </citations> </tool>
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/elastic_tool_macros.xml Sun Apr 01 16:48:53 2018 -0400 @@ -0,0 +1,31 @@ +<macros> + + <!--tool requirements--> + <xml name="tool_requirements"> + <requirement type="package" version="1.15.0.6-0">pandoc</requirement> + <requirement type="package" version="1.6">r-rmarkdown</requirement> + <requirement type="package" version="0.6.2">parallel-fastq-dump</requirement> + </xml> + + <!--tool description--> + <xml name="tool_description"> + <description>fastq-dump tool wrapper with elastic user interface.</description> + </xml> + + <!--tool options--> + <xml name="tool_options"> + <option value="tool_command" selected="false">tool command</option> + </xml> + + <!--tool help--> + <xml name="tool_help"> + <help><![CDATA[ + Here is your tool help documentation. + ]]></help> + </xml> + + <!--tool citations--> + <xml name="citations"> + <citation type="doi"></citation> + </xml> +</macros>
--- a/elastic_tool_wrappers_macros.xml Mon Mar 26 00:09:35 2018 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,169 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<macros> - <xml name="rmarkdown_requirements"> - <requirement type="package" version="1.15.0.6-0">pandoc</requirement> - <requirement type="package" version="1.6">r-rmarkdown</requirement> - <requirement type="package" version="0.6.2">parallel-fastq-dump</requirement> - </xml> - <xml name="stdio"> - <stdio> - <regex match="XXX" source="stderr" level="warning" description="Check the tool log output file for more details."/> - </stdio> - </xml> - <!--Tool name--> - <xml name="tool_name"> - <param type="select" name="tool_name" multiple="false" label="Tool name"> - <option value="fastq-dump" selected="false">fastq-dump</option></param> - </xml> - <!--OPTION/ARGUMENT PAIRS and INPUT types--> - <!--input files--> - <xml name="option_argument_single_dataset"> - <repeat name="option_argument_single_dataset_repeat" title="INPUT DATA FROM HISTORY: single data set" min="0" default="0"> - <param type="text" name="flag" optional="true" label="flag"/> - <param name="value" type="data" optional="false" multiple="false" label="value"/> - </repeat> - </xml> - <xml name="option_argument_multiple_datasets"> - <repeat name="option_argument_multiple_datasets_repeat" title="INPUT DATA FROM HISTORY: multiple data sets" min="0" default="0"> - <param type="text" name="flag" optional="true" label="flag"/> - <param name="value" type="data" optional="false" multiple="true" label="value"/> - <param type="select" name="delimiter" multiple="false" label="file delimiter"> - <option value=" " selected="false">space</option> - <option value="," selected="false">comma</option> - </param> - </repeat> - </xml> - <!--input path--> - <xml name="option_argument_path_relative_to_a_tool"> - <repeat name="option_argument_path_relative_to_a_tool_repeat" title="INPUT DATA PATH: relative to a tool output directory" min="0" default="0"> - <param type="text" name="flag" optional="true" label="flag"/> - <param name="a_tool_output_dir" type="data" optional="false" multiple="false" label="a tool output directory"/> - <param type="text" name="value" optional="true" label="value"/> - </repeat> - </xml> - <xml name="option_argument_path_relative_to_this_tool"> - <repeat name="option_argument_path_relative_to_this_tool_repeat" title="OUTPUT PATH: relative to this tool output directory" min="0" default="0"> - <param type="text" name="flag" optional="true" label="flag"/> - <param type="text" name="value" optional="false" label="value"/> - <param type="select" name="path_type" multiple="false" label="path type"> - <option value="file_path" selected="false">file path</option> - <option value="dir_path" selected="false">directory path</option> - </param> - </repeat> - </xml> - <!--input values: string, float and integer--> - <xml name="option_argument_string_float_integer"> - <repeat name="option_argument_string_float_integer_repeat" title="INPUT VALUE: string/float/integer" min="0" default="0"> - <param type="text" name="flag" label="flag"/> - <param type="text" name="value" label="value"/> - </repeat> - </xml> - <!--output set--> - <xml name="output_set"> - <data format="html" name="report" label="${tool.name} report"/> - <data format="txt" name="tool_help_doc" label="${tool.name} help doc"/> - <data format="txt" name="tool_output_dir" label="${tool.name} output directory"/> - <data format="txt" name="tool_log" label="${tool.name} log"/> - </xml> - <!--shell command fragment--> - <xml name="shell_command"> - <command><![CDATA[ - - ############ first, create a directory to store all files - mkdir -p $report.files_path && cd $report.files_path && - - - ############ save the tool installation directory to an environment variable - export TOOL_INSTALL_DIR='${__tool_directory__}' && - - - ######################### - ## - ## save user input option/argument pairs into a file - ## - ######################### - - ## first line will be the header - echo 'type|flag|value|path_type' > options_and_arguments.txt && - - ############ tool_name - ##-t '$tool_name' - echo "tool_name|not available|${tool_name}|not available" >> options_and_arguments.txt && - - #################################################### - ## loop through repeats to get option/argument pairs - #################################################### - - ############ option_argument_single_dataset - #for i in $option_argument_single_dataset_repeat: - #set $item = 'single_dataset|' + str($i.flag) + "|" + str($i.value) + "|" - echo '$item' >> options_and_arguments.txt && - #end for - - ############ option_argument_multiple_datasets - #for i in $option_argument_multiple_datasets_repeat: - #set $item = 'multiple_datasets|' + str($i.flag) + "|" + str($i.value).replace(',', str($i.delimiter)) + "|" - echo '$item' >> options_and_arguments.txt && - #end for - - - ############ option_argument_path_relative_to_a_tool - #for i in $option_argument_path_relative_to_a_tool_repeat: - #set $item = 'path_relative_to_a_tool|' + str($i.flag) + "|" + str($i.value) + "|" + str($i.a_tool_output_dir) - echo '$item' >> options_and_arguments.txt && - #end for - - ############ option_argument_path_relative_to_this_tool - #for i in $option_argument_path_relative_to_this_tool_repeat: - #set $item = 'path_relative_to_this_tool|' + str($i.flag) + "|" + str($i.value) + "|" + str($i.path_type) - echo '$item' >> options_and_arguments.txt && - #end for - - ############ option_argument_string_float_integer - #for i in $option_argument_string_float_integer_repeat: - #set $item = 'value|' + str($i.flag) + "|" + str($i.value) + "|" - echo '$item' >> options_and_arguments.txt && - #end for - - ############ output_set - ## '$report' - ## '$report.files_path' - ## '$tool_help_doc' - ## '$tool_output_dir' - ## '$tool_log' - ####################### - export REPORT='$report' && - export REPORT_FILES_PATH='$report.files_path' && - export TOOL_HELP_DOC='$tool_help_doc' && - export TOOL_OUTPUT_DIR='$tool_output_dir' && - export TOOL_LOG='$tool_log' && - - - ############ run render R script to render R markdowns - Rscript '${__tool_directory__}/elastic_tool_render.R' - - ]]></command> - </xml> - <!--tool citations--> - <xml name="citations"> - <citation type="bibtex"><![CDATA[ - @article{allaire2016rmarkdown, - title={rmarkdown: Dynamic Documents for R, 2016}, - author={Allaire, J and Cheng, Joe and Xie, Yihui and McPherson, Jonathan and Chang, Winston and Allen, Jeff - and Wickham, Hadley and Atkins, Aron and Hyndman, Rob}, - journal={R package version 0.9}, - volume={6}, - year={2016} - } - ]]></citation> - <citation type="bibtex"><![CDATA[ - @book{xie2015elastic, - title={Dynamic Documents with R and knitr}, - author={Xie, Yihui}, - volume={29}, - year={2015}, - publisher={CRC Press} - } - ]]></citation> - </xml> -</macros>