Mercurial > repos > fubar > tool_factory_2
changeset 3:1c652687a08f draft default tip
Uploaded
| author | fubar |
|---|---|
| date | Fri, 30 Apr 2021 07:06:57 +0000 |
| parents | 5fc0c9a93072 |
| children | |
| files | toolfactory/ToolFactory.py toolfactory/ToolFactory.xml toolfactory/ToolFactory_tester.xml toolfactory/install-history.py toolfactory/install_tf_demos.py toolfactory/rgToolFactory2.py toolfactory/rgToolFactory2.xml toolfactory/testclass.py toolfactory/toolwatcher.py |
| diffstat | 9 files changed, 2293 insertions(+), 1762 deletions(-) [+] |
line wrap: on
line diff
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/toolfactory/ToolFactory.py Fri Apr 30 07:06:57 2021 +0000 @@ -0,0 +1,975 @@ + +# see https://github.com/fubar2/toolfactory +# +# copyright ross lazarus (ross stop lazarus at gmail stop com) May 2012 +# +# all rights reserved +# Licensed under the LGPL +# suggestions for improvement and bug fixes welcome at +# https://github.com/fubar2/toolfactory +# +# April 2021: Refactored into two tools - generate and test/install +# as part of GTN tutorial development and biocontainer adoption +# The tester runs planemo on a non-tested archive, creates the test outputs +# and returns a new proper tool with test. +# The tester was generated from the ToolFactory_tester.py script + + +import argparse +import copy +import json +import logging +import os +import re +import shlex +import shutil +import subprocess +import sys +import tarfile +import tempfile +import time +import urllib + +from bioblend import ConnectionError +from bioblend import galaxy +from bioblend import toolshed + +import galaxyxml.tool as gxt +import galaxyxml.tool.parameters as gxtp + +import lxml.etree as ET + +import yaml + +myversion = "V2.3 April 2021" +verbose = True +debug = True +toolFactoryURL = "https://github.com/fubar2/toolfactory" +FAKEEXE = "~~~REMOVE~~~ME~~~" +# need this until a PR/version bump to fix galaxyxml prepending the exe even +# with override. + + +def timenow(): + """return current time as a string""" + return time.strftime("%d/%m/%Y %H:%M:%S", time.localtime(time.time())) + +cheetah_escape_table = {"$": "\\$", "#": "\\#"} + +def cheetah_escape(text): + """Produce entities within text.""" + return "".join([cheetah_escape_table.get(c, c) for c in text]) + +def parse_citations(citations_text): + """""" + citations = [c for c in citations_text.split("**ENTRY**") if c.strip()] + citation_tuples = [] + for citation in citations: + if citation.startswith("doi"): + citation_tuples.append(("doi", citation[len("doi") :].strip())) + else: + citation_tuples.append(("bibtex", citation[len("bibtex") :].strip())) + return citation_tuples + + +class Tool_Conf_Updater(): + # update config/tool_conf.xml with a new tool unpacked in /tools + # requires highly insecure docker settings - like write to tool_conf.xml and to tools ! + # if in a container possibly not so courageous. + # Fine on your own laptop but security red flag for most production instances + + def __init__(self, args, tool_conf_path, new_tool_archive_path, new_tool_name, tool_dir): + self.args = args + self.tool_conf_path = os.path.join(args.galaxy_root,tool_conf_path) + self.tool_dir = os.path.join(args.galaxy_root, tool_dir) + self.our_name = 'ToolFactory' + tff = tarfile.open(new_tool_archive_path, "r:*") + flist = tff.getnames() + ourdir = os.path.commonpath(flist) # eg pyrevpos + self.tool_id = ourdir # they are the same for TF tools + ourxml = [x for x in flist if x.lower().endswith('.xml')] + res = tff.extractall() + tff.close() + self.run_rsync(ourdir, self.tool_dir) + self.update_toolconf(ourdir,ourxml) + + def run_rsync(self, srcf, dstf): + src = os.path.abspath(srcf) + dst = os.path.abspath(dstf) + if os.path.isdir(src): + cll = ['rsync', '-vr', src, dst] + else: + cll = ['rsync', '-v', src, dst] + p = subprocess.run( + cll, + capture_output=False, + encoding='utf8', + shell=False, + ) + + def install_deps(self): + gi = galaxy.GalaxyInstance(url=self.args.galaxy_url, key=self.args.galaxy_api_key) + x = gi.tools.install_dependencies(self.tool_id) + print(f"Called install_dependencies on {self.tool_id} - got {x}") + + def update_toolconf(self,ourdir,ourxml): # path is relative to tools + updated = False + localconf = './local_tool_conf.xml' + self.run_rsync(self.tool_conf_path,localconf) + tree = ET.parse(localconf) + root = tree.getroot() + hasTF = False + TFsection = None + for e in root.findall('section'): + if e.attrib['name'] == self.our_name: + hasTF = True + TFsection = e + if not hasTF: + TFsection = ET.Element('section') + root.insert(0,TFsection) # at the top! + our_tools = TFsection.findall('tool') + conf_tools = [x.attrib['file'] for x in our_tools] + for xml in ourxml: # may be > 1 + if not xml in conf_tools: # new + updated = True + ET.SubElement(TFsection, 'tool', {'file':xml}) + ET.indent(tree) + newconf = f"{self.tool_id}_conf" + tree.write(newconf, pretty_print=True) + self.run_rsync(newconf,self.tool_conf_path) + if False and self.args.packages and self.args.packages > '': + self.install_deps() + +class Tool_Factory: + """Wrapper for an arbitrary script + uses galaxyxml + + """ + + def __init__(self, args=None): # noqa + """ + prepare command line cl for running the tool here + and prepare elements needed for galaxyxml tool generation + """ + self.ourcwd = os.getcwd() + self.collections = [] + if len(args.collection) > 0: + try: + self.collections = [ + json.loads(x) for x in args.collection if len(x.strip()) > 1 + ] + except Exception: + print( + f"--collections parameter {str(args.collection)} is malformed - should be a dictionary" + ) + try: + self.infiles = [ + json.loads(x) for x in args.input_files if len(x.strip()) > 1 + ] + except Exception: + print( + f"--input_files parameter {str(args.input_files)} is malformed - should be a dictionary" + ) + try: + self.outfiles = [ + json.loads(x) for x in args.output_files if len(x.strip()) > 1 + ] + except Exception: + print( + f"--output_files parameter {args.output_files} is malformed - should be a dictionary" + ) + try: + self.addpar = [ + json.loads(x) for x in args.additional_parameters if len(x.strip()) > 1 + ] + except Exception: + print( + f"--additional_parameters {args.additional_parameters} is malformed - should be a dictionary" + ) + try: + self.selpar = [ + json.loads(x) for x in args.selecttext_parameters if len(x.strip()) > 1 + ] + except Exception: + print( + f"--selecttext_parameters {args.selecttext_parameters} is malformed - should be a dictionary" + ) + self.args = args + self.cleanuppar() + self.lastxclredirect = None + self.xmlcl = [] + self.is_positional = self.args.parampass == "positional" + if self.args.sysexe: + if ' ' in self.args.sysexe: + self.executeme = self.args.sysexe.split(' ') + else: + self.executeme = [self.args.sysexe, ] + else: + if self.args.packages: + self.executeme = [self.args.packages.split(",")[0].split(":")[0].strip(), ] + else: + self.executeme = None + aXCL = self.xmlcl.append + assert args.parampass in [ + "0", + "argparse", + "positional", + ], 'args.parampass must be "0","positional" or "argparse"' + self.tool_name = re.sub("[^a-zA-Z0-9_]+", "", args.tool_name) + self.tool_id = self.tool_name + self.newtool = gxt.Tool( + self.tool_name, + self.tool_id, + self.args.tool_version, + self.args.tool_desc, + FAKEEXE, + ) + self.newtarpath = "%s_toolshed.gz" % self.tool_name + self.tooloutdir = "./tfout" + self.repdir = "./TF_run_report" + self.testdir = os.path.join(self.tooloutdir, "test-data") + if not os.path.exists(self.tooloutdir): + os.mkdir(self.tooloutdir) + if not os.path.exists(self.testdir): + os.mkdir(self.testdir) + if not os.path.exists(self.repdir): + os.mkdir(self.repdir) + self.tinputs = gxtp.Inputs() + self.toutputs = gxtp.Outputs() + self.testparam = [] + if self.args.script_path: + self.prepScript() + if self.args.command_override: + scos = open(self.args.command_override, "r").readlines() + self.command_override = [x.rstrip() for x in scos] + else: + self.command_override = None + if self.args.test_override: + stos = open(self.args.test_override, "r").readlines() + self.test_override = [x.rstrip() for x in stos] + else: + self.test_override = None + if self.args.script_path: + for ex in self.executeme: + aXCL(ex) + aXCL("$runme") + else: + for ex in self.executeme: + aXCL(ex) + + if self.args.parampass == "0": + self.clsimple() + else: + if self.args.parampass == "positional": + self.prepclpos() + self.clpositional() + else: + self.prepargp() + self.clargparse() + + def clsimple(self): + """no parameters or repeats - uses < and > for i/o""" + aXCL = self.xmlcl.append + if len(self.infiles) > 0: + aXCL("<") + aXCL("$%s" % self.infiles[0]["infilename"]) + if len(self.outfiles) > 0: + aXCL(">") + aXCL("$%s" % self.outfiles[0]["name"]) + if self.args.cl_user_suffix: # DIY CL end + clp = shlex.split(self.args.cl_user_suffix) + for c in clp: + aXCL(c) + + def prepargp(self): + xclsuffix = [] + for i, p in enumerate(self.infiles): + nam = p["infilename"] + if p["origCL"].strip().upper() == "STDIN": + xappendme = [ + nam, + nam, + "< $%s" % nam, + ] + else: + rep = p["repeat"] == "1" + over = "" + if rep: + over = f'#for $rep in $R_{nam}:\n--{nam} "$rep.{nam}"\n#end for' + xappendme = [p["CL"], "$%s" % p["CL"], over] + xclsuffix.append(xappendme) + for i, p in enumerate(self.outfiles): + if p["origCL"].strip().upper() == "STDOUT": + self.lastxclredirect = [">", "$%s" % p["name"]] + else: + xclsuffix.append([p["name"], "$%s" % p["name"], ""]) + for p in self.addpar: + nam = p["name"] + rep = p["repeat"] == "1" + if rep: + over = f'#for $rep in $R_{nam}:\n--{nam} "$rep.{nam}"\n#end for' + else: + over = p["override"] + xclsuffix.append([p["CL"], '"$%s"' % nam, over]) + for p in self.selpar: + xclsuffix.append([p["CL"], '"$%s"' % p["name"], p["override"]]) + self.xclsuffix = xclsuffix + + def prepclpos(self): + xclsuffix = [] + for i, p in enumerate(self.infiles): + if p["origCL"].strip().upper() == "STDIN": + xappendme = [ + "999", + p["infilename"], + "< $%s" % p["infilename"], + ] + else: + xappendme = [p["CL"], "$%s" % p["infilename"], ""] + xclsuffix.append(xappendme) + for i, p in enumerate(self.outfiles): + if p["origCL"].strip().upper() == "STDOUT": + self.lastxclredirect = [">", "$%s" % p["name"]] + else: + xclsuffix.append([p["CL"], "$%s" % p["name"], ""]) + for p in self.addpar: + nam = p["name"] + rep = p["repeat"] == "1" # repeats make NO sense + if rep: + print(f'### warning. Repeats for {nam} ignored - not permitted in positional parameter command lines!') + over = p["override"] + xclsuffix.append([p["CL"], '"$%s"' % nam, over]) + for p in self.selpar: + xclsuffix.append([p["CL"], '"$%s"' % p["name"], p["override"]]) + xclsuffix.sort() + self.xclsuffix = xclsuffix + + def prepScript(self): + rx = open(self.args.script_path, "r").readlines() + rx = [x.rstrip() for x in rx] + rxcheck = [x.strip() for x in rx if x.strip() > ""] + assert len(rxcheck) > 0, "Supplied script is empty. Cannot run" + self.script = "\n".join(rx) + fhandle, self.sfile = tempfile.mkstemp( + prefix=self.tool_name, suffix="_%s" % (self.executeme[0]) + ) + tscript = open(self.sfile, "w") + tscript.write(self.script) + tscript.close() + self.spacedScript = [f" {x}" for x in rx if x.strip() > ""] + rx.insert(0,'#raw') + rx.append('#end raw') + self.escapedScript = rx + art = "%s.%s" % (self.tool_name, self.executeme[0]) + artifact = open(art, "wb") + artifact.write(bytes(self.script, "utf8")) + artifact.close() + + def cleanuppar(self): + """ positional parameters are complicated by their numeric ordinal""" + if self.args.parampass == "positional": + for i, p in enumerate(self.infiles): + assert ( + p["CL"].isdigit() or p["CL"].strip().upper() == "STDIN" + ), "Positional parameters must be ordinal integers - got %s for %s" % ( + p["CL"], + p["label"], + ) + for i, p in enumerate(self.outfiles): + assert ( + p["CL"].isdigit() or p["CL"].strip().upper() == "STDOUT" + ), "Positional parameters must be ordinal integers - got %s for %s" % ( + p["CL"], + p["name"], + ) + for i, p in enumerate(self.addpar): + assert p[ + "CL" + ].isdigit(), "Positional parameters must be ordinal integers - got %s for %s" % ( + p["CL"], + p["name"], + ) + for i, p in enumerate(self.infiles): + infp = copy.copy(p) + infp["origCL"] = infp["CL"] + if self.args.parampass in ["positional", "0"]: + infp["infilename"] = infp["label"].replace(" ", "_") + else: + infp["infilename"] = infp["CL"] + self.infiles[i] = infp + for i, p in enumerate(self.outfiles): + p["origCL"] = p["CL"] # keep copy + self.outfiles[i] = p + for i, p in enumerate(self.addpar): + p["origCL"] = p["CL"] + self.addpar[i] = p + + def clpositional(self): + # inputs in order then params + aXCL = self.xmlcl.append + for (k, v, koverride) in self.xclsuffix: + aXCL(v) + if self.lastxclredirect: + aXCL(self.lastxclredirect[0]) + aXCL(self.lastxclredirect[1]) + if self.args.cl_user_suffix: # DIY CL end + clp = shlex.split(self.args.cl_user_suffix) + for c in clp: + aXCL(c) + + + def clargparse(self): + """argparse style""" + aXCL = self.xmlcl.append + # inputs then params in argparse named form + + for (k, v, koverride) in self.xclsuffix: + if koverride > "": + k = koverride + aXCL(k) + else: + if len(k.strip()) == 1: + k = "-%s" % k + else: + k = "--%s" % k + aXCL(k) + aXCL(v) + if self.lastxclredirect: + aXCL(self.lastxclredirect[0]) + aXCL(self.lastxclredirect[1]) + if self.args.cl_user_suffix: # DIY CL end + clp = shlex.split(self.args.cl_user_suffix) + for c in clp: + aXCL(c) + + def getNdash(self, newname): + if self.is_positional: + ndash = 0 + else: + ndash = 2 + if len(newname) < 2: + ndash = 1 + return ndash + + def doXMLparam(self): # noqa + """Add all needed elements to tool""" + for p in self.outfiles: + newname = p["name"] + newfmt = p["format"] + newcl = p["CL"] + test = p["test"] + oldcl = p["origCL"] + test = test.strip() + ndash = self.getNdash(newcl) + aparm = gxtp.OutputData( + name=newname, format=newfmt, num_dashes=ndash, label=newname + ) + aparm.positional = self.is_positional + if self.is_positional: + if oldcl.upper() == "STDOUT": + aparm.positional = 9999999 + aparm.command_line_override = "> $%s" % newname + else: + aparm.positional = int(oldcl) + aparm.command_line_override = "$%s" % newname + self.toutputs.append(aparm) + ld = None + if test.strip() > "": + if test.startswith("diff"): + c = "diff" + ld = 0 + if test.split(":")[1].isdigit: + ld = int(test.split(":")[1]) + tp = gxtp.TestOutput( + name=newname, + value="%s_sample" % newname, + compare=c, + lines_diff=ld, + ) + elif test.startswith("sim_size"): + c = "sim_size" + tn = test.split(":")[1].strip() + if tn > "": + if "." in tn: + delta = None + delta_frac = min(1.0, float(tn)) + else: + delta = int(tn) + delta_frac = None + tp = gxtp.TestOutput( + name=newname, + value="%s_sample" % newname, + compare=c, + delta=delta, + delta_frac=delta_frac, + ) + else: + c = test + tp = gxtp.TestOutput( + name=newname, + value="%s_sample" % newname, + compare=c, + ) + self.testparam.append(tp) + for p in self.infiles: + newname = p["infilename"] + newfmt = p["format"] + ndash = self.getNdash(newname) + reps = p.get("repeat", "0") == "1" + if not len(p["label"]) > 0: + alab = p["CL"] + else: + alab = p["label"] + aninput = gxtp.DataParam( + newname, + optional=False, + label=alab, + help=p["help"], + format=newfmt, + multiple=False, + num_dashes=ndash, + ) + aninput.positional = self.is_positional + if self.is_positional: + if p["origCL"].upper() == "STDIN": + aninput.positional = 9999998 + aninput.command_line_override = "> $%s" % newname + else: + aninput.positional = int(p["origCL"]) + aninput.command_line_override = "$%s" % newname + if reps: + repe = gxtp.Repeat(name=f"R_{newname}", title=f"Add as many {alab} as needed") + repe.append(aninput) + self.tinputs.append(repe) + tparm = gxtp.TestRepeat(name=f"R_{newname}") + tparm2 = gxtp.TestParam(newname, value="%s_sample" % newname) + tparm.append(tparm2) + self.testparam.append(tparm) + else: + self.tinputs.append(aninput) + tparm = gxtp.TestParam(newname, value="%s_sample" % newname) + self.testparam.append(tparm) + for p in self.addpar: + newname = p["name"] + newval = p["value"] + newlabel = p["label"] + newhelp = p["help"] + newtype = p["type"] + newcl = p["CL"] + oldcl = p["origCL"] + reps = p["repeat"] == "1" + if not len(newlabel) > 0: + newlabel = newname + ndash = self.getNdash(newname) + if newtype == "text": + aparm = gxtp.TextParam( + newname, + label=newlabel, + help=newhelp, + value=newval, + num_dashes=ndash, + ) + elif newtype == "integer": + aparm = gxtp.IntegerParam( + newname, + label=newlabel, + help=newhelp, + value=newval, + num_dashes=ndash, + ) + elif newtype == "float": + aparm = gxtp.FloatParam( + newname, + label=newlabel, + help=newhelp, + value=newval, + num_dashes=ndash, + ) + elif newtype == "boolean": + aparm = gxtp.BooleanParam( + newname, + label=newlabel, + help=newhelp, + value=newval, + num_dashes=ndash, + ) + else: + raise ValueError( + 'Unrecognised parameter type "%s" for\ + additional parameter %s in makeXML' + % (newtype, newname) + ) + aparm.positional = self.is_positional + if self.is_positional: + aparm.positional = int(oldcl) + if reps: + repe = gxtp.Repeat(name=f"R_{newname}", title=f"Add as many {newlabel} as needed") + repe.append(aparm) + self.tinputs.append(repe) + tparm = gxtp.TestRepeat(name=f"R_{newname}") + tparm2 = gxtp.TestParam(newname, value=newval) + tparm.append(tparm2) + self.testparam.append(tparm) + else: + self.tinputs.append(aparm) + tparm = gxtp.TestParam(newname, value=newval) + self.testparam.append(tparm) + for p in self.selpar: + newname = p["name"] + newval = p["value"] + newlabel = p["label"] + newhelp = p["help"] + newtype = p["type"] + newcl = p["CL"] + if not len(newlabel) > 0: + newlabel = newname + ndash = self.getNdash(newname) + if newtype == "selecttext": + newtext = p["texts"] + aparm = gxtp.SelectParam( + newname, + label=newlabel, + help=newhelp, + num_dashes=ndash, + ) + for i in range(len(newval)): + anopt = gxtp.SelectOption( + value=newval[i], + text=newtext[i], + ) + aparm.append(anopt) + aparm.positional = self.is_positional + if self.is_positional: + aparm.positional = int(newcl) + self.tinputs.append(aparm) + tparm = gxtp.TestParam(newname, value=newval) + self.testparam.append(tparm) + else: + raise ValueError( + 'Unrecognised parameter type "%s" for\ + selecttext parameter %s in makeXML' + % (newtype, newname) + ) + for p in self.collections: + newkind = p["kind"] + newname = p["name"] + newlabel = p["label"] + newdisc = p["discover"] + collect = gxtp.OutputCollection(newname, label=newlabel, type=newkind) + disc = gxtp.DiscoverDatasets( + pattern=newdisc, directory=f"{newname}", visible="false" + ) + collect.append(disc) + self.toutputs.append(collect) + try: + tparm = gxtp.TestOutputCollection(newname) # broken until PR merged. + self.testparam.append(tparm) + except Exception: + print("#### WARNING: Galaxyxml version does not have the PR merged yet - tests for collections must be over-ridden until then!") + + def doNoXMLparam(self): + """filter style package - stdin to stdout""" + if len(self.infiles) > 0: + alab = self.infiles[0]["label"] + if len(alab) == 0: + alab = self.infiles[0]["infilename"] + max1s = ( + "Maximum one input if parampass is 0 but multiple input files supplied - %s" + % str(self.infiles) + ) + assert len(self.infiles) == 1, max1s + newname = self.infiles[0]["infilename"] + aninput = gxtp.DataParam( + newname, + optional=False, + label=alab, + help=self.infiles[0]["help"], + format=self.infiles[0]["format"], + multiple=False, + num_dashes=0, + ) + aninput.command_line_override = "< $%s" % newname + aninput.positional = True + self.tinputs.append(aninput) + tp = gxtp.TestParam(name=newname, value="%s_sample" % newname) + self.testparam.append(tp) + if len(self.outfiles) > 0: + newname = self.outfiles[0]["name"] + newfmt = self.outfiles[0]["format"] + anout = gxtp.OutputData(newname, format=newfmt, num_dashes=0) + anout.command_line_override = "> $%s" % newname + anout.positional = self.is_positional + self.toutputs.append(anout) + tp = gxtp.TestOutput(name=newname, value="%s_sample" % newname) + self.testparam.append(tp) + + def makeXML(self): # noqa + """ + Create a Galaxy xml tool wrapper for the new script + Uses galaxyhtml + Hmmm. How to get the command line into correct order... + """ + if self.command_override: + self.newtool.command_override = self.command_override # config file + else: + self.newtool.command_override = self.xmlcl + cite = gxtp.Citations() + acite = gxtp.Citation(type="doi", value="10.1093/bioinformatics/bts573") + cite.append(acite) + self.newtool.citations = cite + safertext = "" + if self.args.help_text: + helptext = open(self.args.help_text, "r").readlines() + safertext = "\n".join([cheetah_escape(x) for x in helptext]) + if len(safertext.strip()) == 0: + safertext = ( + "Ask the tool author (%s) to rebuild with help text please\n" + % (self.args.user_email) + ) + if self.args.script_path: + if len(safertext) > 0: + safertext = safertext + "\n\n------\n" # transition allowed! + scr = [x for x in self.spacedScript if x.strip() > ""] + scr.insert(0, "\n\nScript::\n") + if len(scr) > 300: + scr = ( + scr[:100] + + [" >300 lines - stuff deleted", " ......"] + + scr[-100:] + ) + scr.append("\n") + safertext = safertext + "\n".join(scr) + self.newtool.help = safertext + self.newtool.version_command = f'echo "{self.args.tool_version}"' + std = gxtp.Stdios() + std1 = gxtp.Stdio() + std.append(std1) + self.newtool.stdios = std + requirements = gxtp.Requirements() + if self.args.packages: + try: + for d in self.args.packages.split(","): + ver = "" + d = d.replace("==", ":") + d = d.replace("=", ":") + if ":" in d: + packg, ver = d.split(":") + else: + packg = d + requirements.append( + gxtp.Requirement("package", packg.strip(), ver.strip()) + ) + except Exception: + print('### malformed packages string supplied - cannot parse =',self.args.packages) + sys.exit(2) + self.newtool.requirements = requirements + if self.args.parampass == "0": + self.doNoXMLparam() + else: + self.doXMLparam() + self.newtool.outputs = self.toutputs + self.newtool.inputs = self.tinputs + if self.args.script_path: + configfiles = gxtp.Configfiles() + configfiles.append( + gxtp.Configfile(name="runme", text="\n".join(self.escapedScript)) + ) + self.newtool.configfiles = configfiles + tests = gxtp.Tests() + test_a = gxtp.Test() + for tp in self.testparam: + test_a.append(tp) + tests.append(test_a) + self.newtool.tests = tests + self.newtool.add_comment( + "Created by %s at %s using the Galaxy Tool Factory." + % (self.args.user_email, timenow()) + ) + self.newtool.add_comment("Source in git at: %s" % (toolFactoryURL)) + exml0 = self.newtool.export() + exml = exml0.replace(FAKEEXE, "") # temporary work around until PR accepted + if ( + self.test_override + ): # cannot do this inside galaxyxml as it expects lxml objects for tests + part1 = exml.split("<tests>")[0] + part2 = exml.split("</tests>")[1] + fixed = "%s\n%s\n%s" % (part1, "\n".join(self.test_override), part2) + exml = fixed + # exml = exml.replace('range="1:"', 'range="1000:"') + xf = open("%s.xml" % self.tool_name, "w") + xf.write(exml) + xf.write("\n") + xf.close() + # ready for the tarball + + def writeShedyml(self): + """for planemo""" + yuser = self.args.user_email.split("@")[0] + yfname = os.path.join(self.tooloutdir, ".shed.yml") + yamlf = open(yfname, "w") + odict = { + "name": self.tool_name, + "owner": yuser, + "type": "unrestricted", + "description": self.args.tool_desc, + "synopsis": self.args.tool_desc, + "category": "TF Generated Tools", + } + yaml.dump(odict, yamlf, allow_unicode=True) + yamlf.close() + + def makeTool(self): + """write xmls and input samples into place""" + if self.args.parampass == 0: + self.doNoXMLparam() + else: + self.makeXML() + if self.args.script_path: + stname = os.path.join(self.tooloutdir, self.sfile) + if not os.path.exists(stname): + shutil.copyfile(self.sfile, stname) + xreal = "%s.xml" % self.tool_name + xout = os.path.join(self.tooloutdir, xreal) + shutil.copyfile(xreal, xout) + for p in self.infiles: + pth = p["name"] + dest = os.path.join(self.testdir, "%s_sample" % p["infilename"]) + shutil.copyfile(pth, dest) + dest = os.path.join(self.repdir, "%s_sample.%s" % (p["infilename"],p["format"])) + shutil.copyfile(pth, dest) + + def makeToolTar(self, report_fail=False): + """move outputs into test-data and prepare the tarball""" + excludeme = "_planemo_test_report.html" + + def exclude_function(tarinfo): + filename = tarinfo.name + return None if filename.endswith(excludeme) else tarinfo + + for p in self.outfiles: + oname = p["name"] + tdest = os.path.join(self.testdir, "%s_sample" % oname) + src = os.path.join(self.testdir, oname) + if not os.path.isfile(tdest): + if os.path.isfile(src): + shutil.copyfile(src, tdest) + dest = os.path.join(self.repdir, "%s.sample" % (oname)) + shutil.copyfile(src, dest) + else: + if report_fail: + print( + "###Tool may have failed - output file %s not found in testdir after planemo run %s." + % (tdest, self.testdir) + ) + tf = tarfile.open(self.newtarpath, "w:gz") + tf.add( + name=self.tooloutdir, + arcname=self.tool_name, + filter=exclude_function, + ) + tf.close() + shutil.copyfile(self.newtarpath, self.args.new_tool) + + def moveRunOutputs(self): + """need to move planemo or run outputs into toolfactory collection""" + with os.scandir(self.tooloutdir) as outs: + for entry in outs: + if not entry.is_file(): + continue + if not entry.name.endswith('.html'): + _, ext = os.path.splitext(entry.name) + newname = f"{entry.name.replace('.','_')}.txt" + dest = os.path.join(self.repdir, newname) + src = os.path.join(self.tooloutdir, entry.name) + shutil.copyfile(src, dest) + if self.args.include_tests: + with os.scandir(self.testdir) as outs: + for entry in outs: + if (not entry.is_file()) or entry.name.endswith( + "_planemo_test_report.html" + ): + continue + if "." in entry.name: + _, ext = os.path.splitext(entry.name) + if ext in [".tgz", ".json"]: + continue + if ext in [".yml", ".xml", ".yaml"]: + newname = f"{entry.name.replace('.','_')}.txt" + else: + newname = entry.name + else: + newname = f"{entry.name}.txt" + dest = os.path.join(self.repdir, newname) + src = os.path.join(self.testdir, entry.name) + shutil.copyfile(src, dest) + + +def main(): + """ + This is a Galaxy wrapper. + It expects to be called by a special purpose tool.xml + + """ + parser = argparse.ArgumentParser() + a = parser.add_argument + a("--script_path", default=None) + a("--history_test", default=None) + a("--cl_user_suffix", default=None) + a("--sysexe", default=None) + a("--packages", default=None) + a("--tool_name", default="newtool") + a("--tool_dir", default=None) + a("--input_files", default=[], action="append") + a("--output_files", default=[], action="append") + a("--user_email", default="Unknown") + a("--bad_user", default=None) + a("--help_text", default=None) + a("--tool_desc", default=None) + a("--tool_version", default=None) + a("--citations", default=None) + a("--command_override", default=None) + a("--test_override", default=None) + a("--additional_parameters", action="append", default=[]) + a("--selecttext_parameters", action="append", default=[]) + a("--edit_additional_parameters", action="store_true", default=False) + a("--parampass", default="positional") + a("--tfout", default="./tfout") + a("--new_tool", default="new_tool") + a("--galaxy_root", default="/galaxy-central") + a("--galaxy_venv", default="/galaxy_venv") + a("--collection", action="append", default=[]) + a("--include_tests", default=False, action="store_true") + a("--admin_only", default=False, action="store_true") + a("--install", default=False, action="store_true") + a("--run_test", default=False, action="store_true") + a("--local_tools", default='tools') # relative to $__root_dir__ + a("--tool_conf_path", default='config/tool_conf.xml') # relative to $__root_dir__ + a("--galaxy_url", default="http://localhost:8080") + a("--toolshed_url", default="http://localhost:9009") + # make sure this is identical to tool_sheds_conf.xml + # localhost != 127.0.0.1 so validation fails + a("--toolshed_api_key", default="fakekey") + a("--galaxy_api_key", default="8993d65865e6d6d1773c2c34a1cc207d") + args = parser.parse_args() + if args.admin_only: + assert not args.bad_user, ( + 'UNAUTHORISED: %s is NOT authorized to use this tool until Galaxy \ +admin adds %s to "admin_users" in the galaxy.yml Galaxy configuration file' + % (args.bad_user, args.bad_user) + ) + assert args.tool_name, "## Tool Factory expects a tool name - eg --tool_name=DESeq" + r = Tool_Factory(args) + r.writeShedyml() + r.makeTool() + r.makeToolTar() + if args.install: + #try: + tcu = Tool_Conf_Updater(args=args, tool_dir=args.local_tools, + new_tool_archive_path=r.newtarpath, tool_conf_path=args.tool_conf_path, + new_tool_name=r.tool_name) + #except Exception: + # print("### Unable to install the new tool. Are you sure you have all the required special settings?") + +if __name__ == "__main__": + main() +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/toolfactory/ToolFactory.xml Fri Apr 30 07:06:57 2021 +0000 @@ -0,0 +1,550 @@ +<tool id="rgtf2" name="toolfactory" version="2.00" profile="16.04" > + <description>Scripts into tools v2.0</description> + <macros> + <xml name="singleText"> + <param name="param_value" type="text" value="" label="Enter this parameter's default text value"> + </param> + </xml> + <xml name="singleInt"> + <param name="param_value" type="integer" value="" label="Enter this parameter's default integer value" > + </param> + </xml> + <xml name="singleFloat"> + <param name="param_value" type="float" value="" label="Enter this parameter's default value"> + </param> + </xml> + <xml name="singleBoolean"> + <param name="param_value" type="boolean" value="" label="Enter this parameter's default value" /> + <param name="truevalue" type="text" value="True" label="Command line value to emit when True" /> + <param name="falsevalue" type="boolean" value="True" label="Command line value to emit when False" /> + </xml> + <xml name="selectText"> + <repeat name="selectTexts" title="Add each option to be presented in a text select box" min="2" default="2" + help="Each text added here will also have a value to be emitted on the command line when the text is chosen"> + <param name="select_text" type="text" value="" label="Enter the explanatory text the user will see for this choice" > + </param> + <param name="select_value" type="text" value="" label="Enter the value for the command line when the user selects this option"> + </param> + </repeat> + </xml> + <xml name="tool_metadata"> + <param name="tool_version" label="Tool Version - bump this to warn users trying to redo old analyses" type="text" value="0.01" + help="If you change your script and regenerate the 'same' tool, you should inform Galaxy (and users) by changing (bumping is traditional) this number"/> + <param name="tool_desc" label="Tool Synopsis" type="text" value="" + help="Supply a brief tool description for the Galaxy tool menu entry" /> + <param name="install" label="Attempt to install in the host Galaxy" + help="This will fail unless running in a very unusual configuration such as a specialised Docker container" + type="boolean" checked="False" truevalue="1" falsevalue="0" /> + <param name="run_test" label="Finalise new archive with test outputs. Runs externally. Outputs will appear in history when ready" + help="Archives must be run and updated using planemo to populate all the test outputs. May take a long time depending on dependencies" + type="boolean" checked="True" truevalue="1" falsevalue="0" /> + <param name="help_text" label="Tool form documentation and help text for users" type="text" area="true" + value="**What it Does**" + help="Supply user documentation to appear on the new tool form as reStructured text - http://docutils.sourceforge.net/docs/ref/rst/restructuredtext.html" > + <sanitizer> + <valid initial="string.printable" /> + <mapping initial="none"/> + </sanitizer> + </param> + <repeat name="citations" title="Citation"> + <conditional name="citation_type"> + <param name="type" type="select" display="radio" label="Citation Type"> + <option value="doi">DOI</option> + <option value="bibtex">BibTeX</option> + </param> + <when value="doi"> + <param name="doi" label="DOI" type="text" value="" + help="Supply a DOI (e.g. doi: 10.1111/j.1740-9713.2007.00258.x) to be cited when this tool is used in published research." /> + </when> + <when value="bibtex"> + <param name="bibtex" label="BibTex" type="text" area="true" + help="Supply a BibTex entry that should be cited when this tool is used in published research." value="" > + <sanitizer> + <valid initial="string.printable" /> + <mapping initial="none"/> + </sanitizer> + </param> + </when> + </conditional> + </repeat> + </xml> + <xml name="io"> + <section name="io" title="Input and output files" expanded="true"> + <repeat name="history_inputs" title="Inputs" min="0" + help="Input data files to pass on the command line to the executable. USE SMALL SAMPLES for the new tool's test. Prompts will form a history item selector as input for users of this new tool"> + <param name="input_files" type="data" format="data" label="Select an input file from your history" multiple="false" + help=""/> + <param name="input_formats" type="select" multiple="true" label="Select the datatype(s) that your tool/script accepts as input" + help="If your datatype is not listed here, it has to be added in galaxy's datatypes_conf.xml" value="tabular"> + <options from_parameter="tool.app.datatypes_registry.upload_file_formats"> + <column name="value" index="0"/> + </options> + </param> + <param name="input_label" type="text" value="" label="This will become the user prompt for the form so please make it informative" > + </param> + <param name="input_help" type="text" value="" label="This will become help text on the form." > + </param> + <param name="input_CL" type="text" label="Positional: ordinal integer. Argparse: argument name. STDIN if the executable/script expects it" + help="If you will pass positional parameters, enter the integer ordinal for this parameter. If Argparse style, '--' will be prepended or '-' if single character" value=""> + </param> + <param name="input_repeat" type="boolean" checked="False" truevalue="1" falsevalue="0" label="Allow user to repeat this input as many times as wanted - script/executable must parse these."/> + </repeat> + <repeat name="history_outputs" min="0" title="Outputs" + help="New history items output by the executable to appear in the user history after the tool runs. The name will become a history item for users of the new tool you are making containing one of it's outputs"> + <param name="history_name" type="text" label="Output Name" optional="false" + help="Name used for this output in history. No spaces! Argparse will also use this name as --[name]"> + <sanitizer invalid_char=" "> + <valid initial="string.ascii_letters,string.digits" > + <add value="_" /> + </valid> + </sanitizer> + </param> + <param name="history_format" type="select" multiple="false" label="Select the datatype for this output" + help="If your datatype is not listed here, it has to be added in galaxy's datatypes_conf.xml" value="tabular"> + <options from_parameter="tool.app.datatypes_registry.upload_file_formats"> + <column name="value" index="0"/> + </options> + </param> + <param name="history_CL" type="text" label="Position" + help="Positional: ordinal integer or 'STDOUT' if '>' required. Otherwise ignored if argparse because name is used. If positional parameters, enter the integer ordinal expected for this parameter. If argparse, ignore unless STDOUT needed" value=""/> + <param name="history_test" type="text" label="Output test criteria" + help="Pass/fail decision criterion for this output compared to test generation. Available options are diff:[lines], sim_size:[delta (integer) or delta_frac (float)" value="diff:0"/> + </repeat> + <repeat name="collection_outputs" title="Output Collections" min="0" + help="New history collections to appear in the user history after the tool runs. Hides outputs not needed for downstream tools such as images and intermediate reports." default="0"> + <param name="name" type="text" label="Name" + help="for new collection with all outputs not individually passed to history outputs. No spaces! Script MUST create and fill a directory with the same name and the tool test MUST be overwritten - see the plotter example for hints"> + <sanitizer invalid_char="_"> + <valid initial="string.ascii_letters,string.digits" > + <add value="_" /> + </valid> + </sanitizer> + </param> + <param name="kind" type="select" multiple="false" label="Select the kind of collection for this output" + help=""> + <option value="list" selected="true">List</option> + <option value="paired">Paired</option> + <option value="list:paired">List of paired</option> + </param> + <param name="label" type="text" label="Label for this collection" help="" value=""/> + <param name="discover" type="text" label="Discover datasets expression" help="__name_and_ext__ should catch all filenames with extensions" value="__name_and_ext__"/> + </repeat> + + </section> + </xml> + <xml name="additparam"> + <section name="addparam" title="Arguments" help="Executable or script settings passed on the command line other than I/O files" expanded="true"> + <param name="edit_params" type="select" display="radio" label="Make these settings on the generated tool form user editable?" + help="If no, users will NOT be able to alter any additional parameters. If yes, these will appear on the tool form as text fields with no validation or sanitizing"> + <option value="yes" selected="true">Yes, allow user to edit all additional parameters on the generated tool form</option> + <option value="no">No - use the fixed values for all additional parameters - no user editing</option> + </param> + <repeat name="additional_parameters" title="Command Line Paramters" min="0" + help="Command line settings for the user to pass to the executable. See examples below to see how these can be parsed by scripts in the various languages"> + <param name="param_name" type="text" value="" label="Choose the name for this parameter - MUST not be blank!"> + <sanitizer invalid_char=""> + <valid initial="string.ascii_letters,string.digits"/> + <mapping initial="none"/> + </sanitizer> + </param> + <conditional name="ap_type"> + <param name="param_type" type="select" label="Select the type for this parameter"> + <option value="text" selected="true">Text string</option> + <option value="integer">Integer</option> + <option value="float">Float</option> + <option value="boolean">Boolean</option> + <option value="selecttext">Select text string</option> + </param> + <when value = "text"> + <expand macro="singleText" /> + </when> + <when value = "integer"> + <expand macro="singleInt" /> + </when> + <when value = "float"> + <expand macro="singleFloat" /> + </when> + <when value = "boolean"> + <expand macro="singleBoolean" /> + </when> + <when value = "selecttext"> + <expand macro="selectText" /> + </when> + </conditional> + <param name="param_label" type="text" value="" label="Enter this parameter's label for the form"> + </param> + <param name="param_help" type="text" value="" label="Help for this parameter"> + </param> + <param name="param_CL" type="text" label="Positional ordinal | argparse argument name" + help="Using positional parameters, enter the integer ordinal for this parameter on the command line. Using Argparse style, '--' will be prepended on the CL" value="" /> + <param name="param_CLprefixed" type="text" label="Override the generated default argparse name prefix if not empty - eg ----foo if needed" + help="Some targets like Planemo expect an unadorned action like 'test' before --galaxy_root." value="" /> + <param name="param_repeat" type="boolean" checked="False" truevalue="1" falsevalue="0" label="Allow user to repeat this as many times as needed." /> + </repeat> + </section> + </xml> + </macros> + +<requirements> + <requirement type="package" version="0.4.14">galaxyxml</requirement> + <requirement type="package" version="0.15.0">bioblend</requirement> + <requirement type="package" version="3.2.3">rsync</requirement> +</requirements> + +<command detect_errors="exit_code"><![CDATA[ +#import os +#set dev_env = os.environ.get('GALAXY_DEVELOPMENT_ENVIRONMENT', '0') == '1' +#if not $dev_env and ( $__user_email__ not in $__admin_users__ ): +python3 $__tool_directory__/rgToolFactory2.py --bad_user $__user_email__ + #else: +python3 $__tool_directory__/ToolFactory.py + #if len(str($cl_suffix)) > 0: +--cl_user_suffix "$cl_suffix" + #end if + #if $cover.commover == "yes": + #if len(str($cover.command_override)) > 10: +--command_override "$commandoverride" + #end if + #if len(str($cover.test_override)) > 10: +--test_override "$testoverride" + #end if + #end if + #if $deps.packages > "": + --packages "$deps.packages" + #end if + #if $deps.usescript.choosescript == "yes": +--script_path "$runme" +--sysexe "$deps.usescript.scriptrunner" + #end if +--tool_name "$tool_name" --user_email "$__user_email__" --citations "$citeme" --parampass "$io_param.ppass.parampass" +--tool_desc "$tool_desc" +--tool_version "$tool_version" +--help_text "$helpme" +--new_tool "$new_tool" + #if $install: + --install + #end if + #if $run_test: + --run_test + #end if + #if $io_param.ppass.parampass != '0': + #if str($io_param.ppass.addparam.edit_params) == "yes": +--edit_additional_parameters + #end if + #for $apar in $io_param.ppass.addparam.additional_parameters: + #if $apar.ap_type.param_type=="selecttext": +--selecttext_parameters '{"name":"$apar.param_name", "label":"$apar.param_label", "help":"$apar.param_help", +"type":"$apar.ap_type.param_type","CL":"$apar.param_CL","override":"$apar.param_CLprefixed","value": [ + #for $i,$st in enumerate($apar.ap_type.selectTexts): + "$st.select_value" + #if ($i < (len($apar.ap_type.selectTexts)-1)): + , + #end if + #end for + ], "texts": [ + #for $i,$st in enumerate($apar.ap_type.selectTexts): + "$st.select_text" + #if ($i < (len($apar.ap_type.selectTexts)-1)): + , + #end if + + #end for + ] + }' + #else: +--additional_parameters '{"name": "$apar.param_name", "value": "$apar.ap_type.param_value", "label": "$apar.param_label", "help": "$apar.param_help", +"type": "$apar.ap_type.param_type","CL": "$apar.param_CL","override": "$apar.param_CLprefixed", "repeat": "$apar.param_repeat"}' + #end if + #end for + #end if + #for $intab in $io_param.ppass.io.history_inputs: +--input_files '{"name": "$intab.input_files", "CL": "$intab.input_CL", "format": "$intab.input_formats", "label": "$intab.input_label", "help": "$intab.input_help", "repeat": "$intab.input_repeat"}' + #end for + #for $otab in $io_param.ppass.io.history_outputs: +--output_files '{"name": "$otab.history_name", "format": "$otab.history_format", "CL": "$otab.history_CL", "test": "$otab.history_test"}' + #end for + #for $collect in $io_param.ppass.io.collection_outputs: +--collection '{"name": "$collect.name", "kind": "$collect.kind", "discover": "$collect.discover", "label": "$collect.label"}' + #end for +--galaxy_root "$__root_dir__" +--tool_dir "$__tool_directory__" + #end if +]]></command> + <configfiles> + <configfile name="runme"> +$deps.usescript.dynScript + </configfile> + <configfile name="commandoverride"> +#if $cover.commover == "yes" and len(str($cover.command_override).strip()) > 1: +$cover.command_override +#end if + </configfile> + <configfile name="testoverride"> +#if $cover.commover == "yes" and len(str($cover.test_override).strip()) > 1: +$cover.test_override +#end if + </configfile> + <configfile name="helpme"> +${help_text} + </configfile> + <configfile name="citeme"> + +#for $citation in $citations: + #if $citation.citation_type.type == "bibtex": + **ENTRY**bibtex + ${citation.citation_type.bibtex} + #else + **ENTRY**doi + ${citation.citation_type.doi} + #end if +#end for + + </configfile> + </configfiles> + <inputs> + <param name="tool_name" type="text" value="tool1" label="New tool ID and title for outputs" + help="Toolshed repository name. Choose thoughtfully to avoid namespace clashes with other tool writers. Lower case, digits and underscores only"> + <sanitizer invalid_char=""> + <valid initial="string.ascii_lowercase,string.digits"> + <add value="_"/> + </valid> + </sanitizer> + </param> + + <section name="deps" title="Dependencies, optional script and script interpreter" expanded="true"> + + <param name="packages" type="text" value="" label="Conda dependencies as package name[:version, name:version...]. These will always be available when this tool executes" + help="Use =[ver] or :[ver] for specific version - 'bwa=0.17.0'. Default is latest. Will be used every time the tool is (re)run. Only Conda is currently supported" /> + + <conditional name="usescript"> + <param name="choosescript" type="select" display="radio" label="Supply a script for a dependency (e.g. python/R/bash) or a system executable such as Bash" + help = "For script interpreters like Python or bash, parameters and i/o specified below must match script's expectations - if you pass parameters, the script must deal with them"> + <option value="no">No script required for this tool - just pass parameters on the command line to the first dependency listed above</option> + <option value="yes" selected="true">Yes, a script is ready to be pasted below</option> + </param> + <when value="no"> + <param name="dynScript" type="hidden" value="" /> + <param name="scriptrunner" type="hidden" value="" /> + </when> + <when value="yes"> + <param name="scriptrunner" type="text" value="" label="Interpreter for the script - eg bash or python. Can be one of the dependencies named above or a system executable" + help="Scripts are interpreted by the executable named here. For conda r-base, 'Rscript --vanilla' or for conda planemo, 'planemo test' for example"> + </param> + <param name="dynScript" type="text" area="True" value="" label="Script for executable above to interpret. It can be one of the Conda dependency names " + help="Script must handle all i/o and parameters as specified below using the parameters and passing method chosen below"> + <sanitizer> + <valid initial="string.printable"/> + <mapping initial="none"/> + </sanitizer> + </param> + </when> + </conditional> + </section> + <section name="io_param" title="Data file input, output and settings forming the executable or script command line" expanded="true"> + <conditional name="ppass"> + <param name="parampass" type="select" display="radio" label="Command line parameter passing method to use"> + <option value="argparse" selected="true">Argparse: passed in the form of --clname value</option> + <option value="positional">Positional: Passed in the order of positional ordinals ...foo.bam bar.idx zot.xls</option> + <option value="0">Tool reads selected input file from STDIN and writes STDOUT with no parameters</option> + </param> + <when value="argparse"> + <expand macro="io" /> + <expand macro="additparam" /> + </when> + <when value="positional"> + <expand macro="io" /> + <expand macro="additparam" /> + </when> + <when value="0"> + <expand macro="io"/> + </when> + </conditional> + </section> + <param name="cl_suffix" type="text" value="" label="Suffix for generated command line. Useful for bash post processing. Use override below to replace completely" + help="';' separated bash commands can be used here for post processing - added at end of autogenerated command line"> + <sanitizer> + <valid initial="string.printable" /> + <mapping initial="none"/> + </sanitizer> + </param> + <conditional name="cover"> + <param name="commover" type="select" display="radio" label="Add Human wrought code to override the generated XML command and/or test section - DIY" + help = "For arbitrary and artfull command lines. All i/o and parameters must be passed. Choose No unless needed. Not for the faint of heart"> + <option value="no" selected="true">No. Use automatically generated command/test XML </option> + <option value="yes">Yes. XML needed to override autogenerated command and/or test segments will be pasted below</option> + </param> + <when value="no"> + <param name="command_override" type="hidden" value="" /> + <param name="test_override" type="hidden" value="" /> + </when> + <when value="yes"> + <param name="command_override" type="text" area="True" value="" label="Optional. Human wrought command element override XML/template - e.g. for bwa" + help="For arbitrary and artfull command lines. All i/o and parameters must be passed. Leave blank unless needed. Not for the faint of heart"> + <sanitizer> + <valid initial="string.printable"/> + <mapping initial="none"/> + </sanitizer> + </param> + <param name="test_override" type="text" area="True" value="" label="Optional. Human wrought test element override XML/template - e.g. for bwa" + help="For arbitrary and artfull scripts. Leave blank unless needed. Not for the faint of heart"> + <sanitizer> + <valid initial="string.printable" /> + <mapping initial="none"/> + </sanitizer> + </param> + </when> + </conditional> + <expand macro="tool_metadata" /> + </inputs> + <outputs> + <data format="toolshed.gz" name="new_tool" label="${tool_name}_not_tested.toolshed.gz" /> + </outputs> +<tests> +<test> + <param name="user_email" value="admin@galaxy.org"/> + <param name="input_files" value="input1_sample" /> + <param name="input_CL" value="1" /> + <param name="input_formats" value="txt" /> + <param name="input_label" value="input" /> + <param name="input_help" value="help" /> + <param name="tool_name" value="pyrevpos" /> + <param name="parampass" value="positional" /> + <param name="tool_version" value="0.01" /> + <param name="tool_desc" value="positional reverse" /> + <param name="help_text" value="help text goes here" /> + <param name="packages" value="python"/> + <param name="history_name" value="output2" /> + <param name="history_format" value="txt" /> + <param name="history_CL" value="2" /> + <param name="dynScript" value="import sys; inp = sys.argv[1]; outp = sys.argv[2]; inlist = open(inp,'r').readlines(); o = open(outp,'w'); rs = [''.join(list(reversed(x.rstrip()))) for x in inlist]; o.write('\n'.join(rs)); o.close()"/> + <param name="choosescript" value="yes" /> + <param name="script_path" value="$runme"/> + <param name="install" value="0"/> + <output name="new_tool" file="toolfactory_pyrevpos_tgz_sample" compare="sim_size" delta="6000" /> +</test> +</tests> +<help> + +.. class:: warningmark + +**Details and attribution** +(see GTF_) + +**Local Admins ONLY** +Only users whose IDs found in the local admin_user configuration setting in universe_wsgi.ini can run this tool. + +**If you find a bug** +Please raise an issue, or even better, submit a pull request fixing it, on the github repository GTF_ + +**What it does** +This tool optionally generates normal workflow compatible first class Galaxy tools + +Generated tools can run existing binary packages that become requirements, existing scripts, or new scripts pasted into this tool form. +Pasted scripts are written so they are part of the new tool and cannot be adjusted by the downstream user. +Binary packages are managed by the dependency subsystem - conda usually, so anything in bioconda or conda_forge is available for example. + +Any number of parameters can be built into the new tool form for passing in to the script or executable at runtime. +These can be editable by the downstream user or baked in. + +A new tarball compatible with any Galaxy toolshed is created in your history, however, it does not have the test prepared. +There is a separate tool you can run to update the archive after testing with Planemo + +If the configuration in job_conf.xml allows tools to write to [galaxy_root]/tools, the new tool will be installed so you can view and test it locally. + +.. class:: warningmark + +**Note to system administrators** +This tool offers *NO* built in protection against malicious scripts. It should only be installed on private/personnal Galaxy instances. +Admin_users will have the power to do anything they want as the Galaxy user if you install this tool. + +.. class:: warningmark + +**Use on public servers** is STRONGLY discouraged for obvious reasons + +The tools generated by this tool will run just as securely as any other normal installed Galaxy tool but like any other new tools, should always be checked carefully before installation. +We recommend that you follow the good code hygiene practices associated with safe toolshed practices. + +Here's a sample python script that can be cut and pasted into the tool form, suitable for positional parameter passing: + +:: + + # reverse order of text by row + import sys + inp = sys.argv[1] + outp = sys.argv[2] + i = open(inp,'r').readlines() + o = open(outp,'w') + for row in i: + rs = row.rstrip() + rs = list(rs) + rs.reverse() + o.write(''.join(rs)) + o.write('\n') + o.close() + +With argparse style parameters: + +:: + + # reverse order of text by row + import argparse + parser = argparse.ArgumentParser() + a = parser.add_argument + a('--infile',default='') + a('--outfile',default=None) + args = parser.parse_args() + inp = args.infile + outp = args.outfile + i = open(inp,'r').readlines() + o = open(outp,'w') + for row in i: + rs = row.rstrip() + rs = list(rs) + rs.reverse() + o.write(''.join(rs)) + o.write('\n') + o.close() + +R script to draw some plots - use a collection. + +:: + + + \# note this script takes NO input because it generates random data + dir.create('plots') + for (i in 1:10) { + foo = runif(100) + bar = rnorm(100) + bar = foo + 0.05*bar + pdf(paste('plots/yet',i,"anotherplot.pdf",sep='_')) + plot(foo,bar,main=paste("Foo by Bar plot \#",i),col="maroon", pch=3,cex=0.6) + dev.off() + foo = data.frame(a=runif(100),b=runif(100),c=runif(100),d=runif(100),e=runif(100),f=runif(100)) + bar = as.matrix(foo) + pdf(paste('plots/yet',i,"anotherheatmap.pdf",sep='_')) + heatmap(bar,main='Random Heatmap') + dev.off() + } + + + +Paper_ + +*Licensing* + +Copyright Ross Lazarus (ross period lazarus at gmail period com) May 2012 +All rights reserved. +Licensed under the LGPL_ + +.. _LGPL: http://www.gnu.org/copyleft/lesser.html +.. _GTF: https://github.com/fubar2/toolfactory +.. _Paper: https://academic.oup.com/bioinformatics/article/28/23/3139/192853 + + +</help> +<citations> + <citation type="doi">10.1093/bioinformatics/bts573</citation> +</citations> +</tool> + +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/toolfactory/ToolFactory_tester.xml Fri Apr 30 07:06:57 2021 +0000 @@ -0,0 +1,454 @@ +<tool name="toolfactory_tester" id="toolfactory_tester" version="1"> + <!--Source in git at: https://github.com/fubar2/toolfactory--> + <!--Created by admin@galaxy.org at 23/04/2021 10:25:58 using the Galaxy Tool Factory.--> + <description>Test an untested tool and update it</description> + <requirements> + <requirement type="package" version="2.30.2">git</requirement> + <requirement type="package" version="0.74.3">planemo</requirement> </requirements> + <stdio> + <exit_code range="1:" level="fatal"/> + </stdio> + <version_command><![CDATA[echo "1"]]></version_command> + <command><![CDATA[ +mkdir 'TF_run_report'; +python +$runme +--in_tool_archive +$in_tool_archive +--new_tested_tool_archive +$new_tested_tool_archive +--galaxy_root +"$__root_dir__" +> +"TF_run_report/${in_tool_archive.name}_test_log.txt" +]]></command> + <configfiles> + <configfile name="runme"><![CDATA[#raw + +# see https://github.com/fubar2/toolfactory +# +# copyright ross lazarus (ross stop lazarus at gmail stop com) May 2012 +# +# all rights reserved +# Licensed under the LGPL +# suggestions for improvement and bug fixes welcome at +# https://github.com/fubar2/toolfactory +# +# July 2020: BCC was fun and I feel like rip van winkle after 5 years. +# Decided to +# 1. Fix the toolfactory so it works - done for simplest case +# 2. Fix planemo so the toolfactory function works +# 3. Rewrite bits using galaxyxml functions where that makes sense - done + +import argparse +import copy +import os +import subprocess +import shutil +import sys +import tarfile +import tempfile +import time +import xml.etree.ElementTree as ET + + +myversion = "V2.2 April 2021" +verbose = True +debug = True +toolFactoryURL = "https://github.com/fubar2/toolfactory" + +def timenow(): + """return current time as a string""" + return time.strftime("%d/%m/%Y %H:%M:%S", time.localtime(time.time())) + +class ToolTester(): + # uncompress passed tar, run planemo and rebuild a new tarball with tests + + def __init__(self, args=None, in_tool_archive='/galaxy-central/tools/newtool/newtool_toolshed.gz', new_tool_archive=None): + self.args = args + self.new_tool_archive = new_tool_archive + assert tarfile.is_tarfile(in_tool_archive) + # this is not going to go well with arbitrary names. + tff = tarfile.open(in_tool_archive, "r:*") + flist = tff.getnames() + ourdir = os.path.commonpath(flist) # eg pyrevpos + self.tool_name = ourdir + ourxmls = [x for x in flist if x.lower().endswith('.xml') and os.path.split(x)[0] == ourdir] + # planemo_test/planemo_test.xml + assert len(ourxmls) > 0 + self.ourxmls = ourxmls # [os.path.join(tool_path,x) for x in ourxmls] + res = tff.extractall() + self.update_tests(ourdir) + tff.close() + self.tooloutdir = "./tfout" + self.repdir = "./TF_run_report" + self.testdir = os.path.join(self.tooloutdir, "test-data") + if not os.path.exists(self.tooloutdir): + os.mkdir(self.tooloutdir) + if not os.path.exists(self.testdir): + os.mkdir(self.testdir) + if not os.path.exists(self.repdir): + os.mkdir(self.repdir) + self.makeTool() + self.moveRunOutputs() + self.makeToolTar() + + def call_planemo(self,xmlpath,ourdir): + penv = os.environ + penv['HOME'] = os.path.join(self.args.galaxy_root,'planemo') + newpath = f"{penv['HOME']}:{penv['PATH']}" + penv['PATH'] = newpath + penv["GALAXY_VIRTUAL_ENV"] = os.path.join(self.args.galaxy_root,'.venv') + penv["PIP_CACHE_DIR"] = os.path.join(self.args.galaxy_root,'pipcache') + toolfile = os.path.split(xmlpath)[1] + tool_name = self.tool_name + tool_test_output = f"{tool_name}_planemo_test_report.html" + cll = [ + "planemo", + "test", + "--no_cleanup", + "--test_data", + os.path.abspath(ourdir), + "--test_output", + os.path.abspath(self.tool_test_output), + "--galaxy_root", + self.args.galaxy_root, + "--update_test_data", + os.path.abspath(xmlpath), + ] + print(cll) + p = subprocess.run( + cll, + #capture_output=True, + encoding='utf8', + cwd = os.path.abspath(self.tool_name), + env = penv, + shell=False, + ) + return p + + def makeTool(self): + """write xmls and input samples into place""" + for xreal in self.ourxmls: + x = os.path.split(xreal)[1] + xout = os.path.join(self.tooloutdir,x) + shutil.copyfile(xreal, xout) + + def makeToolTar(self): + """move outputs into test-data and prepare the tarball""" + excludeme = "_planemo_test_report.html" + + def exclude_function(tarinfo): + filename = tarinfo.name + return None if filename.endswith(excludeme) else tarinfo + + newtar = 'new_%s_toolshed.gz' % self.tool_name + ttf = tarfile.open(newtar, "w:gz") + ttf.add(name=self.tool_name, + arcname=self.tool_name, + filter=exclude_function) + ttf.close() + shutil.copyfile(newtar, self.new_tool_archive) + + def moveRunOutputs(self): + """need to move planemo or run outputs into toolfactory collection""" + with os.scandir(self.tooloutdir) as outs: + for entry in outs: + if not entry.is_file(): + continue + if "." in entry.name: + _, ext = os.path.splitext(entry.name) + if ext in [".tgz", ".json"]: + continue + if ext in [".yml", ".xml", ".yaml"]: + newname = f"{entry.name.replace('.','_')}.txt" + else: + newname = entry.name + else: + newname = f"{entry.name}.txt" + dest = os.path.join(self.repdir, newname) + src = os.path.join(self.tooloutdir, entry.name) + shutil.copyfile(src, dest) + with os.scandir('.') as outs: + for entry in outs: + if not entry.is_file() or entry.name == "conda_activate.log": + continue + if "." in entry.name: + _, ext = os.path.splitext(entry.name) + if ext in [".yml", ".xml", ".yaml"]: + newname = f"{entry.name.replace('.','_')}.txt" + else: + newname = entry.name + else: + newname = f"{entry.name}.txt" + dest = os.path.join(self.repdir, newname) + src =entry.name + shutil.copyfile(src, dest) + if True or self.args.include_tests: + with os.scandir(self.testdir) as outs: + for entry in outs: + if (not entry.is_file()) or entry.name.endswith( + "_planemo_test_report.html" + ): + continue + if "." in entry.name: + _, ext = os.path.splitext(entry.name) + if ext in [".tgz", ".json"]: + continue + if ext in [".yml", ".xml", ".yaml"]: + newname = f"{entry.name.replace('.','_')}.txt" + else: + newname = entry.name + else: + newname = f"{entry.name}.txt" + dest = os.path.join(self.repdir, newname) + src = os.path.join(self.testdir, entry.name) + shutil.copyfile(src, dest) + + + def update_tests(self,ourdir): + for xmlf in self.ourxmls: + capture = self.call_planemo(xmlf,ourdir) + +def main(): + """ + This is a Galaxy wrapper. + It expects to be called by a special purpose tool.xml + + """ + parser = argparse.ArgumentParser() + a = parser.add_argument + a("--in_tool_archive", default=None) + a("--new_tested_tool_archive", default=None) + a("--galaxy_root", default="/home/ross/gal21/") + args = parser.parse_args() + print('Hello from',os.getcwd()) + tt = ToolTester(args=args, in_tool_archive=args.in_tool_archive, new_tool_archive=args.new_tested_tool_archive) + +if __name__ == "__main__": + main() + + +#end raw]]></configfile> + </configfiles> + <inputs> + <param name="new_tool_name" value="" type="hidden"/> + <param name="in_tool_archive" type="data" optional="false" label="Select a no_test tarfile to test and update for a toolshed" help="" format="toolshed.gz" multiple="false"/> + </inputs> + <outputs> + <data name="new_tested_tool_archive" format="toolshed.gz" label="${in_tool_archive.name.split('_')[0]}_tested_toolshed.gz" hidden="false"/> + <collection name="TF_run_report" type="list" label="${in_tool_archive.name} test Run reports"> + <discover_datasets pattern="__name_and_ext__" directory="TF_run_report" visible="false"/> + </collection> + </outputs> + <tests> + <test> + <output name="new_tested_tool_archive" value="new_tested_tool_archive_sample" compare="sim_size" delta_frac="0.5"/> + <param name="in_tool_archive" value="in_tool_archive_sample"/> + <output_collection name="TF_run_report"/> + </test> + </tests> + <help><![CDATA[ + +**What it Does** + +------ + +Script:: + + # see https://github.com/fubar2/toolfactory + # + # copyright ross lazarus (ross stop lazarus at gmail stop com) May 2012 + # + # all rights reserved + # Licensed under the LGPL + # suggestions for improvement and bug fixes welcome at + # https://github.com/fubar2/toolfactory + + import argparse + import copy + import os + import subprocess + import shutil + import sys + import tarfile + import tempfile + import time + import xml.etree.ElementTree as ET + + + myversion = "V2.2 April 2021" + verbose = True + debug = True + toolFactoryURL = "https://github.com/fubar2/toolfactory" + + def timenow(): + """return current time as a string""" + return time.strftime("%d/%m/%Y %H:%M:%S", time.localtime(time.time())) + + class ToolTester(): + # uncompress passed tar, run planemo and rebuild a new tarball with tests + + def __init__(self, args=None, in_tool_archive='/galaxy-central/tools/newtool/newtool_toolshed.gz', new_tool_archive=None): + self.args = args + self.new_tool_archive = new_tool_archive + assert tarfile.is_tarfile(in_tool_archive) + # this is not going to go well with arbitrary names. + tff = tarfile.open(in_tool_archive, "r:*") + flist = tff.getnames() + ourdir = os.path.commonpath(flist) # eg pyrevpos + self.tool_name = ourdir + ourxmls = [x for x in flist if x.lower().endswith('.xml') and os.path.split(x)[0] == ourdir] + assert len(ourxmls) > 0 + self.ourxmls = ourxmls # [os.path.join(tool_path,x) for x in ourxmls] + res = tff.extractall() + self.update_tests(ourdir) + tff.close() + self.tooloutdir = "./tfout" + self.repdir = "./TF_run_report" + self.testdir = os.path.join(self.tooloutdir, "test-data") + if not os.path.exists(self.tooloutdir): + os.mkdir(self.tooloutdir) + if not os.path.exists(self.testdir): + os.mkdir(self.testdir) + if not os.path.exists(self.repdir): + os.mkdir(self.repdir) + self.makeTool() + self.moveRunOutputs() + self.makeToolTar() + + def call_planemo(self,xmlpath,ourdir): + penv = os.environ + penv["PIP_CACHE_DIR"] = os.path.join(self.args.galaxy_root,'pipcache') + toolfile = os.path.split(xmlpath)[1] + tool_name = self.tool_name + tool_test_output = f"{tool_name}_planemo_test_report.html" + cll = [ + "planemo", + "test", + "--biocontainers", + "--test_output", + os.path.abspath(tool_test_output), + "--galaxy_root", + self.args.galaxy_root, + "--update_test_data", + os.path.abspath(xmlpath), + ] + print(cll) + p = subprocess.run( + cll, + #capture_output=True, + encoding='utf8', + env = penv, + shell=False, + ) + return p + + def makeTool(self): + """write xmls and input samples into place""" + for xreal in self.ourxmls: + x = os.path.split(xreal)[1] + xout = os.path.join(self.tooloutdir,x) + shutil.copyfile(xreal, xout) + + def makeToolTar(self): + """move outputs into test-data and prepare the tarball""" + excludeme = "_planemo_test_report.html" + + def exclude_function(tarinfo): + filename = tarinfo.name + return None if filename.endswith(excludeme) else tarinfo + + newtar = 'new_%s_toolshed.gz' % self.tool_name + ttf = tarfile.open(newtar, "w:gz") + ttf.add(name=self.tool_name, + arcname=self.tool_name, + filter=exclude_function) + ttf.close() + shutil.copyfile(newtar, self.new_tool_archive) + + def moveRunOutputs(self): + """need to move planemo or run outputs into toolfactory collection""" + with os.scandir(self.tooloutdir) as outs: + for entry in outs: + if not entry.is_file(): + continue + if "." in entry.name: + _, ext = os.path.splitext(entry.name) + if ext in [".tgz", ".json"]: + continue + if ext in [".yml", ".xml", ".yaml"]: + newname = f"{entry.name.replace('.','_')}.txt" + else: + newname = entry.name + else: + newname = f"{entry.name}.txt" + dest = os.path.join(self.repdir, newname) + src = os.path.join(self.tooloutdir, entry.name) + shutil.copyfile(src, dest) + with os.scandir('.') as outs: + for entry in outs: + if not entry.is_file(): + continue + if "." in entry.name: + _, ext = os.path.splitext(entry.name) + if ext in [".yml", ".xml", ".yaml"]: + newname = f"{entry.name.replace('.','_')}.txt" + else: + newname = entry.name + else: + newname = f"{entry.name}.txt" + dest = os.path.join(self.repdir, newname) + src =entry.name + shutil.copyfile(src, dest) + if True or self.args.include_tests: + with os.scandir(self.testdir) as outs: + for entry in outs: + if (not entry.is_file()) or entry.name.endswith( + "_planemo_test_report.html" + ): + continue + if "." in entry.name: + _, ext = os.path.splitext(entry.name) + if ext in [".tgz", ".json"]: + continue + if ext in [".yml", ".xml", ".yaml"]: + newname = f"{entry.name.replace('.','_')}.txt" + else: + newname = entry.name + else: + newname = f"{entry.name}.txt" + dest = os.path.join(self.repdir, newname) + src = os.path.join(self.testdir, entry.name) + shutil.copyfile(src, dest) + + + def update_tests(self,ourdir): + for xmlf in self.ourxmls: + capture = self.call_planemo(xmlf,ourdir) + + def main(): + """ + This is a Galaxy wrapper. + It expects to be called by a special purpose tool.xml + + """ + parser = argparse.ArgumentParser() + a = parser.add_argument + a("--in_tool_archive", default=None) + a("--new_tested_tool_archive", default=None) + a("--galaxy_root", default="/home/ross/gal21/") + args = parser.parse_args() + print('Hello from',os.getcwd()) + tt = ToolTester(args=args, in_tool_archive=args.in_tool_archive, new_tool_archive=args.new_tested_tool_archive) + + if __name__ == "__main__": + main() + + +]]></help> + <citations> + <citation type="doi">10.1093/bioinformatics/bts573</citation> + </citations> +</tool> +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/toolfactory/install-history.py Fri Apr 30 07:06:57 2021 +0000 @@ -0,0 +1,41 @@ +import argparse +import os + + +from bioblend import galaxy + + +def _parser(): + parser = argparse.ArgumentParser() + parser.add_argument("-g", "--galaxy", help='URL of target galaxy') + parser.add_argument("-p", "--password", help='Galaxy admin password') + parser.add_argument("-e", "--email", help='Galaxy admin email') + parser.add_argument("-a", "--key", help='Galaxy admin key', default=None) + parser.add_argument("-i", "--history_path", help='Path to history gz files to be loaded') + return parser + +def main(): + """ + load a folder of histories or a single gz + """ + args = _parser().parse_args() + if args.key: + gi = galaxy.GalaxyInstance(url=args.galaxy, key=args.key) + else: + gi = galaxy.GalaxyInstance(url=args.galaxy, email=args.email, password=args.password) + hdir = args.history_path + # h = gi.histories.get_most_recently_used_history() + if os.path.isdir(hdir): + for fp in os.listdir(hdir): + hp = os.path.join(hdir,fp) + if os.path.isfile(hp): + x = gi.histories.import_history(file_path=hp, url=None) + print('installed ',hp,'res=',x) + else: + x = gi.histories.import_history(file_path=hdir, url=None) + print('installed',hdir,'res=',x) + + +if __name__ == "__main__": + main() +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/toolfactory/install_tf_demos.py Fri Apr 30 07:06:57 2021 +0000 @@ -0,0 +1,42 @@ +import argparse +import urllib.request + +from bioblend import galaxy + +WF = "https://drive.google.com/uc?export=download&id=13xE8o7tucHGNA0qYkEP98FfUGl2wdOU5" +HIST = ( + "https://zenodo.org/record/4686436/files/TFdemo_wf_april13_planemo.ga?download=1" +) +WF_FILE = "tf_workflow.ga" +HIST_FILE = "tf_history.tgz" + + +def _parser(): + parser = argparse.ArgumentParser() + parser.add_argument( + "-g", "--galaxy", help="URL of target galaxy", default="http://localhost:8080" + ) + parser.add_argument("-a", "--key", help="Galaxy admin key", default="8993d65865e6d6d1773c2c34a1cc207d") + return parser + + +def main(): + """ + load the planemo tool_factory demonstration history and tool generating workflow + fails in planemo served galaxies because there seems to be no user in trans? + """ + args = _parser().parse_args() + urllib.request.urlretrieve(WF, WF_FILE) + urllib.request.urlretrieve(HIST, HIST_FILE) + assert args.key, "Need an administrative key for the target Galaxy supplied please" + gi = galaxy.GalaxyInstance( + url=args.galaxy, key=args.key, email="planemo@galaxyproject.org" + ) + x = gi.workflows.import_workflow_from_local_path(WF_FILE, publish=True) + print(f"installed {WF_FILE} Returned = {x}\n") + x = gi.histories.import_history(file_path=HIST_FILE) + print(f"installed {HIST_FILE} Returned = {x}\n") + + +if __name__ == "__main__": + main()
--- a/toolfactory/rgToolFactory2.py Sun Apr 18 03:55:56 2021 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,1150 +0,0 @@ -# replace with shebang for biocontainer -# see https://github.com/fubar2/toolfactory -# -# copyright ross lazarus (ross stop lazarus at gmail stop com) May 2012 -# -# all rights reserved -# Licensed under the LGPL -# suggestions for improvement and bug fixes welcome at -# https://github.com/fubar2/toolfactory -# -# July 2020: BCC was fun and I feel like rip van winkle after 5 years. -# Decided to -# 1. Fix the toolfactory so it works - done for simplest case -# 2. Fix planemo so the toolfactory function works -# 3. Rewrite bits using galaxyxml functions where that makes sense - done - -import argparse -import copy -import json -import logging -import os -import re -import shlex -import shutil -import subprocess -import sys -import tarfile -import tempfile -import time - -from bioblend import ConnectionError -from bioblend import toolshed - -import galaxyxml.tool as gxt -import galaxyxml.tool.parameters as gxtp - -import lxml - -import yaml - -myversion = "V2.2 February 2021" -verbose = True -debug = True -toolFactoryURL = "https://github.com/fubar2/toolfactory" -foo = len(lxml.__version__) -FAKEEXE = "~~~REMOVE~~~ME~~~" -# need this until a PR/version bump to fix galaxyxml prepending the exe even -# with override. - - -def timenow(): - """return current time as a string""" - return time.strftime("%d/%m/%Y %H:%M:%S", time.localtime(time.time())) - - -cheetah_escape_table = {"$": "\\$", "#": "\\#"} - - -def cheetah_escape(text): - """Produce entities within text.""" - return "".join([cheetah_escape_table.get(c, c) for c in text]) - - -def parse_citations(citations_text): - """""" - citations = [c for c in citations_text.split("**ENTRY**") if c.strip()] - citation_tuples = [] - for citation in citations: - if citation.startswith("doi"): - citation_tuples.append(("doi", citation[len("doi") :].strip())) - else: - citation_tuples.append(("bibtex", citation[len("bibtex") :].strip())) - return citation_tuples - - -class ScriptRunner: - """Wrapper for an arbitrary script - uses galaxyxml - - """ - - def __init__(self, args=None): # noqa - """ - prepare command line cl for running the tool here - and prepare elements needed for galaxyxml tool generation - """ - self.ourcwd = os.getcwd() - self.collections = [] - if len(args.collection) > 0: - try: - self.collections = [ - json.loads(x) for x in args.collection if len(x.strip()) > 1 - ] - except Exception: - print( - f"--collections parameter {str(args.collection)} is malformed - should be a dictionary" - ) - try: - self.infiles = [ - json.loads(x) for x in args.input_files if len(x.strip()) > 1 - ] - except Exception: - print( - f"--input_files parameter {str(args.input_files)} is malformed - should be a dictionary" - ) - try: - self.outfiles = [ - json.loads(x) for x in args.output_files if len(x.strip()) > 1 - ] - except Exception: - print( - f"--output_files parameter {args.output_files} is malformed - should be a dictionary" - ) - try: - self.addpar = [ - json.loads(x) for x in args.additional_parameters if len(x.strip()) > 1 - ] - except Exception: - print( - f"--additional_parameters {args.additional_parameters} is malformed - should be a dictionary" - ) - try: - self.selpar = [ - json.loads(x) for x in args.selecttext_parameters if len(x.strip()) > 1 - ] - except Exception: - print( - f"--selecttext_parameters {args.selecttext_parameters} is malformed - should be a dictionary" - ) - self.args = args - self.cleanuppar() - self.lastclredirect = None - self.lastxclredirect = None - self.cl = [] - self.xmlcl = [] - self.is_positional = self.args.parampass == "positional" - if self.args.sysexe: - if ' ' in self.args.sysexe: - self.executeme = self.args.sysexe.split(' ') - else: - self.executeme = [self.args.sysexe, ] - else: - if self.args.packages: - self.executeme = [self.args.packages.split(",")[0].split(":")[0].strip(), ] - else: - self.executeme = None - aCL = self.cl.append - aXCL = self.xmlcl.append - assert args.parampass in [ - "0", - "argparse", - "positional", - ], 'args.parampass must be "0","positional" or "argparse"' - self.tool_name = re.sub("[^a-zA-Z0-9_]+", "", args.tool_name) - self.tool_id = self.tool_name - self.newtool = gxt.Tool( - self.tool_name, - self.tool_id, - self.args.tool_version, - self.args.tool_desc, - FAKEEXE, - ) - self.newtarpath = "%s_toolshed.gz" % self.tool_name - self.tooloutdir = "./tfout" - self.repdir = "./TF_run_report_tempdir" - self.testdir = os.path.join(self.tooloutdir, "test-data") - if not os.path.exists(self.tooloutdir): - os.mkdir(self.tooloutdir) - if not os.path.exists(self.testdir): - os.mkdir(self.testdir) - if not os.path.exists(self.repdir): - os.mkdir(self.repdir) - self.tinputs = gxtp.Inputs() - self.toutputs = gxtp.Outputs() - self.testparam = [] - if self.args.script_path: - self.prepScript() - if self.args.command_override: - scos = open(self.args.command_override, "r").readlines() - self.command_override = [x.rstrip() for x in scos] - else: - self.command_override = None - if self.args.test_override: - stos = open(self.args.test_override, "r").readlines() - self.test_override = [x.rstrip() for x in stos] - else: - self.test_override = None - if self.args.script_path: - for ex in self.executeme: - aCL(ex) - aXCL(ex) - aCL(self.sfile) - aXCL("$runme") - else: - for ex in self.executeme: - aCL(ex) - aXCL(ex) - - self.elog = os.path.join(self.repdir, "%s_error_log.txt" % self.tool_name) - self.tlog = os.path.join(self.repdir, "%s_runner_log.txt" % self.tool_name) - if self.args.parampass == "0": - self.clsimple() - else: - if self.args.parampass == "positional": - self.prepclpos() - self.clpositional() - else: - self.prepargp() - self.clargparse() - if self.args.cl_suffix: # DIY CL end - clp = shlex.split(self.args.cl_suffix) - for c in clp: - aCL(c) - aXCL(c) - - def clsimple(self): - """no parameters or repeats - uses < and > for i/o""" - aCL = self.cl.append - aXCL = self.xmlcl.append - if len(self.infiles) > 0: - aCL("<") - aCL(self.infiles[0]["infilename"]) - aXCL("<") - aXCL("$%s" % self.infiles[0]["infilename"]) - if len(self.outfiles) > 0: - aCL(">") - aCL(self.outfiles[0]["name"]) - aXCL(">") - aXCL("$%s" % self.outfiles[0]["name"]) - - def prepargp(self): - clsuffix = [] - xclsuffix = [] - for i, p in enumerate(self.infiles): - nam = p["infilename"] - if p["origCL"].strip().upper() == "STDIN": - appendme = [ - nam, - nam, - "< %s" % nam, - ] - xappendme = [ - nam, - nam, - "< $%s" % nam, - ] - else: - rep = p["repeat"] == "1" - over = "" - if rep: - over = f'#for $rep in $R_{nam}:\n--{nam} "$rep.{nam}"\n#end for' - appendme = [p["CL"], p["CL"], ""] - xappendme = [p["CL"], "$%s" % p["CL"], over] - clsuffix.append(appendme) - xclsuffix.append(xappendme) - for i, p in enumerate(self.outfiles): - if p["origCL"].strip().upper() == "STDOUT": - self.lastclredirect = [">", p["name"]] - self.lastxclredirect = [">", "$%s" % p["name"]] - else: - clsuffix.append([p["name"], p["name"], ""]) - xclsuffix.append([p["name"], "$%s" % p["name"], ""]) - for p in self.addpar: - nam = p["name"] - rep = p["repeat"] == "1" - if rep: - over = f'#for $rep in $R_{nam}:\n--{nam} "$rep.{nam}"\n#end for' - else: - over = p["override"] - clsuffix.append([p["CL"], nam, over]) - xclsuffix.append([p["CL"], nam, over]) - for p in self.selpar: - clsuffix.append([p["CL"], p["name"], p["override"]]) - xclsuffix.append([p["CL"], '"$%s"' % p["name"], p["override"]]) - self.xclsuffix = xclsuffix - self.clsuffix = clsuffix - - def prepclpos(self): - clsuffix = [] - xclsuffix = [] - for i, p in enumerate(self.infiles): - if p["origCL"].strip().upper() == "STDIN": - appendme = [ - "999", - p["infilename"], - "< $%s" % p["infilename"], - ] - xappendme = [ - "999", - p["infilename"], - "< $%s" % p["infilename"], - ] - else: - appendme = [p["CL"], p["infilename"], ""] - xappendme = [p["CL"], "$%s" % p["infilename"], ""] - clsuffix.append(appendme) - xclsuffix.append(xappendme) - for i, p in enumerate(self.outfiles): - if p["origCL"].strip().upper() == "STDOUT": - self.lastclredirect = [">", p["name"]] - self.lastxclredirect = [">", "$%s" % p["name"]] - else: - clsuffix.append([p["CL"], p["name"], ""]) - xclsuffix.append([p["CL"], "$%s" % p["name"], ""]) - for p in self.addpar: - nam = p["name"] - rep = p["repeat"] == "1" # repeats make NO sense - if rep: - print(f'### warning. Repeats for {nam} ignored - not permitted in positional parameter command lines!') - over = p["override"] - clsuffix.append([p["CL"], nam, over]) - xclsuffix.append([p["CL"], '"$%s"' % nam, over]) - for p in self.selpar: - clsuffix.append([p["CL"], p["name"], p["override"]]) - xclsuffix.append([p["CL"], '"$%s"' % p["name"], p["override"]]) - clsuffix.sort() - xclsuffix.sort() - self.xclsuffix = xclsuffix - self.clsuffix = clsuffix - - def prepScript(self): - rx = open(self.args.script_path, "r").readlines() - rx = [x.rstrip() for x in rx] - rxcheck = [x.strip() for x in rx if x.strip() > ""] - assert len(rxcheck) > 0, "Supplied script is empty. Cannot run" - self.script = "\n".join(rx) - fhandle, self.sfile = tempfile.mkstemp( - prefix=self.tool_name, suffix="_%s" % (self.executeme[0]) - ) - tscript = open(self.sfile, "w") - tscript.write(self.script) - tscript.close() - self.escapedScript = [cheetah_escape(x) for x in rx] - self.spacedScript = [f" {x}" for x in rx if x.strip() > ""] - art = "%s.%s" % (self.tool_name, self.executeme[0]) - artifact = open(art, "wb") - artifact.write(bytes("\n".join(self.escapedScript), "utf8")) - artifact.close() - - def cleanuppar(self): - """ positional parameters are complicated by their numeric ordinal""" - if self.args.parampass == "positional": - for i, p in enumerate(self.infiles): - assert ( - p["CL"].isdigit() or p["CL"].strip().upper() == "STDIN" - ), "Positional parameters must be ordinal integers - got %s for %s" % ( - p["CL"], - p["label"], - ) - for i, p in enumerate(self.outfiles): - assert ( - p["CL"].isdigit() or p["CL"].strip().upper() == "STDOUT" - ), "Positional parameters must be ordinal integers - got %s for %s" % ( - p["CL"], - p["name"], - ) - for i, p in enumerate(self.addpar): - assert p[ - "CL" - ].isdigit(), "Positional parameters must be ordinal integers - got %s for %s" % ( - p["CL"], - p["name"], - ) - for i, p in enumerate(self.infiles): - infp = copy.copy(p) - infp["origCL"] = infp["CL"] - if self.args.parampass in ["positional", "0"]: - infp["infilename"] = infp["label"].replace(" ", "_") - else: - infp["infilename"] = infp["CL"] - self.infiles[i] = infp - for i, p in enumerate(self.outfiles): - p["origCL"] = p["CL"] # keep copy - self.outfiles[i] = p - for i, p in enumerate(self.addpar): - p["origCL"] = p["CL"] - self.addpar[i] = p - - def clpositional(self): - # inputs in order then params - aCL = self.cl.append - for (k, v, koverride) in self.clsuffix: - if " " in v: - aCL("%s" % v) - else: - aCL(v) - aXCL = self.xmlcl.append - for (k, v, koverride) in self.xclsuffix: - aXCL(v) - if self.lastxclredirect: - aXCL(self.lastxclredirect[0]) - aXCL(self.lastxclredirect[1]) - - def clargparse(self): - """argparse style""" - aCL = self.cl.append - aXCL = self.xmlcl.append - # inputs then params in argparse named form - - for (k, v, koverride) in self.xclsuffix: - if koverride > "": - k = koverride - aXCL(k) - else: - if len(k.strip()) == 1: - k = "-%s" % k - else: - k = "--%s" % k - aXCL(k) - aXCL(v) - for (k, v, koverride) in self.clsuffix: - if koverride > "": - k = koverride - elif len(k.strip()) == 1: - k = "-%s" % k - else: - k = "--%s" % k - aCL(k) - aCL(v) - if self.lastxclredirect: - aXCL(self.lastxclredirect[0]) - aXCL(self.lastxclredirect[1]) - - def getNdash(self, newname): - if self.is_positional: - ndash = 0 - else: - ndash = 2 - if len(newname) < 2: - ndash = 1 - return ndash - - def doXMLparam(self): - """Add all needed elements to tool""" # noqa - for p in self.outfiles: - newname = p["name"] - newfmt = p["format"] - newcl = p["CL"] - test = p["test"] - oldcl = p["origCL"] - test = test.strip() - ndash = self.getNdash(newcl) - aparm = gxtp.OutputData( - name=newname, format=newfmt, num_dashes=ndash, label=newname - ) - aparm.positional = self.is_positional - if self.is_positional: - if oldcl.upper() == "STDOUT": - aparm.positional = 9999999 - aparm.command_line_override = "> $%s" % newname - else: - aparm.positional = int(oldcl) - aparm.command_line_override = "$%s" % newname - self.toutputs.append(aparm) - ld = None - if test.strip() > "": - if test.startswith("diff"): - c = "diff" - ld = 0 - if test.split(":")[1].isdigit: - ld = int(test.split(":")[1]) - tp = gxtp.TestOutput( - name=newname, - value="%s_sample" % newname, - compare=c, - lines_diff=ld, - ) - elif test.startswith("sim_size"): - c = "sim_size" - tn = test.split(":")[1].strip() - if tn > "": - if "." in tn: - delta = None - delta_frac = min(1.0, float(tn)) - else: - delta = int(tn) - delta_frac = None - tp = gxtp.TestOutput( - name=newname, - value="%s_sample" % newname, - compare=c, - delta=delta, - delta_frac=delta_frac, - ) - else: - c = test - tp = gxtp.TestOutput( - name=newname, - value="%s_sample" % newname, - compare=c, - ) - self.testparam.append(tp) - for p in self.infiles: - newname = p["infilename"] - newfmt = p["format"] - ndash = self.getNdash(newname) - reps = p.get("repeat", "0") == "1" - if not len(p["label"]) > 0: - alab = p["CL"] - else: - alab = p["label"] - aninput = gxtp.DataParam( - newname, - optional=False, - label=alab, - help=p["help"], - format=newfmt, - multiple=False, - num_dashes=ndash, - ) - aninput.positional = self.is_positional - if self.is_positional: - if p["origCL"].upper() == "STDIN": - aninput.positional = 9999998 - aninput.command_line_override = "> $%s" % newname - else: - aninput.positional = int(p["origCL"]) - aninput.command_line_override = "$%s" % newname - if reps: - repe = gxtp.Repeat(name=f"R_{newname}", title=f"Add as many {alab} as needed") - repe.append(aninput) - self.tinputs.append(repe) - tparm = gxtp.TestRepeat(name=f"R_{newname}") - tparm2 = gxtp.TestParam(newname, value="%s_sample" % newname) - tparm.append(tparm2) - self.testparam.append(tparm) - else: - self.tinputs.append(aninput) - tparm = gxtp.TestParam(newname, value="%s_sample" % newname) - self.testparam.append(tparm) - for p in self.addpar: - newname = p["name"] - newval = p["value"] - newlabel = p["label"] - newhelp = p["help"] - newtype = p["type"] - newcl = p["CL"] - oldcl = p["origCL"] - reps = p["repeat"] == "1" - if not len(newlabel) > 0: - newlabel = newname - ndash = self.getNdash(newname) - if newtype == "text": - aparm = gxtp.TextParam( - newname, - label=newlabel, - help=newhelp, - value=newval, - num_dashes=ndash, - ) - elif newtype == "integer": - aparm = gxtp.IntegerParam( - newname, - label=newlabel, - help=newhelp, - value=newval, - num_dashes=ndash, - ) - elif newtype == "float": - aparm = gxtp.FloatParam( - newname, - label=newlabel, - help=newhelp, - value=newval, - num_dashes=ndash, - ) - elif newtype == "boolean": - aparm = gxtp.BooleanParam( - newname, - label=newlabel, - help=newhelp, - value=newval, - num_dashes=ndash, - ) - else: - raise ValueError( - 'Unrecognised parameter type "%s" for\ - additional parameter %s in makeXML' - % (newtype, newname) - ) - aparm.positional = self.is_positional - if self.is_positional: - aparm.positional = int(oldcl) - if reps: - repe = gxtp.Repeat(name=f"R_{newname}", title=f"Add as many {newlabel} as needed") - repe.append(aparm) - self.tinputs.append(repe) - tparm = gxtp.TestRepeat(name=f"R_{newname}") - tparm2 = gxtp.TestParam(newname, value=newval) - tparm.append(tparm2) - self.testparam.append(tparm) - else: - self.tinputs.append(aparm) - tparm = gxtp.TestParam(newname, value=newval) - self.testparam.append(tparm) - for p in self.selpar: - newname = p["name"] - newval = p["value"] - newlabel = p["label"] - newhelp = p["help"] - newtype = p["type"] - newcl = p["CL"] - if not len(newlabel) > 0: - newlabel = newname - ndash = self.getNdash(newname) - if newtype == "selecttext": - newtext = p["texts"] - aparm = gxtp.SelectParam( - newname, - label=newlabel, - help=newhelp, - num_dashes=ndash, - ) - for i in range(len(newval)): - anopt = gxtp.SelectOption( - value=newval[i], - text=newtext[i], - ) - aparm.append(anopt) - aparm.positional = self.is_positional - if self.is_positional: - aparm.positional = int(newcl) - self.tinputs.append(aparm) - tparm = gxtp.TestParam(newname, value=newval) - self.testparam.append(tparm) - else: - raise ValueError( - 'Unrecognised parameter type "%s" for\ - selecttext parameter %s in makeXML' - % (newtype, newname) - ) - for p in self.collections: - newkind = p["kind"] - newname = p["name"] - newlabel = p["label"] - newdisc = p["discover"] - collect = gxtp.OutputCollection(newname, label=newlabel, type=newkind) - disc = gxtp.DiscoverDatasets( - pattern=newdisc, directory=f"{newname}", visible="false" - ) - collect.append(disc) - self.toutputs.append(collect) - try: - tparm = gxtp.TestOutputCollection(newname) # broken until PR merged. - self.testparam.append(tparm) - except Exception: - print("#### WARNING: Galaxyxml version does not have the PR merged yet - tests for collections must be over-ridden until then!") - - def doNoXMLparam(self): - """filter style package - stdin to stdout""" - if len(self.infiles) > 0: - alab = self.infiles[0]["label"] - if len(alab) == 0: - alab = self.infiles[0]["infilename"] - max1s = ( - "Maximum one input if parampass is 0 but multiple input files supplied - %s" - % str(self.infiles) - ) - assert len(self.infiles) == 1, max1s - newname = self.infiles[0]["infilename"] - aninput = gxtp.DataParam( - newname, - optional=False, - label=alab, - help=self.infiles[0]["help"], - format=self.infiles[0]["format"], - multiple=False, - num_dashes=0, - ) - aninput.command_line_override = "< $%s" % newname - aninput.positional = True - self.tinputs.append(aninput) - tp = gxtp.TestParam(name=newname, value="%s_sample" % newname) - self.testparam.append(tp) - if len(self.outfiles) > 0: - newname = self.outfiles[0]["name"] - newfmt = self.outfiles[0]["format"] - anout = gxtp.OutputData(newname, format=newfmt, num_dashes=0) - anout.command_line_override = "> $%s" % newname - anout.positional = self.is_positional - self.toutputs.append(anout) - tp = gxtp.TestOutput(name=newname, value="%s_sample" % newname) - self.testparam.append(tp) - - def makeXML(self): # noqa - """ - Create a Galaxy xml tool wrapper for the new script - Uses galaxyhtml - Hmmm. How to get the command line into correct order... - """ - if self.command_override: - self.newtool.command_override = self.command_override # config file - else: - self.newtool.command_override = self.xmlcl - cite = gxtp.Citations() - acite = gxtp.Citation(type="doi", value="10.1093/bioinformatics/bts573") - cite.append(acite) - self.newtool.citations = cite - safertext = "" - if self.args.help_text: - helptext = open(self.args.help_text, "r").readlines() - safertext = "\n".join([cheetah_escape(x) for x in helptext]) - if len(safertext.strip()) == 0: - safertext = ( - "Ask the tool author (%s) to rebuild with help text please\n" - % (self.args.user_email) - ) - if self.args.script_path: - if len(safertext) > 0: - safertext = safertext + "\n\n------\n" # transition allowed! - scr = [x for x in self.spacedScript if x.strip() > ""] - scr.insert(0, "\n\nScript::\n") - if len(scr) > 300: - scr = ( - scr[:100] - + [" >300 lines - stuff deleted", " ......"] - + scr[-100:] - ) - scr.append("\n") - safertext = safertext + "\n".join(scr) - self.newtool.help = safertext - self.newtool.version_command = f'echo "{self.args.tool_version}"' - std = gxtp.Stdios() - std1 = gxtp.Stdio() - std.append(std1) - self.newtool.stdios = std - requirements = gxtp.Requirements() - if self.args.packages: - for d in self.args.packages.split(","): - ver = "" - d = d.replace("==", ":") - d = d.replace("=", ":") - if ":" in d: - packg, ver = d.split(":") - else: - packg = d - requirements.append( - gxtp.Requirement("package", packg.strip(), ver.strip()) - ) - self.newtool.requirements = requirements - if self.args.parampass == "0": - self.doNoXMLparam() - else: - self.doXMLparam() - self.newtool.outputs = self.toutputs - self.newtool.inputs = self.tinputs - if self.args.script_path: - configfiles = gxtp.Configfiles() - configfiles.append( - gxtp.Configfile(name="runme", text="\n".join(self.escapedScript)) - ) - self.newtool.configfiles = configfiles - tests = gxtp.Tests() - test_a = gxtp.Test() - for tp in self.testparam: - test_a.append(tp) - tests.append(test_a) - self.newtool.tests = tests - self.newtool.add_comment( - "Created by %s at %s using the Galaxy Tool Factory." - % (self.args.user_email, timenow()) - ) - self.newtool.add_comment("Source in git at: %s" % (toolFactoryURL)) - exml0 = self.newtool.export() - exml = exml0.replace(FAKEEXE, "") # temporary work around until PR accepted - if ( - self.test_override - ): # cannot do this inside galaxyxml as it expects lxml objects for tests - part1 = exml.split("<tests>")[0] - part2 = exml.split("</tests>")[1] - fixed = "%s\n%s\n%s" % (part1, "\n".join(self.test_override), part2) - exml = fixed - # exml = exml.replace('range="1:"', 'range="1000:"') - xf = open("%s.xml" % self.tool_name, "w") - xf.write(exml) - xf.write("\n") - xf.close() - # ready for the tarball - - def run(self): - """ - generate test outputs by running a command line - won't work if command or test override in play - planemo is the - easiest way to generate test outputs for that case so is - automagically selected - """ - scl = " ".join(self.cl) - err = None - if self.args.parampass != "0": - if os.path.exists(self.elog): - ste = open(self.elog, "a") - else: - ste = open(self.elog, "w") - if self.lastclredirect: - sto = open(self.lastclredirect[1], "wb") # is name of an output file - else: - if os.path.exists(self.tlog): - sto = open(self.tlog, "a") - else: - sto = open(self.tlog, "w") - sto.write( - "## Executing Toolfactory generated command line = %s\n" % scl - ) - sto.flush() - subp = subprocess.run( - self.cl, shell=False, stdout=sto, stderr=ste - ) - sto.close() - ste.close() - retval = subp.returncode - else: # work around special case - stdin and write to stdout - if len(self.infiles) > 0: - sti = open(self.infiles[0]["name"], "rb") - else: - sti = sys.stdin - if len(self.outfiles) > 0: - sto = open(self.outfiles[0]["name"], "wb") - else: - sto = sys.stdout - subp = subprocess.run( - self.cl, shell=False, stdout=sto, stdin=sti - ) - sto.write("## Executing Toolfactory generated command line = %s\n" % scl) - retval = subp.returncode - sto.close() - sti.close() - if os.path.isfile(self.tlog) and os.stat(self.tlog).st_size == 0: - os.unlink(self.tlog) - if os.path.isfile(self.elog) and os.stat(self.elog).st_size == 0: - os.unlink(self.elog) - if retval != 0 and err: # problem - sys.stderr.write(err) - logging.debug("run done") - return retval - - def shedLoad(self): - """ - use bioblend to create new repository - or update existing - - """ - if os.path.exists(self.tlog): - sto = open(self.tlog, "a") - else: - sto = open(self.tlog, "w") - - ts = toolshed.ToolShedInstance( - url=self.args.toolshed_url, - key=self.args.toolshed_api_key, - verify=False, - ) - repos = ts.repositories.get_repositories() - rnames = [x.get("name", "?") for x in repos] - rids = [x.get("id", "?") for x in repos] - tfcat = "ToolFactory generated tools" - if self.tool_name not in rnames: - tscat = ts.categories.get_categories() - cnames = [x.get("name", "?").strip() for x in tscat] - cids = [x.get("id", "?") for x in tscat] - catID = None - if tfcat.strip() in cnames: - ci = cnames.index(tfcat) - catID = cids[ci] - res = ts.repositories.create_repository( - name=self.args.tool_name, - synopsis="Synopsis:%s" % self.args.tool_desc, - description=self.args.tool_desc, - type="unrestricted", - remote_repository_url=self.args.toolshed_url, - homepage_url=None, - category_ids=catID, - ) - tid = res.get("id", None) - sto.write(f"#create_repository {self.args.tool_name} tid={tid} res={res}\n") - else: - i = rnames.index(self.tool_name) - tid = rids[i] - try: - res = ts.repositories.update_repository( - id=tid, tar_ball_path=self.newtarpath, commit_message=None - ) - sto.write(f"#update res id {id} ={res}\n") - except ConnectionError: - sto.write( - "####### Is the toolshed running and the API key correct? Bioblend shed upload failed\n" - ) - sto.close() - - def eph_galaxy_load(self): - """ - use ephemeris to load the new tool from the local toolshed after planemo uploads it - """ - if os.path.exists(self.tlog): - tout = open(self.tlog, "a") - else: - tout = open(self.tlog, "w") - cll = [ - "shed-tools", - "install", - "-g", - self.args.galaxy_url, - "--latest", - "-a", - self.args.galaxy_api_key, - "--name", - self.tool_name, - "--owner", - "fubar", - "--toolshed", - self.args.toolshed_url, - "--section_label", - "ToolFactory", - ] - tout.write("running\n%s\n" % " ".join(cll)) - subp = subprocess.run( - cll, - cwd=self.ourcwd, - shell=False, - stderr=tout, - stdout=tout, - ) - tout.write( - "installed %s - got retcode %d\n" % (self.tool_name, subp.returncode) - ) - tout.close() - return subp.returncode - - def writeShedyml(self): - """for planemo""" - yuser = self.args.user_email.split("@")[0] - yfname = os.path.join(self.tooloutdir, ".shed.yml") - yamlf = open(yfname, "w") - odict = { - "name": self.tool_name, - "owner": yuser, - "type": "unrestricted", - "description": self.args.tool_desc, - "synopsis": self.args.tool_desc, - "category": "TF Generated Tools", - } - yaml.dump(odict, yamlf, allow_unicode=True) - yamlf.close() - - def makeTool(self): - """write xmls and input samples into place""" - if self.args.parampass == 0: - self.doNoXMLparam() - else: - self.makeXML() - if self.args.script_path: - stname = os.path.join(self.tooloutdir, self.sfile) - if not os.path.exists(stname): - shutil.copyfile(self.sfile, stname) - xreal = "%s.xml" % self.tool_name - xout = os.path.join(self.tooloutdir, xreal) - shutil.copyfile(xreal, xout) - for p in self.infiles: - pth = p["name"] - dest = os.path.join(self.testdir, "%s_sample" % p["infilename"]) - shutil.copyfile(pth, dest) - dest = os.path.join(self.repdir, "%s_sample" % p["infilename"]) - shutil.copyfile(pth, dest) - - def makeToolTar(self, report_fail=False): - """move outputs into test-data and prepare the tarball""" - excludeme = "_planemo_test_report.html" - - def exclude_function(tarinfo): - filename = tarinfo.name - return None if filename.endswith(excludeme) else tarinfo - - if os.path.exists(self.tlog): - tout = open(self.tlog, "a") - else: - tout = open(self.tlog, "w") - for p in self.outfiles: - oname = p["name"] - tdest = os.path.join(self.testdir, "%s_sample" % oname) - src = os.path.join(self.testdir, oname) - if not os.path.isfile(tdest): - if os.path.isfile(src): - shutil.copyfile(src, tdest) - dest = os.path.join(self.repdir, "%s.sample" % (oname)) - shutil.copyfile(src, dest) - else: - if report_fail: - tout.write( - "###Tool may have failed - output file %s not found in testdir after planemo run %s." - % (tdest, self.testdir) - ) - tf = tarfile.open(self.newtarpath, "w:gz") - tf.add( - name=self.tooloutdir, - arcname=self.tool_name, - filter=exclude_function, - ) - tf.close() - shutil.copyfile(self.newtarpath, self.args.new_tool) - - def moveRunOutputs(self): - """need to move planemo or run outputs into toolfactory collection""" - with os.scandir(self.tooloutdir) as outs: - for entry in outs: - if not entry.is_file(): - continue - if "." in entry.name: - _, ext = os.path.splitext(entry.name) - if ext in [".tgz", ".json"]: - continue - if ext in [".yml", ".xml", ".yaml"]: - newname = f"{entry.name.replace('.','_')}.txt" - else: - newname = entry.name - else: - newname = f"{entry.name}.txt" - dest = os.path.join(self.repdir, newname) - src = os.path.join(self.tooloutdir, entry.name) - shutil.copyfile(src, dest) - if self.args.include_tests: - with os.scandir(self.testdir) as outs: - for entry in outs: - if (not entry.is_file()) or entry.name.endswith( - "_planemo_test_report.html" - ): - continue - if "." in entry.name: - _, ext = os.path.splitext(entry.name) - if ext in [".tgz", ".json"]: - continue - if ext in [".yml", ".xml", ".yaml"]: - newname = f"{entry.name.replace('.','_')}.txt" - else: - newname = entry.name - else: - newname = f"{entry.name}.txt" - dest = os.path.join(self.repdir, newname) - src = os.path.join(self.testdir, entry.name) - shutil.copyfile(src, dest) - - def planemo_test_once(self): - """planemo is a requirement so is available for testing but needs a - different call if in the biocontainer - see above - and for generating test outputs if command or test overrides are - supplied test outputs are sent to repdir for display - # "--galaxy_python_version", - # self.args.python_version, - - """ - xreal = "%s.xml" % self.tool_name - tool_test_path = os.path.join( - self.repdir, f"{self.tool_name}_planemo_test_report.html" - ) - if os.path.exists(self.tlog): - tout = open(self.tlog, "a") - else: - tout = open(self.tlog, "w") - cll = [ - "planemo", - "test", - "--test_data", - os.path.abspath(self.testdir), - "--test_output", - os.path.abspath(tool_test_path), - "--galaxy_root", - self.args.galaxy_root, - "--update_test_data", - os.path.abspath(xreal), - ] - p = subprocess.run( - cll, - shell=False, - cwd=self.tooloutdir, - stderr=tout, - stdout=tout, - ) - tout.close() - return p.returncode - -def main(): - """ - This is a Galaxy wrapper. - It expects to be called by a special purpose tool.xml - - """ - parser = argparse.ArgumentParser() - a = parser.add_argument - a("--script_path", default=None) - a("--history_test", default=None) - a("--cl_suffix", default=None) - a("--sysexe", default=None) - a("--packages", default=None) - a("--tool_name", default="newtool") - a("--tool_dir", default=None) - a("--input_files", default=[], action="append") - a("--output_files", default=[], action="append") - a("--user_email", default="Unknown") - a("--bad_user", default=None) - a("--make_Tool", default="runonly") - a("--help_text", default=None) - a("--tool_desc", default=None) - a("--tool_version", default=None) - a("--citations", default=None) - a("--command_override", default=None) - a("--test_override", default=None) - a("--additional_parameters", action="append", default=[]) - a("--selecttext_parameters", action="append", default=[]) - a("--edit_additional_parameters", action="store_true", default=False) - a("--parampass", default="positional") - a("--tfout", default="./tfout") - a("--new_tool", default="new_tool") - a("--galaxy_url", default="http://localhost:8080") - a("--toolshed_url", default="http://localhost:9009") - # make sure this is identical to tool_sheds_conf.xml - # localhost != 127.0.0.1 so validation fails - a("--toolshed_api_key", default="fakekey") - a("--galaxy_api_key", default="fakekey") - a("--galaxy_root", default="/galaxy-central") - a("--galaxy_venv", default="/galaxy_venv") - a("--collection", action="append", default=[]) - a("--include_tests", default=False, action="store_true") - a("--python_version", default="3.9") - args = parser.parse_args() - assert not args.bad_user, ( - 'UNAUTHORISED: %s is NOT authorized to use this tool until Galaxy \ -admin adds %s to "admin_users" in the galaxy.yml Galaxy configuration file' - % (args.bad_user, args.bad_user) - ) - assert args.tool_name, "## Tool Factory expects a tool name - eg --tool_name=DESeq" - assert ( - args.sysexe or args.packages - ), "## Tool Factory wrapper expects an interpreter \ -or an executable package in --sysexe or --packages" - r = ScriptRunner(args) - r.writeShedyml() - r.makeTool() - if args.make_Tool == "generate": - r.run() - r.moveRunOutputs() - r.makeToolTar() - else: - r.planemo_test_once() - r.moveRunOutputs() - r.makeToolTar(report_fail=True) - if args.make_Tool == "gentestinstall": - r.shedLoad() - r.eph_galaxy_load() - - -if __name__ == "__main__": - main()
--- a/toolfactory/rgToolFactory2.xml Sun Apr 18 03:55:56 2021 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,612 +0,0 @@ -<tool id="rgtf2" name="toolfactory" version="2.00" profile="16.04" > - <description>Scripts into tools v2.0</description> - <macros> - <xml name="singleText"> - <param name="param_value" type="text" value="" label="Enter this parameter's default text value"> - </param> - </xml> - <xml name="singleInt"> - <param name="param_value" type="integer" value="" label="Enter this parameter's default integer value" > - </param> - </xml> - <xml name="singleFloat"> - <param name="param_value" type="float" value="" label="Enter this parameter's default value"> - </param> - </xml> - <xml name="singleBoolean"> - <param name="param_value" type="boolean" value="" label="Enter this parameter's default value" /> - <param name="truevalue" type="text" value="True" label="Command line value to emit when True" /> - <param name="falsevalue" type="boolean" value="True" label="Command line value to emit when False" /> - </xml> - <xml name="selectText"> - <repeat name="selectTexts" title="Add each option to be presented in a text select box" min="2" default="2" - help="Each text added here will also have a value to be emitted on the command line when the text is chosen"> - <param name="select_text" type="text" value="" label="Enter the explanatory text the user will see for this choice" > - </param> - <param name="select_value" type="text" value="" label="Enter the value for the command line when the user selects this option"> - </param> - </repeat> - </xml> - <xml name="tool_metadata"> - <param name="tool_version" label="Tool Version - bump this to warn users trying to redo old analyses" type="text" value="0.01" - help="If you change your script and regenerate the 'same' tool, you should inform Galaxy (and users) by changing (bumping is traditional) this number"/> - <param name="tool_desc" label="Tool Synopsis" type="text" value="" - help="Supply a brief tool description for the Galaxy tool menu entry" /> - <param name="help_text" label="Tool form documentation and help text for users" type="text" area="true" - value="**What it Does**" - help="Supply user documentation to appear on the new tool form as reStructured text - http://docutils.sourceforge.net/docs/ref/rst/restructuredtext.html" > - <sanitizer> - <valid initial="string.printable" /> - <mapping initial="none"/> - </sanitizer> - </param> - <repeat name="citations" title="Citation"> - <conditional name="citation_type"> - <param name="type" type="select" display="radio" label="Citation Type"> - <option value="doi">DOI</option> - <option value="bibtex">BibTeX</option> - </param> - <when value="doi"> - <param name="doi" label="DOI" type="text" value="" - help="Supply a DOI (e.g. doi: 10.1111/j.1740-9713.2007.00258.x) to be cited when this tool is used in published research." /> - </when> - <when value="bibtex"> - <param name="bibtex" label="BibTex" type="text" area="true" - help="Supply a BibTex entry that should be cited when this tool is used in published research." value="" > - <sanitizer> - <valid initial="string.printable" /> - <mapping initial="none"/> - </sanitizer> - </param> - </when> - </conditional> - </repeat> - </xml> - <xml name="io"> - <section name="io" title="Input and output files" expanded="true"> - <repeat name="history_inputs" title="zero or more input data files to pass on the command line to the executable." - help="USE SMALL SAMPLES for the new tool's test. Prompts will form a history item selector as input for users of this new tool"> - <param name="input_files" type="data" format="data" label="Select an input file from your history" multiple="false" - help=""/> - <param name="input_formats" type="select" multiple="true" label="Select the datatype(s) that your tool/script accepts as input" - help="If your datatype is not listed here, it has to be added in galaxy's datatypes_conf.xml" value="tabular"> - <options from_parameter="tool.app.datatypes_registry.upload_file_formats"> - <column name="value" index="0"/> - </options> - </param> - <param name="input_label" type="text" value="" label="This will become the user prompt for the form so please make it informative" > - </param> - <param name="input_help" type="text" value="" label="This will become help text on the form." > - </param> - <param name="input_CL" type="text" label="Positional: ordinal integer. Argparse: argument name. STDIN if the executable/script expects it" - help="If you will pass positional parameters, enter the integer ordinal for this parameter. If Argparse style, '--' will be prepended or '-' if single character" value=""> - </param> - <param name="input_repeat" type="boolean" checked="False" truevalue="1" falsevalue="0" label="Allow user to repeat this input as many times as wanted - script/executable must parse these."/> - </repeat> - <repeat name="history_outputs" title="one or more new history items output by the executable to appear in the user history after the tool runs" - help="The name will become a history item for users of the new tool you are making containing one of it's outputs"> - <param name="history_name" type="text" label="Name for this output to appear in new history" optional="false" - help="No spaces! Argparse will also use this name as --[name]"> - <sanitizer invalid_char="_"> - <valid initial="string.ascii_letters,string.digits" > - <add value="_" /> - </valid> - </sanitizer> - </param> - <param name="history_format" type="select" multiple="false" label="Select the datatype for this output" - help="If your datatype is not listed here, it has to be added in galaxy's datatypes_conf.xml" value="tabular"> - <options from_parameter="tool.app.datatypes_registry.upload_file_formats"> - <column name="value" index="0"/> - </options> - </param> - <param name="history_CL" type="text" label="Positional: ordinal integer. Use STDOUT if '>' required. Otherwise ignored if argparse because name is used" - help="If positional parameters, enter the integer ordinal expected for this parameter. If argparse, ignore unless STDOUT needed" value=""/> - <param name="history_test" type="text" label="Test pass decision criterion for this output compared to test generation" - help="Available options are diff:[lines], sim_size:[delta (integer) or delta_frac (float)" value="diff:0"/> - </repeat> - <repeat name="collection_outputs" title="zero or more new history collections to appear in the user history after the tool runs" - help="Hides outputs not needed for downstream tools such as images and intermediate reports." default="0"> - <param name="name" type="text" label="Name for new collection with all outputs not individually passed to history outputs" - help="No spaces! Script MUST create and fill a directory with the same name and the tool test MUST be overwritten - see the plotter example for hints"> - <sanitizer invalid_char="_"> - <valid initial="string.ascii_letters,string.digits" > - <add value="_" /> - </valid> - </sanitizer> - </param> - <param name="kind" type="select" multiple="false" label="Select the kind of collection for this output" - help=""> - <option value="list" selected="true">List</option> - <option value="paired">Paired</option> - <option value="list:paired">List of paired</option> - </param> - <param name="label" type="text" label="Label for this collection" help="" value=""/> - <param name="discover" type="text" label="Discover datasets expression" help="__name_and_ext__ should catch all filenames with extensions" value="__name_and_ext__"/> - </repeat> - - </section> - </xml> - <xml name="additparam"> - <section name="addparam" title="Executable or script settings passed on the command line other than I/O files" expanded="true"> - <param name="edit_params" type="select" display="radio" label="Make these settings on the generated tool form user editable?" - help="If no, users will NOT be able to alter any additional parameters. If yes, these will appear on the tool form as text fields with no validation or sanitizing"> - <option value="yes" selected="true">Yes, allow user to edit all additional parameters on the generated tool form</option> - <option value="no">No - use the fixed values for all additional parameters - no user editing</option> - </param> - <repeat name="additional_parameters" title="zero or more command line settings for the user to pass to the executable" - help="See examples below to see how these can be parsed by scripts in the various languages"> - <param name="param_name" type="text" value="" label="Choose the name for this parameter - MUST not be blank!"> - <sanitizer invalid_char=""> - <valid initial="string.ascii_letters,string.digits"/> - <mapping initial="none"/> - </sanitizer> - </param> - <conditional name="ap_type"> - <param name="param_type" type="select" label="Select the type for this parameter"> - <option value="text" selected="true">Text string</option> - <option value="integer">Integer</option> - <option value="float">Float</option> - <option value="boolean">Boolean</option> - <option value="selecttext">Select text string</option> - </param> - <when value = "text"> - <expand macro="singleText" /> - </when> - <when value = "integer"> - <expand macro="singleInt" /> - </when> - <when value = "float"> - <expand macro="singleFloat" /> - </when> - <when value = "boolean"> - <expand macro="singleBoolean" /> - </when> - <when value = "selecttext"> - <expand macro="selectText" /> - </when> - </conditional> - <param name="param_label" type="text" value="" label="Enter this parameter's label for the form"> - </param> - <param name="param_help" type="text" value="" label="Help for this parameter"> - </param> - <param name="param_CL" type="text" label="Positional ordinal | argparse argument name" - help="Using positional parameters, enter the integer ordinal for this parameter on the command line. Using Argparse style, '--' will be prepended on the CL" value="" /> - <param name="param_CLprefixed" type="text" label="Override the generated default argparse name prefix if not empty - eg ----foo if needed" - help="Some targets like Planemo expect an unadorned action like 'test' before --galaxy_root." value="" /> - <param name="param_repeat" type="boolean" checked="False" truevalue="1" falsevalue="0" label="Allow user to repeat this as many times as needed." /> - </repeat> - </section> - </xml> - </macros> - -<requirements> - <container type="docker">quay.io/biocontainers/mulled-v2-0a86ccf22d71945a175383bcea5edd6b51c25ed0:55f1b8132a10ad91f0016ba9f4c692627ca2d25b-0</container> - <requirement type="package" version="0.4.14">galaxyxml</requirement> - <requirement type="package" version="0.74.3">planemo</requirement> -</requirements> - -<command detect_errors="exit_code"><![CDATA[ -#import os -#set dev_env = os.environ.get('GALAXY_DEVELOPMENT_ENVIRONMENT', '0') == '1' -#if not $dev_env and ( $__user_email__ not in $__admin_users__ ): -python3 $__tool_directory__/rgToolFactory2.py --bad_user $__user_email__ - #else: -python3 $__tool_directory__/rgToolFactory2.py - #if len(str($cl_suffix)) > 0: ---cl_suffix "$cl_suffix" - #end if - #if $cover.commover == "yes": - #if len(str($cover.command_override)) > 10: ---command_override "$commandoverride" - #end if - #if len(str($cover.test_override)) > 10: ---test_override "$testoverride" - #end if - #end if ---packages "$deps.packages" - #if $deps.usescript.choosescript == "yes": ---script_path "$runme" ---sysexe "$deps.usescript.scriptrunner" - #end if ---tool_name "$tool_name" --user_email "$__user_email__" --citations "$citeme" --parampass "$io_param.ppass.parampass" - - #if str($make.makeMode.make_Tool)!="runonly": ---make_Tool "$make.makeMode.make_Tool" ---tool_desc "$make.makeMode.tool_desc" ---tool_version "$make.makeMode.tool_version" ---help_text "$helpme" ---new_tool "$new_tool" ---toolshed_api_key "$make.makeMode.toolshed_apikey" ---galaxy_api_key "$make.makeMode.galaxy_apikey" ---toolshed_url "$make.makeMode.toolshed_url" ---galaxy_url "$make.makeMode.galaxy_url" - #end if - #if $io_param.ppass.parampass != '0': - #if str($io_param.ppass.addparam.edit_params) == "yes": ---edit_additional_parameters - #end if - #for $apar in $io_param.ppass.addparam.additional_parameters: - #if $apar.ap_type.param_type=="selecttext": ---selecttext_parameters '{"name":"$apar.param_name", "label":"$apar.param_label", "help":"$apar.param_help", -"type":"$apar.ap_type.param_type","CL":"$apar.param_CL","override":"$apar.param_CLprefixed","value": [ - #for $i,$st in enumerate($apar.ap_type.selectTexts): - "$st.select_value" - #if ($i < (len($apar.ap_type.selectTexts)-1)): - , - #end if - #end for - ], "texts": [ - #for $i,$st in enumerate($apar.ap_type.selectTexts): - "$st.select_text" - #if ($i < (len($apar.ap_type.selectTexts)-1)): - , - #end if - - #end for - ] - }' - #else: ---additional_parameters '{"name": "$apar.param_name", "value": "$apar.ap_type.param_value", "label": "$apar.param_label", "help": "$apar.param_help", -"type": "$apar.ap_type.param_type","CL": "$apar.param_CL","override": "$apar.param_CLprefixed", "repeat": "$apar.param_repeat"}' - #end if - #end for - #end if - #for $intab in $io_param.ppass.io.history_inputs: ---input_files '{"name": "$intab.input_files", "CL": "$intab.input_CL", "format": "$intab.input_formats", "label": "$intab.input_label", "help": "$intab.input_help", "repeat": "$intab.input_repeat"}' - #end for - #for $otab in $io_param.ppass.io.history_outputs: ---output_files '{"name": "$otab.history_name", "format": "$otab.history_format", "CL": "$otab.history_CL", "test": "$otab.history_test"}' - #end for - #for $collect in $io_param.ppass.io.collection_outputs: ---collection '{"name": "$collect.name", "kind": "$collect.kind", "discover": "$collect.discover", "label": "$collect.label"}' - #end for ---galaxy_root "$__root_dir__" ---tool_dir "$__tool_directory__" - #end if -]]></command> - <configfiles> - <configfile name="runme"> -$deps.usescript.dynScript - </configfile> - <configfile name="commandoverride"> -#if $cover.commover == "yes" and len(str($cover.command_override).strip()) > 1: -$cover.command_override -#end if - </configfile> - <configfile name="testoverride"> -#if $cover.commover == "yes" and len(str($cover.test_override).strip()) > 1: -$cover.test_override -#end if - </configfile> - <configfile name="helpme"> - #if $make.makeMode.make_Tool != "runonly": -${make.makeMode.help_text} - #else -$tool_name help goes here - #end if - </configfile> - <configfile name="citeme"> -#if $make.makeMode.make_Tool != "runonly": - #for $citation in $make.makeMode.citations: - #if $citation.citation_type.type == "bibtex": - **ENTRY**bibtex - ${citation.citation_type.bibtex} - #else - **ENTRY**doi - ${citation.citation_type.doi} - #end if - #end for -#end if - </configfile> - </configfiles> - <inputs> - <param name="tool_name" type="text" value="tool1" label="New tool ID and title for outputs" - help="Toolshed repository name. Choose thoughtfully to avoid namespace clashes with other tool writers. Lower case, digits and underscores only"> - <sanitizer invalid_char=""> - <valid initial="string.ascii_lowercase,string.digits"> - <add value="_"/> - </valid> - </sanitizer> - </param> - - <section name="deps" title="Dependencies, optional script and script interpreter" expanded="true"> - - <param name="packages" type="text" value="" label="Conda dependencies as package name[:version, name:version...]. These will always be available when this tool executes" - help="Use =[ver] or :[ver] for specific version - 'bwa=0.17.0'. Default is latest. Will be used every time the tool is (re)run. Only Conda is currently supported" /> - - <conditional name="usescript"> - <param name="choosescript" type="select" display="radio" label="Supply a script for a dependency (e.g. python/R/bash) or a system executable such as Bash" - help = "For script interpreters like Python or bash, parameters and i/o specified below must match script's expectations - if you pass parameters, the script must deal with them"> - <option value="no">No script required for this tool - just pass parameters on the command line to the first dependency listed above</option> - <option value="yes" selected="true">Yes, a script is ready to be pasted below</option> - </param> - <when value="no"> - <param name="dynScript" type="hidden" value="" /> - <param name="scriptrunner" type="hidden" value="" /> - </when> - <when value="yes"> - <param name="scriptrunner" type="text" value="" label="Interpreter for the script - eg bash or python. Can be one of the dependencies named above or a system executable" - help="Scripts are interpreted by the executable named here. For conda r-base, 'Rscript --vanilla' or for conda planemo, 'planemo test' for example"> - </param> - <param name="dynScript" type="text" area="True" value="" label="Script for executable above to interpret. It can be one of the Conda dependency names " - help="Script must handle all i/o and parameters as specified below using the parameters and passing method chosen below"> - <sanitizer> - <valid initial="string.printable"/> - <mapping initial="none"/> - </sanitizer> - </param> - </when> - </conditional> - </section> - <section name="io_param" title="Data file input, output and settings forming the executable or script command line" expanded="true"> - <conditional name="ppass"> - <param name="parampass" type="select" display="radio" label="Command line parameter passing method to use"> - <option value="argparse" selected="true">Argparse: passed in the form of --clname value</option> - <option value="positional">Positional: Passed in the order of positional ordinals ...foo.bam bar.idx zot.xls</option> - <option value="0">Tool reads selected input file from STDIN and writes STDOUT with no parameters</option> - </param> - <when value="argparse"> - <expand macro="io" /> - <expand macro="additparam" /> - </when> - <when value="positional"> - <expand macro="io" /> - <expand macro="additparam" /> - </when> - <when value="0"> - <expand macro="io"/> - </when> - </conditional> - </section> - <param name="cl_suffix" type="text" value="" label="Suffix for generated command line. Useful for bash post processing. Use override below to replace completely" - help="';' separated bash commands can be used here for post processing - added at end of autogenerated command line"> - <sanitizer> - <valid initial="string.printable" /> - <mapping initial="none"/> - </sanitizer> - </param> - <conditional name="cover"> - <param name="commover" type="select" display="radio" label="Add Human wrought code to override the generated XML command and/or test section - DIY" - help = "For arbitrary and artfull command lines. All i/o and parameters must be passed. Choose No unless needed. Not for the faint of heart"> - <option value="no" selected="true">No. Use automatically generated command/test XML </option> - <option value="yes">Yes. XML needed to override autogenerated command and/or test segments will be pasted below</option> - </param> - <when value="no"> - <param name="command_override" type="hidden" value="" /> - <param name="test_override" type="hidden" value="" /> - </when> - <when value="yes"> - <param name="command_override" type="text" area="True" value="" label="Optional. Human wrought command element override XML/template - e.g. for bwa" - help="For arbitrary and artfull command lines. All i/o and parameters must be passed. Leave blank unless needed. Not for the faint of heart"> - <sanitizer> - <valid initial="string.printable"/> - <mapping initial="none"/> - </sanitizer> - </param> - <param name="test_override" type="text" area="True" value="" label="Optional. Human wrought test element override XML/template - e.g. for bwa" - help="For arbitrary and artfull scripts. Leave blank unless needed. Not for the faint of heart"> - <sanitizer> - <valid initial="string.printable" /> - <mapping initial="none"/> - </sanitizer> - </param> - </when> - </conditional> - <section name="make" title="Generate, toolshed and local Galaxy installation options" expanded="true"> - <conditional name="makeMode"> - <param name="make_Tool" type="select" display="radio" label="Choose the steps you want to run. The TF Docker container is recommended for local installation" - help="Installation in this Galaxy is optional" > - <option value="generate" >Run to generate tests only. Should fail if dependencies needed.</option> - <option value="gentest" selected="true">Test with planemo after generating.</option> - <option value="gentestinstall">Install in this Galaxy after generation and testing. Must have local ToolShed as in the TF Docker container</option> - </param> - <when value="generate"> - <param name="galaxy_apikey" value="" type="hidden" ></param> - <param name="toolshed_apikey" value="" type="hidden" ></param> - <param name="galaxy_url" value="" type="hidden" ></param> - <param name="toolshed_url" value="" type="hidden" ></param> - <expand macro="tool_metadata" /> - </when> - <when value="gentest"> - <param name="galaxy_apikey" value="" type="hidden" ></param> - <param name="toolshed_apikey" value="" type="hidden" ></param> - <param name="galaxy_url" value="" type="hidden" ></param> - <param name="toolshed_url" value="" type="hidden" ></param> - <expand macro="tool_metadata" /> - </when> - <when value="gentestinstall"> - <param name="galaxy_url" type="text" value="http://localhost:8080" label="URL for the Galaxy server where the new tool should be installed" - help="Default is localhost"> - <sanitizer> - <valid initial="string.printable" /> - <mapping initial="none"/> - </sanitizer> - </param> - <param name="galaxy_apikey" type="text" value="fakekey" label="API key for the Galaxy to install the new tool" - help="Cut and paste from the admin user properties screen"> - <sanitizer> - <valid initial="string.ascii_letters,string.digits" /> - <mapping initial="none"/> - </sanitizer> - </param> - <param name="toolshed_url" type="text" value="http://localhost:9009" label="URL for the Toolshed where the new tool should be installed" - help="Default value is localhost:9009"> - <sanitizer> - <valid initial="string.printable" /> - <mapping initial="none"/> - </sanitizer> - </param> - <param name="toolshed_apikey" type="text" value="fakekey" label="API key for the local toolshed to use when installing the tool" - help="Cut and paste from the admin user properties screen"> - <sanitizer> - <valid initial="string.ascii_letters,string.digits" /> - <mapping initial="none"/> - </sanitizer> - </param> - <expand macro="tool_metadata" /> - </when> - </conditional> - </section> - </inputs> - <outputs> - - <data format="toolshed.gz" name="new_tool" label="${tool_name}_toolshed.gz" > - <filter>makeMode['make_Tool'] != "runonly"</filter> - </data> - - <collection name="TF_run_report" type="list" label="${tool_name} outputs"> - <discover_datasets pattern="__name_and_ext__" directory="TF_run_report_tempdir" /> - </collection> - </outputs> -<tests> -<test> - <param name="user_email" value="admin@galaxy.org"/> - <param name="input_files" value="input1_sample" /> - <param name="input_CL" value="1" /> - <param name="input_formats" value="txt" /> - <param name="input_label" value="input" /> - <param name="input_help" value="help" /> - <param name="tool_name" value="pyrevpos" /> - <param name="parampass" value="positional" /> - <param name="make_Tool" value="gentest" /> - <param name="tool_version" value="0.01" /> - <param name="tool_desc" value="positional reverse" /> - <param name="help_text" value="help text goes here" /> - <param name="packages" value="python"/> - <param name="history_name" value="output2" /> - <param name="history_format" value="txt" /> - <param name="history_CL" value="2" /> - <param name="dynScript" value="import sys; inp = sys.argv[1]; outp = sys.argv[2]; inlist = open(inp,'r').readlines(); o = open(outp,'w'); rs = [''.join(list(reversed(x.rstrip()))) for x in inlist]; o.write('\n'.join(rs)); o.close()"/> - <param name="choosescript" value="yes" /> - <param name="script_path" value="$runme"/> - <output name="new_tool" file="toolfactory_pyrevpos_tgz_sample" compare="sim_size" delta="6000" /> -</test> -</tests> -<help> - -.. class:: warningmark - -**Details and attribution** -(see GTF_) - -**Local Admins ONLY** -Only users whose IDs found in the local admin_user configuration setting in universe_wsgi.ini can run this tool. - -**If you find a bug** -Please raise an issue, or even better, submit a pull request fixing it, on the github repository GTF_ - -**What it does** -This tool optionally generates normal workflow compatible first class Galaxy tools - -Generated tools can run existing binary packages that become requirements, existing scripts, or new scripts pasted into this tool form. -Pasted scripts are written so they are part of the new tool and cannot be adjusted by the downstream user. -Binary packages are managed by the dependency subsystem - conda usually, so anything in bioconda or conda_forge is available for example. - -Any number of parameters can be built into the new tool form for passing in to the script or executable at runtime. -These can be editable by the downstream user or baked in. - -When you run this tool, your executable or script and supplied parameter values will be run to produce a canonical -set of outputs - these are used to construct a test for the new tool. - -If tool generation is required, a new tarball compatible with any Galaxy toolshed is created. -It can be unpacked in your galaxy/tools directory and manually added to tool_conf.xml, or -installed into any toolshed from where it can be installed into your Galaxy. - - -.. class:: warningmark - -**Note to system administrators** -This tool offers *NO* built in protection against malicious scripts. It should only be installed on private/personnal Galaxy instances. -Admin_users will have the power to do anything they want as the Galaxy user if you install this tool. - -.. class:: warningmark - -**Use on public servers** is STRONGLY discouraged for obvious reasons - -The tools generated by this tool will run just as securely as any other normal installed Galaxy tool but like any other new tools, should always be checked carefully before installation. -We recommend that you follow the good code hygiene practices associated with safe toolshed practices. - -Here's a sample python script that can be cut and pasted into the tool form, suitable for positional parameter passing: - -:: - - # reverse order of text by row - import sys - inp = sys.argv[1] - outp = sys.argv[2] - i = open(inp,'r').readlines() - o = open(outp,'w') - for row in i: - rs = row.rstrip() - rs = list(rs) - rs.reverse() - o.write(''.join(rs)) - o.write('\n') - o.close() - -With argparse style parameters: - -:: - - # reverse order of text by row - import argparse - parser = argparse.ArgumentParser() - a = parser.add_argument - a('--infile',default='') - a('--outfile',default=None) - args = parser.parse_args() - inp = args.infile - outp = args.outfile - i = open(inp,'r').readlines() - o = open(outp,'w') - for row in i: - rs = row.rstrip() - rs = list(rs) - rs.reverse() - o.write(''.join(rs)) - o.write('\n') - o.close() - -R script to draw some plots - use a collection. - -:: - - - \# note this script takes NO input because it generates random data - dir.create('plots') - for (i in 1:10) { - foo = runif(100) - bar = rnorm(100) - bar = foo + 0.05*bar - pdf(paste('plots/yet',i,"anotherplot.pdf",sep='_')) - plot(foo,bar,main=paste("Foo by Bar plot \#",i),col="maroon", pch=3,cex=0.6) - dev.off() - foo = data.frame(a=runif(100),b=runif(100),c=runif(100),d=runif(100),e=runif(100),f=runif(100)) - bar = as.matrix(foo) - pdf(paste('plots/yet',i,"anotherheatmap.pdf",sep='_')) - heatmap(bar,main='Random Heatmap') - dev.off() - } - - - -Paper_ - -*Licensing* - -Copyright Ross Lazarus (ross period lazarus at gmail period com) May 2012 -All rights reserved. -Licensed under the LGPL_ - -.. _LGPL: http://www.gnu.org/copyleft/lesser.html -.. _GTF: https://github.com/fubar2/toolfactory -.. _Paper: https://academic.oup.com/bioinformatics/article/28/23/3139/192853 - - -</help> -<citations> - <citation type="doi">10.1093/bioinformatics/bts573</citation> -</citations> -</tool> - -
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/toolfactory/testclass.py Fri Apr 30 07:06:57 2021 +0000 @@ -0,0 +1,108 @@ +import argparse +import json +import os +import requests + + +from bioblend import galaxy + +class ToolTester(): + # test a newly installed tool using bioblend + """ + + https://github.com/nsoranzo/bioblend-tutorial/blob/master/historical_exercises/api-scripts.exercises/run_tool.py +import sys +import json +import requests +import output + +BASE_URL = 'http://localhost:8080' + +# ----------------------------------------------------------------------------- +def run_tool( tool_id, history_id, **kwargs ): + full_url = BASE_URL + '/api/tools' + + #EXERCISE: POST... + +# ----------------------------------------------------------------------------- +if __name__ == '__main__': + # e.g. ./run_tool.py Filter1 ebfb8f50c6abde6d '{ "input" : { "src": "hda", "id": "77f74776fd03cbc5" }, "cond" : "c6>=100.0" }' + # e.g. ./run_tool.py sort1 f597429621d6eb2b '{ "input": { "src": "hda", "id": "b472e2eb553fa0d1" }, "column": "c6", "style": "alpha", "column_set_0|other_column" : "c2", "column_set_0|other_style": "num" }' + tool_id, history_id = sys.argv[1:3] + params = json.loads( sys.argv[3] ) if len( sys.argv ) >= 4 else {} + response = run_tool( tool_id, history_id, **params ) + output.output_response( response ) + + + def get_testdata(self,urlin,fout): + ''' + grab a test file + GET /api/tools/{tool_id}/test_data_download?tool_version={tool_version}&filename={filename} + http://localhost:8080/api/tools/tacrev/test_data_download?tool_version=2.00&filename=in + ''' + """ + def __init__(self, args): + self.galaxy = args.galaxy + self.key = args.key + self.tool_id = args.tool_id + + def run_test(self): + """ + GET /api/tools/{tool_id}/test_data_download?tool_version={tool_version}&filename={filename} + http://localhost:8080/api/tools/tacrev/test_data_download?tool_version=2.00&filename=input1 + """ + inputs = {} + gi = galaxy.GalaxyInstance(url=self.galaxy, key=self.key, verify=False) + chistory = gi.histories.get_most_recently_used_history() + chistory_id = chistory['id'] + #contents = gi.histories.show_history(chistory_id, contents=True) + #print('####chistory',chistory,'\n#### contents=',contents) + #history = gi.histories.create_history(name=f"{self.tool_id}_test_history") + #new_hist_id = history['id'] + fapi = ''.join([self.galaxy, '/api/tools/', self.tool_id, '/build']) + build = gi.make_get_request(url=fapi,params={"history_id":chistory_id}).json() + fapi = ''.join([self.galaxy, '/api/tools/', self.tool_id, '/test_data']) + test_data = requests.get(fapi, params={'key':self.key, 'history_id':chistory_id})# gi.make_get_request(url=fapi,params={"history_id":chistory_id,'key':self.key}).json() + print(test_data) + testinputs = test_data.json()[0].get('inputs',None) + print('testinputs',testinputs) + stateinputs = build.get('state_inputs',None) # 'input1': {'values': [{'id': '7b326180327c3fcc', 'src': 'hda'}]}} + if testinputs: + for k in testinputs.keys(): + v = testinputs[k] + if '|' in k: + nk = k.split('|')[-1] + inputs[nk] = v + else: + inputs[k] = v + if stateinputs: + print('stateinputs',stateinputs) + for k in stateinputs.keys(): + inp = stateinputs[k] + if isinstance(inp,dict): + if inp.get('values',None): + for anin in inp['values']: + if anin.get('id', None) and anin.get('src', None): + gi.histories.copy_dataset(chistory_id, anin['id'], source=anin['src']) + print('******copied id', anin['id']) + up = {k:anin} + print(up) + inputs.update(up) # replace the input def + print('after state inputs', inputs) + fapi = ''.join([self.galaxy, '/api/tools']) + r = gi.tools.run_tool(chistory_id, self.tool_id, inputs, input_format='legacy') + print(f"Called test on {self.tool_id} - got {r}") + +def _parser(): + parser = argparse.ArgumentParser() + parser.add_argument("-g", "--galaxy", help='URL of target galaxy',default="http://localhost:8080") + parser.add_argument("-a", "--key", help='Galaxy admin key', default="13073fde17d06591ce36e596e3c29904") + parser.add_argument("-t", "--tool_id", help='Tool id to test', default="plotter") + return parser + + +if __name__ == "__main__": + args = _parser().parse_args() + tt = ToolTester(args) + tt.run_test() +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/toolfactory/toolwatcher.py Fri Apr 30 07:06:57 2021 +0000 @@ -0,0 +1,123 @@ +#!/usr/bin/python +from datetime import datetime, timedelta +from io import BytesIO as BIO +import logging +import os +import subprocess +import tarfile +import time +from watchdog.observers import Observer +from watchdog.events import FileSystemEventHandler +from watchdog.events import PatternMatchingEventHandler + +class ToolHandler(PatternMatchingEventHandler): + + def __init__(self, watchme): + PatternMatchingEventHandler.__init__(self, patterns=['*.xml'], + ignore_directories=False, case_sensitive=False) + self.last_modified = datetime.now() + self.tool_dir = watchme + self.work_dir = os.getcwd() + self.galaxy_root = os.path.split(watchme)[0] + logging.info(self.galaxy_root) + self.tar_dir = os.path.join(self.galaxy_root, 'tooltardir') + if not os.path.exists(self.tar_dir): + os.mkdir(self.tar_dir) + + def on_created(self, event): + self.on_modified(event) + + def on_modified(self, event): + if datetime.now() - self.last_modified < timedelta(seconds=1): + return + else: + if os.path.exists(event.src_path): + self.last_modified = datetime.now() + logging.info(f"{event.src_path} was {event.event_type}") + p = self.planemo_test(event.src_path) + if p: + if p.returncode == 0: + newtarpath = self.makeToolTar(event.src_path) + logging.info('### Tested toolshed tarball %s written' % newtarpath) + else: + logging.debug('### planemo stdout:') + logging.debug(p.stdout) + logging.debug('### planemo stderr:') + logging.debug(p.stderr) + logging.info('### Planemo call return code =' % p.returncode) + else: + logging.info('Directory %s deleted' % event.src_path) + + def planemo_test(self, xml_path): + toolpath, toolfile = os.path.split(xml_path) + dirlist = os.listdir(toolpath) + toolname = os.path.basename(toolpath) + logging.info('### test dirlist %s, path %s toolname %s' % (dirlist, xml_path, toolname)) + xmls = [x for x in dirlist if os.path.splitext(x)[1] == '.xml'] + if not len(xmls) > 0: + logging.warning('Found no xml files after change to %s' % xml_path) + return None + tool_test_output = os.path.join(toolpath, f"{toolname}_planemo_test_report.html") + cll = [ + "planemo", + "test", + "--test_output", + tool_test_output, + "--galaxy_root", + self.galaxy_root, + "--update_test_data", + xml_path, + ] + logging.info('### calling %s' % ' '.join(cll)) + p = subprocess.run( + cll, + cwd = toolpath, + shell=False, + capture_output=True, + encoding='utf8', + ) + return p + + def makeToolTar(self, xml_path): + """move outputs into test-data and prepare the tarball""" + excludeme = "_planemo_test_report.html" + + def exclude_function(tarinfo): + filename = tarinfo.name + return None if filename.endswith(excludeme) else tarinfo + + tooldir, xml_file = os.path.split(xml_path) + os.chdir(self.tool_dir) + toolname = os.path.splitext(xml_file)[0] + newtarpath = os.path.join(self.tar_dir, '%s_toolshed.gz' % toolname) + tf = tarfile.open(newtarpath, "w:gz") + tf.add( + name=toolname, + arcname=toolname, + filter=exclude_function, + ) + tf.close() + os.chdir(self.work_dir) + return newtarpath + + +if __name__ == "__main__": + watchme = '/home/ross/gal21/tools' + logging.basicConfig(level=logging.INFO, + #filename = os.path.join(watchme,"toolwatcher.log") + #filemode = "w", + format='%(asctime)s - %(message)s', + datefmt='%Y-%m-%d %H:%M:%S') + event_handler = ToolHandler(watchme=watchme) + observer = Observer() + observer.schedule(event_handler, path=watchme, recursive=True) + observer.start() + try: + while True: + time.sleep(1) + except KeyboardInterrupt: + observer.stop() + observer.join() + + +