|
3
|
1 <tool name="toolfactory_tester" id="toolfactory_tester" version="1">
|
|
|
2 <!--Source in git at: https://github.com/fubar2/toolfactory-->
|
|
|
3 <!--Created by admin@galaxy.org at 23/04/2021 10:25:58 using the Galaxy Tool Factory.-->
|
|
|
4 <description>Test an untested tool and update it</description>
|
|
|
5 <requirements>
|
|
|
6 <requirement type="package" version="2.30.2">git</requirement>
|
|
|
7 <requirement type="package" version="0.74.3">planemo</requirement> </requirements>
|
|
|
8 <stdio>
|
|
|
9 <exit_code range="1:" level="fatal"/>
|
|
|
10 </stdio>
|
|
|
11 <version_command><![CDATA[echo "1"]]></version_command>
|
|
|
12 <command><![CDATA[
|
|
|
13 mkdir 'TF_run_report';
|
|
|
14 python
|
|
|
15 $runme
|
|
|
16 --in_tool_archive
|
|
|
17 $in_tool_archive
|
|
|
18 --new_tested_tool_archive
|
|
|
19 $new_tested_tool_archive
|
|
|
20 --galaxy_root
|
|
|
21 "$__root_dir__"
|
|
|
22 >
|
|
|
23 "TF_run_report/${in_tool_archive.name}_test_log.txt"
|
|
|
24 ]]></command>
|
|
|
25 <configfiles>
|
|
|
26 <configfile name="runme"><![CDATA[#raw
|
|
|
27
|
|
|
28 # see https://github.com/fubar2/toolfactory
|
|
|
29 #
|
|
|
30 # copyright ross lazarus (ross stop lazarus at gmail stop com) May 2012
|
|
|
31 #
|
|
|
32 # all rights reserved
|
|
|
33 # Licensed under the LGPL
|
|
|
34 # suggestions for improvement and bug fixes welcome at
|
|
|
35 # https://github.com/fubar2/toolfactory
|
|
|
36 #
|
|
|
37 # July 2020: BCC was fun and I feel like rip van winkle after 5 years.
|
|
|
38 # Decided to
|
|
|
39 # 1. Fix the toolfactory so it works - done for simplest case
|
|
|
40 # 2. Fix planemo so the toolfactory function works
|
|
|
41 # 3. Rewrite bits using galaxyxml functions where that makes sense - done
|
|
|
42
|
|
|
43 import argparse
|
|
|
44 import copy
|
|
|
45 import os
|
|
|
46 import subprocess
|
|
|
47 import shutil
|
|
|
48 import sys
|
|
|
49 import tarfile
|
|
|
50 import tempfile
|
|
|
51 import time
|
|
|
52 import xml.etree.ElementTree as ET
|
|
|
53
|
|
|
54
|
|
|
55 myversion = "V2.2 April 2021"
|
|
|
56 verbose = True
|
|
|
57 debug = True
|
|
|
58 toolFactoryURL = "https://github.com/fubar2/toolfactory"
|
|
|
59
|
|
|
60 def timenow():
|
|
|
61 """return current time as a string"""
|
|
|
62 return time.strftime("%d/%m/%Y %H:%M:%S", time.localtime(time.time()))
|
|
|
63
|
|
|
64 class ToolTester():
|
|
|
65 # uncompress passed tar, run planemo and rebuild a new tarball with tests
|
|
|
66
|
|
|
67 def __init__(self, args=None, in_tool_archive='/galaxy-central/tools/newtool/newtool_toolshed.gz', new_tool_archive=None):
|
|
|
68 self.args = args
|
|
|
69 self.new_tool_archive = new_tool_archive
|
|
|
70 assert tarfile.is_tarfile(in_tool_archive)
|
|
|
71 # this is not going to go well with arbitrary names.
|
|
|
72 tff = tarfile.open(in_tool_archive, "r:*")
|
|
|
73 flist = tff.getnames()
|
|
|
74 ourdir = os.path.commonpath(flist) # eg pyrevpos
|
|
|
75 self.tool_name = ourdir
|
|
|
76 ourxmls = [x for x in flist if x.lower().endswith('.xml') and os.path.split(x)[0] == ourdir]
|
|
|
77 # planemo_test/planemo_test.xml
|
|
|
78 assert len(ourxmls) > 0
|
|
|
79 self.ourxmls = ourxmls # [os.path.join(tool_path,x) for x in ourxmls]
|
|
|
80 res = tff.extractall()
|
|
|
81 self.update_tests(ourdir)
|
|
|
82 tff.close()
|
|
|
83 self.tooloutdir = "./tfout"
|
|
|
84 self.repdir = "./TF_run_report"
|
|
|
85 self.testdir = os.path.join(self.tooloutdir, "test-data")
|
|
|
86 if not os.path.exists(self.tooloutdir):
|
|
|
87 os.mkdir(self.tooloutdir)
|
|
|
88 if not os.path.exists(self.testdir):
|
|
|
89 os.mkdir(self.testdir)
|
|
|
90 if not os.path.exists(self.repdir):
|
|
|
91 os.mkdir(self.repdir)
|
|
|
92 self.makeTool()
|
|
|
93 self.moveRunOutputs()
|
|
|
94 self.makeToolTar()
|
|
|
95
|
|
|
96 def call_planemo(self,xmlpath,ourdir):
|
|
|
97 penv = os.environ
|
|
|
98 penv['HOME'] = os.path.join(self.args.galaxy_root,'planemo')
|
|
|
99 newpath = f"{penv['HOME']}:{penv['PATH']}"
|
|
|
100 penv['PATH'] = newpath
|
|
|
101 penv["GALAXY_VIRTUAL_ENV"] = os.path.join(self.args.galaxy_root,'.venv')
|
|
|
102 penv["PIP_CACHE_DIR"] = os.path.join(self.args.galaxy_root,'pipcache')
|
|
|
103 toolfile = os.path.split(xmlpath)[1]
|
|
|
104 tool_name = self.tool_name
|
|
|
105 tool_test_output = f"{tool_name}_planemo_test_report.html"
|
|
|
106 cll = [
|
|
|
107 "planemo",
|
|
|
108 "test",
|
|
|
109 "--no_cleanup",
|
|
|
110 "--test_data",
|
|
|
111 os.path.abspath(ourdir),
|
|
|
112 "--test_output",
|
|
|
113 os.path.abspath(self.tool_test_output),
|
|
|
114 "--galaxy_root",
|
|
|
115 self.args.galaxy_root,
|
|
|
116 "--update_test_data",
|
|
|
117 os.path.abspath(xmlpath),
|
|
|
118 ]
|
|
|
119 print(cll)
|
|
|
120 p = subprocess.run(
|
|
|
121 cll,
|
|
|
122 #capture_output=True,
|
|
|
123 encoding='utf8',
|
|
|
124 cwd = os.path.abspath(self.tool_name),
|
|
|
125 env = penv,
|
|
|
126 shell=False,
|
|
|
127 )
|
|
|
128 return p
|
|
|
129
|
|
|
130 def makeTool(self):
|
|
|
131 """write xmls and input samples into place"""
|
|
|
132 for xreal in self.ourxmls:
|
|
|
133 x = os.path.split(xreal)[1]
|
|
|
134 xout = os.path.join(self.tooloutdir,x)
|
|
|
135 shutil.copyfile(xreal, xout)
|
|
|
136
|
|
|
137 def makeToolTar(self):
|
|
|
138 """move outputs into test-data and prepare the tarball"""
|
|
|
139 excludeme = "_planemo_test_report.html"
|
|
|
140
|
|
|
141 def exclude_function(tarinfo):
|
|
|
142 filename = tarinfo.name
|
|
|
143 return None if filename.endswith(excludeme) else tarinfo
|
|
|
144
|
|
|
145 newtar = 'new_%s_toolshed.gz' % self.tool_name
|
|
|
146 ttf = tarfile.open(newtar, "w:gz")
|
|
|
147 ttf.add(name=self.tool_name,
|
|
|
148 arcname=self.tool_name,
|
|
|
149 filter=exclude_function)
|
|
|
150 ttf.close()
|
|
|
151 shutil.copyfile(newtar, self.new_tool_archive)
|
|
|
152
|
|
|
153 def moveRunOutputs(self):
|
|
|
154 """need to move planemo or run outputs into toolfactory collection"""
|
|
|
155 with os.scandir(self.tooloutdir) as outs:
|
|
|
156 for entry in outs:
|
|
|
157 if not entry.is_file():
|
|
|
158 continue
|
|
|
159 if "." in entry.name:
|
|
|
160 _, ext = os.path.splitext(entry.name)
|
|
|
161 if ext in [".tgz", ".json"]:
|
|
|
162 continue
|
|
|
163 if ext in [".yml", ".xml", ".yaml"]:
|
|
|
164 newname = f"{entry.name.replace('.','_')}.txt"
|
|
|
165 else:
|
|
|
166 newname = entry.name
|
|
|
167 else:
|
|
|
168 newname = f"{entry.name}.txt"
|
|
|
169 dest = os.path.join(self.repdir, newname)
|
|
|
170 src = os.path.join(self.tooloutdir, entry.name)
|
|
|
171 shutil.copyfile(src, dest)
|
|
|
172 with os.scandir('.') as outs:
|
|
|
173 for entry in outs:
|
|
|
174 if not entry.is_file() or entry.name == "conda_activate.log":
|
|
|
175 continue
|
|
|
176 if "." in entry.name:
|
|
|
177 _, ext = os.path.splitext(entry.name)
|
|
|
178 if ext in [".yml", ".xml", ".yaml"]:
|
|
|
179 newname = f"{entry.name.replace('.','_')}.txt"
|
|
|
180 else:
|
|
|
181 newname = entry.name
|
|
|
182 else:
|
|
|
183 newname = f"{entry.name}.txt"
|
|
|
184 dest = os.path.join(self.repdir, newname)
|
|
|
185 src =entry.name
|
|
|
186 shutil.copyfile(src, dest)
|
|
|
187 if True or self.args.include_tests:
|
|
|
188 with os.scandir(self.testdir) as outs:
|
|
|
189 for entry in outs:
|
|
|
190 if (not entry.is_file()) or entry.name.endswith(
|
|
|
191 "_planemo_test_report.html"
|
|
|
192 ):
|
|
|
193 continue
|
|
|
194 if "." in entry.name:
|
|
|
195 _, ext = os.path.splitext(entry.name)
|
|
|
196 if ext in [".tgz", ".json"]:
|
|
|
197 continue
|
|
|
198 if ext in [".yml", ".xml", ".yaml"]:
|
|
|
199 newname = f"{entry.name.replace('.','_')}.txt"
|
|
|
200 else:
|
|
|
201 newname = entry.name
|
|
|
202 else:
|
|
|
203 newname = f"{entry.name}.txt"
|
|
|
204 dest = os.path.join(self.repdir, newname)
|
|
|
205 src = os.path.join(self.testdir, entry.name)
|
|
|
206 shutil.copyfile(src, dest)
|
|
|
207
|
|
|
208
|
|
|
209 def update_tests(self,ourdir):
|
|
|
210 for xmlf in self.ourxmls:
|
|
|
211 capture = self.call_planemo(xmlf,ourdir)
|
|
|
212
|
|
|
213 def main():
|
|
|
214 """
|
|
|
215 This is a Galaxy wrapper.
|
|
|
216 It expects to be called by a special purpose tool.xml
|
|
|
217
|
|
|
218 """
|
|
|
219 parser = argparse.ArgumentParser()
|
|
|
220 a = parser.add_argument
|
|
|
221 a("--in_tool_archive", default=None)
|
|
|
222 a("--new_tested_tool_archive", default=None)
|
|
|
223 a("--galaxy_root", default="/home/ross/gal21/")
|
|
|
224 args = parser.parse_args()
|
|
|
225 print('Hello from',os.getcwd())
|
|
|
226 tt = ToolTester(args=args, in_tool_archive=args.in_tool_archive, new_tool_archive=args.new_tested_tool_archive)
|
|
|
227
|
|
|
228 if __name__ == "__main__":
|
|
|
229 main()
|
|
|
230
|
|
|
231
|
|
|
232 #end raw]]></configfile>
|
|
|
233 </configfiles>
|
|
|
234 <inputs>
|
|
|
235 <param name="new_tool_name" value="" type="hidden"/>
|
|
|
236 <param name="in_tool_archive" type="data" optional="false" label="Select a no_test tarfile to test and update for a toolshed" help="" format="toolshed.gz" multiple="false"/>
|
|
|
237 </inputs>
|
|
|
238 <outputs>
|
|
|
239 <data name="new_tested_tool_archive" format="toolshed.gz" label="${in_tool_archive.name.split('_')[0]}_tested_toolshed.gz" hidden="false"/>
|
|
|
240 <collection name="TF_run_report" type="list" label="${in_tool_archive.name} test Run reports">
|
|
|
241 <discover_datasets pattern="__name_and_ext__" directory="TF_run_report" visible="false"/>
|
|
|
242 </collection>
|
|
|
243 </outputs>
|
|
|
244 <tests>
|
|
|
245 <test>
|
|
|
246 <output name="new_tested_tool_archive" value="new_tested_tool_archive_sample" compare="sim_size" delta_frac="0.5"/>
|
|
|
247 <param name="in_tool_archive" value="in_tool_archive_sample"/>
|
|
|
248 <output_collection name="TF_run_report"/>
|
|
|
249 </test>
|
|
|
250 </tests>
|
|
|
251 <help><![CDATA[
|
|
|
252
|
|
|
253 **What it Does**
|
|
|
254
|
|
|
255 ------
|
|
|
256
|
|
|
257 Script::
|
|
|
258
|
|
|
259 # see https://github.com/fubar2/toolfactory
|
|
|
260 #
|
|
|
261 # copyright ross lazarus (ross stop lazarus at gmail stop com) May 2012
|
|
|
262 #
|
|
|
263 # all rights reserved
|
|
|
264 # Licensed under the LGPL
|
|
|
265 # suggestions for improvement and bug fixes welcome at
|
|
|
266 # https://github.com/fubar2/toolfactory
|
|
|
267
|
|
|
268 import argparse
|
|
|
269 import copy
|
|
|
270 import os
|
|
|
271 import subprocess
|
|
|
272 import shutil
|
|
|
273 import sys
|
|
|
274 import tarfile
|
|
|
275 import tempfile
|
|
|
276 import time
|
|
|
277 import xml.etree.ElementTree as ET
|
|
|
278
|
|
|
279
|
|
|
280 myversion = "V2.2 April 2021"
|
|
|
281 verbose = True
|
|
|
282 debug = True
|
|
|
283 toolFactoryURL = "https://github.com/fubar2/toolfactory"
|
|
|
284
|
|
|
285 def timenow():
|
|
|
286 """return current time as a string"""
|
|
|
287 return time.strftime("%d/%m/%Y %H:%M:%S", time.localtime(time.time()))
|
|
|
288
|
|
|
289 class ToolTester():
|
|
|
290 # uncompress passed tar, run planemo and rebuild a new tarball with tests
|
|
|
291
|
|
|
292 def __init__(self, args=None, in_tool_archive='/galaxy-central/tools/newtool/newtool_toolshed.gz', new_tool_archive=None):
|
|
|
293 self.args = args
|
|
|
294 self.new_tool_archive = new_tool_archive
|
|
|
295 assert tarfile.is_tarfile(in_tool_archive)
|
|
|
296 # this is not going to go well with arbitrary names.
|
|
|
297 tff = tarfile.open(in_tool_archive, "r:*")
|
|
|
298 flist = tff.getnames()
|
|
|
299 ourdir = os.path.commonpath(flist) # eg pyrevpos
|
|
|
300 self.tool_name = ourdir
|
|
|
301 ourxmls = [x for x in flist if x.lower().endswith('.xml') and os.path.split(x)[0] == ourdir]
|
|
|
302 assert len(ourxmls) > 0
|
|
|
303 self.ourxmls = ourxmls # [os.path.join(tool_path,x) for x in ourxmls]
|
|
|
304 res = tff.extractall()
|
|
|
305 self.update_tests(ourdir)
|
|
|
306 tff.close()
|
|
|
307 self.tooloutdir = "./tfout"
|
|
|
308 self.repdir = "./TF_run_report"
|
|
|
309 self.testdir = os.path.join(self.tooloutdir, "test-data")
|
|
|
310 if not os.path.exists(self.tooloutdir):
|
|
|
311 os.mkdir(self.tooloutdir)
|
|
|
312 if not os.path.exists(self.testdir):
|
|
|
313 os.mkdir(self.testdir)
|
|
|
314 if not os.path.exists(self.repdir):
|
|
|
315 os.mkdir(self.repdir)
|
|
|
316 self.makeTool()
|
|
|
317 self.moveRunOutputs()
|
|
|
318 self.makeToolTar()
|
|
|
319
|
|
|
320 def call_planemo(self,xmlpath,ourdir):
|
|
|
321 penv = os.environ
|
|
|
322 penv["PIP_CACHE_DIR"] = os.path.join(self.args.galaxy_root,'pipcache')
|
|
|
323 toolfile = os.path.split(xmlpath)[1]
|
|
|
324 tool_name = self.tool_name
|
|
|
325 tool_test_output = f"{tool_name}_planemo_test_report.html"
|
|
|
326 cll = [
|
|
|
327 "planemo",
|
|
|
328 "test",
|
|
|
329 "--biocontainers",
|
|
|
330 "--test_output",
|
|
|
331 os.path.abspath(tool_test_output),
|
|
|
332 "--galaxy_root",
|
|
|
333 self.args.galaxy_root,
|
|
|
334 "--update_test_data",
|
|
|
335 os.path.abspath(xmlpath),
|
|
|
336 ]
|
|
|
337 print(cll)
|
|
|
338 p = subprocess.run(
|
|
|
339 cll,
|
|
|
340 #capture_output=True,
|
|
|
341 encoding='utf8',
|
|
|
342 env = penv,
|
|
|
343 shell=False,
|
|
|
344 )
|
|
|
345 return p
|
|
|
346
|
|
|
347 def makeTool(self):
|
|
|
348 """write xmls and input samples into place"""
|
|
|
349 for xreal in self.ourxmls:
|
|
|
350 x = os.path.split(xreal)[1]
|
|
|
351 xout = os.path.join(self.tooloutdir,x)
|
|
|
352 shutil.copyfile(xreal, xout)
|
|
|
353
|
|
|
354 def makeToolTar(self):
|
|
|
355 """move outputs into test-data and prepare the tarball"""
|
|
|
356 excludeme = "_planemo_test_report.html"
|
|
|
357
|
|
|
358 def exclude_function(tarinfo):
|
|
|
359 filename = tarinfo.name
|
|
|
360 return None if filename.endswith(excludeme) else tarinfo
|
|
|
361
|
|
|
362 newtar = 'new_%s_toolshed.gz' % self.tool_name
|
|
|
363 ttf = tarfile.open(newtar, "w:gz")
|
|
|
364 ttf.add(name=self.tool_name,
|
|
|
365 arcname=self.tool_name,
|
|
|
366 filter=exclude_function)
|
|
|
367 ttf.close()
|
|
|
368 shutil.copyfile(newtar, self.new_tool_archive)
|
|
|
369
|
|
|
370 def moveRunOutputs(self):
|
|
|
371 """need to move planemo or run outputs into toolfactory collection"""
|
|
|
372 with os.scandir(self.tooloutdir) as outs:
|
|
|
373 for entry in outs:
|
|
|
374 if not entry.is_file():
|
|
|
375 continue
|
|
|
376 if "." in entry.name:
|
|
|
377 _, ext = os.path.splitext(entry.name)
|
|
|
378 if ext in [".tgz", ".json"]:
|
|
|
379 continue
|
|
|
380 if ext in [".yml", ".xml", ".yaml"]:
|
|
|
381 newname = f"{entry.name.replace('.','_')}.txt"
|
|
|
382 else:
|
|
|
383 newname = entry.name
|
|
|
384 else:
|
|
|
385 newname = f"{entry.name}.txt"
|
|
|
386 dest = os.path.join(self.repdir, newname)
|
|
|
387 src = os.path.join(self.tooloutdir, entry.name)
|
|
|
388 shutil.copyfile(src, dest)
|
|
|
389 with os.scandir('.') as outs:
|
|
|
390 for entry in outs:
|
|
|
391 if not entry.is_file():
|
|
|
392 continue
|
|
|
393 if "." in entry.name:
|
|
|
394 _, ext = os.path.splitext(entry.name)
|
|
|
395 if ext in [".yml", ".xml", ".yaml"]:
|
|
|
396 newname = f"{entry.name.replace('.','_')}.txt"
|
|
|
397 else:
|
|
|
398 newname = entry.name
|
|
|
399 else:
|
|
|
400 newname = f"{entry.name}.txt"
|
|
|
401 dest = os.path.join(self.repdir, newname)
|
|
|
402 src =entry.name
|
|
|
403 shutil.copyfile(src, dest)
|
|
|
404 if True or self.args.include_tests:
|
|
|
405 with os.scandir(self.testdir) as outs:
|
|
|
406 for entry in outs:
|
|
|
407 if (not entry.is_file()) or entry.name.endswith(
|
|
|
408 "_planemo_test_report.html"
|
|
|
409 ):
|
|
|
410 continue
|
|
|
411 if "." in entry.name:
|
|
|
412 _, ext = os.path.splitext(entry.name)
|
|
|
413 if ext in [".tgz", ".json"]:
|
|
|
414 continue
|
|
|
415 if ext in [".yml", ".xml", ".yaml"]:
|
|
|
416 newname = f"{entry.name.replace('.','_')}.txt"
|
|
|
417 else:
|
|
|
418 newname = entry.name
|
|
|
419 else:
|
|
|
420 newname = f"{entry.name}.txt"
|
|
|
421 dest = os.path.join(self.repdir, newname)
|
|
|
422 src = os.path.join(self.testdir, entry.name)
|
|
|
423 shutil.copyfile(src, dest)
|
|
|
424
|
|
|
425
|
|
|
426 def update_tests(self,ourdir):
|
|
|
427 for xmlf in self.ourxmls:
|
|
|
428 capture = self.call_planemo(xmlf,ourdir)
|
|
|
429
|
|
|
430 def main():
|
|
|
431 """
|
|
|
432 This is a Galaxy wrapper.
|
|
|
433 It expects to be called by a special purpose tool.xml
|
|
|
434
|
|
|
435 """
|
|
|
436 parser = argparse.ArgumentParser()
|
|
|
437 a = parser.add_argument
|
|
|
438 a("--in_tool_archive", default=None)
|
|
|
439 a("--new_tested_tool_archive", default=None)
|
|
|
440 a("--galaxy_root", default="/home/ross/gal21/")
|
|
|
441 args = parser.parse_args()
|
|
|
442 print('Hello from',os.getcwd())
|
|
|
443 tt = ToolTester(args=args, in_tool_archive=args.in_tool_archive, new_tool_archive=args.new_tested_tool_archive)
|
|
|
444
|
|
|
445 if __name__ == "__main__":
|
|
|
446 main()
|
|
|
447
|
|
|
448
|
|
|
449 ]]></help>
|
|
|
450 <citations>
|
|
|
451 <citation type="doi">10.1093/bioinformatics/bts573</citation>
|
|
|
452 </citations>
|
|
|
453 </tool>
|
|
|
454
|
