Mercurial > repos > fubar > brokenandnotdeletablebyowneroradmin

comparison rgToolFactory.py @ 8:220885b2d7ee

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
End to end test works. Add tests next
author ross lazarus ross.lazarus@gmail.com
date Sat, 02 Jun 2012 20:02:11 +1000
parents 7221619caefa
children e09c76551bed
comparison
equal deleted inserted replaced
7:7221619caefa 8:220885b2d7ee
28 import time 28 import time
29 import tempfile 29 import tempfile
30 import optparse 30 import optparse
31 import tarfile 31 import tarfile
32 import re 32 import re
33 import shutil
34
33 progname = os.path.split(sys.argv[0])[1] 35 progname = os.path.split(sys.argv[0])[1]
34 myversion = 'V000.1 May 2012' 36 myversion = 'V000.1 May 2012'
35 verbose = False 37 verbose = False
36 debug = False 38 debug = False
37 39
38
39 galhtmlprefix = """<?xml version="1.0" encoding="utf-8" ?>
40 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
41 <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
42 <head> <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
43 <meta name="generator" content="Galaxy %s tool output - see http://g2.trac.bx.psu.edu/" />
44 <title></title>
45 <link rel="stylesheet" href="/static/style/base.css" type="text/css" />
46 </head>
47 <body>
48 <div class="document">
49 """
50 galhtmlattr = """<b><a href="http://rgenetics.org">Galaxy Rgenetics Base Script Wrapper based </a> tool output %s run at %s</b><br/>"""
51 galhtmlpostfix = """</div></body></html>\n"""
52 40
53 def timenow(): 41 def timenow():
54 """return current time as a string 42 """return current time as a string
55 """ 43 """
56 return time.strftime('%d/%m/%Y %H:%M:%S', time.localtime(time.time())) 44 return time.strftime('%d/%m/%Y %H:%M:%S', time.localtime(time.time()))
95 __cr____cn__inp = read.table(inf,head=T,rownames=F,sep=__sq__Xt__sq__) 83 __cr____cn__inp = read.table(inf,head=T,rownames=F,sep=__sq__Xt__sq__)
96 __cr____cn__ write.table(inp,outf, quote=FALSE, sep=__dq__Xt__dq__,row.names=F) 84 __cr____cn__ write.table(inp,outf, quote=FALSE, sep=__dq__Xt__dq__,row.names=F)
97 __cr____cn__sessionInfo() 85 __cr____cn__sessionInfo()
98 __cr____cn__ 86 __cr____cn__
99 """ 87 """
100 self.myname = sys.argv[0] # get our name because we write ourselves out as a tool later 88 if opts.output_dir: # simplify for the tool tarball
89 os.chdir(opts.output_dir)
101 self.thumbformat = 'jpg' 90 self.thumbformat = 'jpg'
102 self.opts = opts 91 self.opts = opts
103 self.toolname = re.sub('[^a-zA-Z0-9_]+', '', opts.tool_name) 92 self.toolname = re.sub('[^a-zA-Z0-9_]+', '', opts.tool_name)
93 self.toolid = self.toolname
104 s = open(self.opts.script_path,'r').read() 94 s = open(self.opts.script_path,'r').read()
105 self.script = restore_text(s) 95 self.script = restore_text(s)
106 self.pyfile = self.myname 96 self.myname = sys.argv[0] # get our name because we write ourselves out as a tool later
107 self.xmlfile = '%s.xml' % os.path.splitext(self.pyfile)[0] # punt 97 self.pyfile = self.myname # crude but efficient - the cruft won't hurt much
98 self.xmlfile = '%s.xml' % self.toolname
108 self.sfile = '%s.%s' % (self.toolname,opts.interpreter) 99 self.sfile = '%s.%s' % (self.toolname,opts.interpreter)
109 localscript = open(self.sfile,'w') 100 localscript = open(self.sfile,'w')
110 localscript.write(self.script) 101 localscript.write(self.script)
111 localscript.close() 102 localscript.close()
112 if opts.output_dir or self.opts.makeTool: # may not want these complexities if a simple script 103 if opts.output_dir or self.opts.make_Tool: # may not want these complexities if a simple script
113 self.tlog = os.path.join(opts.output_dir,"%s_runner.log" % self.toolname) 104 self.tlog = os.path.join(opts.output_dir,"%s_runner.log" % self.toolname)
114 artifactpath = os.path.join(opts.output_dir,'%s_run.script' % self.toolname) 105 artifactpath = os.path.join(opts.output_dir,'%s_run.script' % self.toolname)
115 artifact = open(artifactpath,'w') 106 artifact = open(artifactpath,'w')
116 artifact.write(self.script) 107 artifact.write(self.script)
117 artifact.write('\n') 108 artifact.write('\n')
121 a = self.cl.append 112 a = self.cl.append
122 a(opts.interpreter) 113 a(opts.interpreter)
123 a('-') # use stdin 114 a('-') # use stdin
124 a(opts.input_tab) 115 a(opts.input_tab)
125 a(opts.output_tab) 116 a(opts.output_tab)
117 self.outFormats = 'tabular' # TODO make this an option at tool generation time
118 self.inputFormats = 'tabular' # TODO make this an option at tool generation time
119
120
121 def makeXML(self):
122 """
123 Create a Galaxy xml tool wrapper for the new script as a string to write out
124 """
125 newXML="""<tool id="%(toolid)s" name="%(toolname)s" version="0.01">
126 %(tooldesc)s
127 %(command)s
128 <inputs>
129 %(inputs)s
130 </inputs>
131 <outputs>
132 %(outputs)s
133 </outputs>
134 <help>
135 %(help)s
136 </help>
137 <configfiles>
138 <configfile name="runMe">
139 %(script)s
140 </configfile>
141 </configfiles>
142 </tool>""" # needs a dict with toolname, toolid, interpreter, scriptname, command, inputs as a multi line string ready to write, outputs ditto, help ditto
143
144 newCommand="""<command interpreter="python">
145 %(toolname)s.py --script_path "$runMe" --interpreter "%(interpreter)s"
146 --tool_name "%(toolname)s" %(command_inputs)s %(command_outputs)s
147 </command>""" # may NOT be an input or htmlout
148
149 xdict = {}
150 xdict['script'] = self.script # we pass this as a configfile because it's less painful that galaxy_tool_data_dir
151 # embed script in tool - remove dependence on something else outside in the wilds
152 if self.opts.help_text:
153 h = open(self.opts.help_text,'r').read()
154 xdict['help'] = restore_text(h)
155 else:
156 xdict['help'] = 'Please ask the tool author for help as none was supplied at tool generation'
157 if self.opts.tool_desc:
158 xdict['tooldesc'] = '<description>%s</description>' % self.opts.tool_desc
159 else:
160 xdict['tooldesc'] = ''
161 xdict['command_outputs'] = '' # will probably be some!
162 xdict['outputs'] = '' # will probably be some!
163 if self.opts.input_tab:
164 xdict['command_inputs'] = '--input_tab "$input1"'
165 xdict['inputs'] = '<param name="input1" type="data" format="%s" label="Select a suitable input file from your history"/>' % self.inputFormats
166 else:
167 xdict['command_inputs'] = '' # assume no input - eg a random data generator
168 xdict['inputs'] = ''
169 xdict['inputs'] += '<param name="job_name" type="text" label="Supply a name for the outputs to remind you what they contain" value="%s"/>\n' % self.toolname
170 xdict['toolname'] = self.toolname
171 xdict['toolid'] = self.toolid
172 xdict['interpreter'] = self.opts.interpreter
173 xdict['scriptname'] = self.sfile
174 if self.opts.make_HTML:
175 xdict['command_outputs'] += '--output_dir "$html_file.files_path" --output_html "$html_file"'
176 xdict['outputs'] += '<data format="html" name="html_file" label="${job_name}.html"/>\n'
177 if self.opts.output_tab:
178 xdict['command_outputs'] += '--output_tab "$tab_file"'
179 xdict['outputs'] += '<data format="%s" name="tab_file" label="${job_name}"/>\n' % self.outFormats
180 xdict['command'] = newCommand % xdict
181 xmls = newXML % xdict
182 xf = open(self.xmlfile,'w')
183 xf.write(xmls)
184 xf.write('\n')
185 xf.close()
186 # ready for the tarball
187
126 188
127 def makeTooltar(self): 189 def makeTooltar(self):
128 """ 190 """
129 a tool is a gz tarball with eg 191 a tool is a gz tarball with eg
130 /toolname/tool.xml /toolname/tool.py /toolname/test-data/test1_in.foo ... 192 /toolname/tool.xml /toolname/tool.py /toolname/test-data/test1_in.foo ...
131 """ 193 """
132 retval = self.run() 194 retval = self.run()
133 if retval: 195 if retval:
134 print >> sys.stderr,'## Run failed. Cannot build yet. Please fix and retry' 196 print >> sys.stderr,'## Run failed. Cannot build yet. Please fix and retry'
135 sys.exit(1) 197 sys.exit(1)
136 tarpath = os.path.join(self.opts.output_dir,"%s.gz" % self.toolname) 198 self.makeXML()
199 tdir = self.toolname
200 os.mkdir(tdir)
201 shutil.copyfile(self.xmlfile,os.path.join(tdir,self.xmlfile))
202 shutil.copyfile(self.pyfile,os.path.join(tdir,'%s.py' % self.toolname))
203 shutil.copyfile(self.sfile,os.path.join(tdir,self.sfile))
204 tarpath = "%s.gz" % self.toolname
137 tar = tarfile.open(tarpath, "w:gz") 205 tar = tarfile.open(tarpath, "w:gz")
138 tar.add(self.xmlfile,arcname='%s.xml' % self.toolname) 206 tar.add(tdir,arcname=self.toolname)
139 tar.add(self.pyfile,arcname=os.path.basename(self.pyfile))
140 tar.add(self.sfile,arcname=self.sfile)
141 tar.close() 207 tar.close()
142 self.makeHtml() 208 shutil.rmtree(tdir)
209 self.makeHtml() # call this to return the new gzip inside the autogenerated html file
210 ## TODO: replace with optional direct upload to local toolshed?
143 return retval 211 return retval
144 212
145 def compressPDF(self,inpdf=None,thumbformat='png'): 213 def compressPDF(self,inpdf=None,thumbformat='png'):
146 """need absolute path to pdf 214 """need absolute path to pdf
147 """ 215 """
179 elif n > 0: 247 elif n > 0:
180 size = ' (%d B)' % (int(n)) 248 size = ' (%d B)' % (int(n))
181 return size 249 return size
182 250
183 def makeHtml(self): 251 def makeHtml(self):
184 """ 252 """ Create an HTML file content to list all the artefacts found in the output_dir
185 """ 253 """
186 flist = os.listdir(self.opts.output_dir) 254
187 flist = [x for x in flist if x <> 'Rplots.pdf'] 255 galhtmlprefix = """<?xml version="1.0" encoding="utf-8" ?>
188 flist.sort() 256 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
189 html = [galhtmlprefix % progname,] 257 <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
190 html.append('<h2>Galaxy %s outputs run at %s</h2><br/>\n' % (self.toolname,timenow())) 258 <head> <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
191 fhtml = [] 259 <meta name="generator" content="Galaxy %s tool output - see http://g2.trac.bx.psu.edu/" />
192 if len(flist) > 0: 260 <title></title>
193 html.append('<table cellpadding="3" cellspacing="3">\n') 261 <link rel="stylesheet" href="/static/style/base.css" type="text/css" />
194 for fname in flist: 262 </head>
263 <body>
264 <div class="document">
265 """
266 galhtmlattr = """<hr/><b><a href="http://rgenetics.org">Galaxy Tool Factory Script Wrapper</a> tool output %s run at %s</b><br/>"""
267 galhtmlpostfix = """</div></body></html>\n"""
268
269 flist = os.listdir(self.opts.output_dir)
270 flist = [x for x in flist if x <> 'Rplots.pdf']
271 flist.sort()
272 html = [galhtmlprefix % progname,]
273 html.append('<h2>Galaxy %s outputs run at %s</h2><br/>\n' % (self.toolname,timenow()))
274 fhtml = []
275 if len(flist) > 0:
276 html.append('<table cellpadding="3" cellspacing="3">\n')
277 for fname in flist:
195 dname,e = os.path.splitext(fname) 278 dname,e = os.path.splitext(fname)
196 sfsize = self.getfSize(fname,self.opts.output_dir) 279 sfsize = self.getfSize(fname,self.opts.output_dir)
197 if e.lower() == '.pdf' : # compress and make a thumbnail 280 if e.lower() == '.pdf' : # compress and make a thumbnail
198 thumb = '%s.%s' % (dname,self.thumbformat) 281 thumb = '%s.%s' % (dname,self.thumbformat)
199 pdff = os.path.join(self.opts.output_dir,fname) 282 pdff = os.path.join(self.opts.output_dir,fname)
202 s= '<tr><td><a href="%s"><img src="%s" title="Click to download a PDF of %s" hspace="10" width="600"></a></td></tr>\n' % (fname,thumb,fname) 285 s= '<tr><td><a href="%s"><img src="%s" title="Click to download a PDF of %s" hspace="10" width="600"></a></td></tr>\n' % (fname,thumb,fname)
203 html.append(s) 286 html.append(s)
204 fhtml.append('<li><a href="%s">%s %s</a></li>' % (fname,fname,sfsize)) 287 fhtml.append('<li><a href="%s">%s %s</a></li>' % (fname,fname,sfsize))
205 else: 288 else:
206 fhtml.append('<li><a href="%s">%s %s</a></li>' % (fname,fname,sfsize)) 289 fhtml.append('<li><a href="%s">%s %s</a></li>' % (fname,fname,sfsize))
207 html.append('</table>\n') 290 html.append('</table>\n')
208 if len(fhtml) > 0: 291 if len(fhtml) > 0:
209 fhtml.insert(0,'<ul>') 292 fhtml.insert(0,'<ul>')
210 fhtml.append('</ul>') 293 fhtml.append('</ul>')
211 html += fhtml # add all non-pdf files to the end of the display 294 html += fhtml # add all non-pdf files to the end of the display
212 else: 295 else:
213 html.append('<h2>### Error - %s returned no files - please confirm that parameters are sane</h1>' % self.opts.interpreter) 296 html.append('<h2>### Error - %s returned no files - please confirm that parameters are sane</h1>' % self.opts.interpreter)
214 html.append('<h3>%s log follows below</h3><hr><pre>\n' % self.opts.interpreter) 297 html.append('<h3>%s log follows below</h3><hr/><pre>\n' % self.opts.interpreter)
215 rlog = open(self.tlog,'r').readlines() 298 rlog = open(self.tlog,'r').readlines()
216 html += rlog 299 html += rlog
217 html.append('%s CL = %s</br>\n' % (self.toolname,' '.join(sys.argv))) 300 html.append('%s CL = %s<br/>\n' % (self.toolname,' '.join(sys.argv)))
218 html.append('CL = %s</br>\n' % (' '.join(self.cl))) 301 html.append('</pre>\n')
219 html.append('</pre>\n') 302 html.append(galhtmlattr % (progname,timenow()))
220 html.append(galhtmlattr % (progname,timenow())) 303 html.append(galhtmlpostfix)
221 html.append(galhtmlpostfix) 304 htmlf = file(self.opts.output_html,'w')
222 htmlf = file(self.opts.output_html,'w') 305 htmlf.write('\n'.join(html))
223 htmlf.write('\n'.join(html)) 306 htmlf.write('\n')
224 htmlf.write('\n') 307 htmlf.close()
225 htmlf.close() 308 self.html = html
226 self.html = html
227 309
228 310
229 def run(self): 311 def run(self):
230 """ 312 """
231 """ 313 """
232 if self.opts.output_dir or self.opts.makeTool: 314 if self.opts.output_dir or self.opts.make_Tool:
233 sto = open(self.tlog,'w') 315 sto = open(self.tlog,'w')
234 p = subprocess.Popen(' '.join(self.cl),shell=True,stdout=sto,stderr=sto,stdin=subprocess.PIPE,cwd=self.opts.output_dir) 316 p = subprocess.Popen(' '.join(self.cl),shell=True,stdout=sto,stderr=sto,stdin=subprocess.PIPE,cwd=self.opts.output_dir)
235 else: 317 else:
236 p = subprocess.Popen(' '.join(self.cl),shell=True,stdin=subprocess.PIPE) 318 p = subprocess.Popen(' '.join(self.cl),shell=True,stdin=subprocess.PIPE)
237 p.stdin.write(self.script) 319 p.stdin.write(self.script)
238 p.stdin.close() 320 p.stdin.close()
239 retval = p.wait() 321 retval = p.wait()
240 if self.opts.output_dir or self.opts.makeTool: 322 if self.opts.make_HTML or self.opts.make_Tool:
241 sto.close() 323 sto.close()
242 self.makeHtml() 324 self.makeHtml()
243 return retval 325 return retval
244 326
245 327
254 a('--script_path',default=None) 336 a('--script_path',default=None)
255 a('--tool_name',default=None) 337 a('--tool_name',default=None)
256 a('--interpreter',default=None) 338 a('--interpreter',default=None)
257 a('--output_dir',default=None) 339 a('--output_dir',default=None)
258 a('--output_html',default=None) 340 a('--output_html',default=None)
259 a('--input_tab',default='NONE') 341 a('--input_tab',default=None)
260 a('--output_tab',default='NONE') 342 a('--output_tab',default=None)
261 a('--user_email',default=None) 343 a('--user_email',default=None)
262 a('--bad_user',default=None) 344 a('--bad_user',default=None)
263 a('--makeTool',default=None) 345 a('--make_Tool',default=None)
346 a('--make_HTML',default=None)
347 a('--help_text',default=None)
348 a('--tool_desc',default=None)
264 opts, args = op.parse_args() 349 opts, args = op.parse_args()
265 assert not opts.bad_user,'%s is NOT authorized to use this tool. Please ask your friendly admin' % opts.bad_user 350 assert not opts.bad_user,'%s is NOT authorized to use this tool. Please ask your friendly admin' % opts.bad_user
266 assert opts.tool_name,'## Tool Factory expects a tool name - eg --tool_name=DESeq' 351 assert opts.tool_name,'## Tool Factory expects a tool name - eg --tool_name=DESeq'
267 assert opts.interpreter,'## Tool Factory wrapper expects an interpreter - eg --interpreter=Rscript' 352 assert opts.interpreter,'## Tool Factory wrapper expects an interpreter - eg --interpreter=Rscript'
268 assert os.path.isfile(opts.script_path),'## Tool Factory wrapper expects a script path - eg --script_path=foo.R' 353 assert os.path.isfile(opts.script_path),'## Tool Factory wrapper expects a script path - eg --script_path=foo.R'
270 try: 355 try:
271 os.makedirs(opts.output_dir) 356 os.makedirs(opts.output_dir)
272 except: 357 except:
273 pass 358 pass
274 r = ScriptRunner(opts) 359 r = ScriptRunner(opts)
275 if opts.makeTool: 360 if opts.make_Tool:
276 retcode = r.makeTooltar() 361 retcode = r.makeTooltar()
277 else: 362 else:
278 retcode = r.run() 363 retcode = r.run()
279 if retcode: 364 if retcode:
280 sys.exit(retcode) # indicate failure to job runner 365 sys.exit(retcode) # indicate failure to job runner