Mercurial > repos > fubar > brokenandnotdeletablebyowneroradmin
comparison rgToolFactory.py @ 7:7221619caefa
Updated name and added crude gzip generator for toolshed
TODO: add tests and new XML tool descriptor as soon as Greg has it nailed down.
| author | ross lazarus ross.lazarus@gmail.com |
|---|---|
| date | Sat, 02 Jun 2012 10:43:08 +1000 |
| parents | |
| children | 220885b2d7ee |
comparison
equal
deleted
inserted
replaced
| 6:78044a3d4a21 | 7:7221619caefa |
|---|---|
| 1 # rgDynamicScriptWrapper.py | |
| 2 # derived from | |
| 3 # rgBaseScriptWrapper.py | |
| 4 # to run some user supplied code | |
| 5 # extremely dangerous | |
| 6 # trusted users only - private site only | |
| 7 # a list in the xml is searched - only users in the list can run this tool. | |
| 8 # | |
| 9 # copyright ross lazarus (ross.lazarus@gmail.com) May 2012 | |
| 10 # | |
| 11 # all rights reserved | |
| 12 # Licensed under the LGPL for your pleasure | |
| 13 # Derived from rgDGE.py in May 2012 | |
| 14 # generalized to run required interpreter | |
| 15 # to make your own tools based on a given script and interpreter such as perl or python | |
| 16 # clone this and the corresponding xml wrapper | |
| 17 # replace the parameters/inputs/outputs and the configfile contents with your script | |
| 18 # Use the $foo syntax to place your parameter values inside the script to assign them - at run time, the script will be used as a template | |
| 19 # and returned as part of the output to the user - with the right values for all the parameters. | |
| 20 # Note that this assumes you want all the outputs arranged as a single Html file output | |
| 21 # after this generic script runner runs your script with the specified interpreter, | |
| 22 # it will collect all output files into the specified output_html, making thumbnails for all the pdfs it finds and making links for all the other files. | |
| 23 | |
| 24 import sys | |
| 25 import shutil | |
| 26 import subprocess | |
| 27 import os | |
| 28 import time | |
| 29 import tempfile | |
| 30 import optparse | |
| 31 import tarfile | |
| 32 import re | |
| 33 progname = os.path.split(sys.argv[0])[1] | |
| 34 myversion = 'V000.1 May 2012' | |
| 35 verbose = False | |
| 36 debug = False | |
| 37 | |
| 38 | |
| 39 galhtmlprefix = """<?xml version="1.0" encoding="utf-8" ?> | |
| 40 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> | |
| 41 <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en"> | |
| 42 <head> <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> | |
| 43 <meta name="generator" content="Galaxy %s tool output - see http://g2.trac.bx.psu.edu/" /> | |
| 44 <title></title> | |
| 45 <link rel="stylesheet" href="/static/style/base.css" type="text/css" /> | |
| 46 </head> | |
| 47 <body> | |
| 48 <div class="document"> | |
| 49 """ | |
| 50 galhtmlattr = """<b><a href="http://rgenetics.org">Galaxy Rgenetics Base Script Wrapper based </a> tool output %s run at %s</b><br/>""" | |
| 51 galhtmlpostfix = """</div></body></html>\n""" | |
| 52 | |
| 53 def timenow(): | |
| 54 """return current time as a string | |
| 55 """ | |
| 56 return time.strftime('%d/%m/%Y %H:%M:%S', time.localtime(time.time())) | |
| 57 # characters that are allowed but need to be escaped | |
| 58 mapped_chars = { '>' :'__gt__', | |
| 59 '<' :'__lt__', | |
| 60 "'" :'__sq__', | |
| 61 '"' :'__dq__', | |
| 62 '{' :'__oc__', | |
| 63 '}' :'__cc__', | |
| 64 '@' : '__at__', | |
| 65 '\n' : '__cn__', | |
| 66 '\r' : '__cr__', | |
| 67 '\t' : '__tc__', | |
| 68 '#' : '__pd__', | |
| 69 '[' :'__ob__', | |
| 70 ']' :'__cb__', | |
| 71 '\t' : 'Xt', | |
| 72 'systemCallsAreNotAllowed' : 'system' | |
| 73 } | |
| 74 | |
| 75 def restore_text(text): | |
| 76 """Restores sanitized text""" | |
| 77 if not text: | |
| 78 return text | |
| 79 for key, value in mapped_chars.items(): | |
| 80 text = text.replace(value, key) | |
| 81 return text | |
| 82 | |
| 83 class ScriptRunner: | |
| 84 """class is a wrapper for an arbitrary script | |
| 85 """ | |
| 86 | |
| 87 def __init__(self,opts=None): | |
| 88 """ | |
| 89 run the script | |
| 90 cheetah/galaxy will provide an escaped string so | |
| 91 __pd__ your script goes here | |
| 92 __cr____cn__ourargs __lt__- commandArgs(TRUE) | |
| 93 __cr____cn__inf = ourargs[1] | |
| 94 __cr____cn__outf = ourargs[2] | |
| 95 __cr____cn__inp = read.table(inf,head=T,rownames=F,sep=__sq__Xt__sq__) | |
| 96 __cr____cn__ write.table(inp,outf, quote=FALSE, sep=__dq__Xt__dq__,row.names=F) | |
| 97 __cr____cn__sessionInfo() | |
| 98 __cr____cn__ | |
| 99 """ | |
| 100 self.myname = sys.argv[0] # get our name because we write ourselves out as a tool later | |
| 101 self.thumbformat = 'jpg' | |
| 102 self.opts = opts | |
| 103 self.toolname = re.sub('[^a-zA-Z0-9_]+', '', opts.tool_name) | |
| 104 s = open(self.opts.script_path,'r').read() | |
| 105 self.script = restore_text(s) | |
| 106 self.pyfile = self.myname | |
| 107 self.xmlfile = '%s.xml' % os.path.splitext(self.pyfile)[0] # punt | |
| 108 self.sfile = '%s.%s' % (self.toolname,opts.interpreter) | |
| 109 localscript = open(self.sfile,'w') | |
| 110 localscript.write(self.script) | |
| 111 localscript.close() | |
| 112 if opts.output_dir or self.opts.makeTool: # may not want these complexities if a simple script | |
| 113 self.tlog = os.path.join(opts.output_dir,"%s_runner.log" % self.toolname) | |
| 114 artifactpath = os.path.join(opts.output_dir,'%s_run.script' % self.toolname) | |
| 115 artifact = open(artifactpath,'w') | |
| 116 artifact.write(self.script) | |
| 117 artifact.write('\n') | |
| 118 artifact.close() | |
| 119 self.cl = [] | |
| 120 self.html = [] | |
| 121 a = self.cl.append | |
| 122 a(opts.interpreter) | |
| 123 a('-') # use stdin | |
| 124 a(opts.input_tab) | |
| 125 a(opts.output_tab) | |
| 126 | |
| 127 def makeTooltar(self): | |
| 128 """ | |
| 129 a tool is a gz tarball with eg | |
| 130 /toolname/tool.xml /toolname/tool.py /toolname/test-data/test1_in.foo ... | |
| 131 """ | |
| 132 retval = self.run() | |
| 133 if retval: | |
| 134 print >> sys.stderr,'## Run failed. Cannot build yet. Please fix and retry' | |
| 135 sys.exit(1) | |
| 136 tarpath = os.path.join(self.opts.output_dir,"%s.gz" % self.toolname) | |
| 137 tar = tarfile.open(tarpath, "w:gz") | |
| 138 tar.add(self.xmlfile,arcname='%s.xml' % self.toolname) | |
| 139 tar.add(self.pyfile,arcname=os.path.basename(self.pyfile)) | |
| 140 tar.add(self.sfile,arcname=self.sfile) | |
| 141 tar.close() | |
| 142 self.makeHtml() | |
| 143 return retval | |
| 144 | |
| 145 def compressPDF(self,inpdf=None,thumbformat='png'): | |
| 146 """need absolute path to pdf | |
| 147 """ | |
| 148 assert os.path.isfile(inpdf), "## Input %s supplied to %s compressPDF not found" % (inpdf,self.myName) | |
| 149 hf,hlog = tempfile.mkstemp(suffix="%s.log" % self.toolname) | |
| 150 sto = open(hlog,'w') | |
| 151 outpdf = '%s_compressed' % inpdf | |
| 152 cl = ["gs", "-sDEVICE=pdfwrite", "-dNOPAUSE", "-dBATCH", "-sOutputFile=%s" % outpdf,inpdf] | |
| 153 x = subprocess.Popen(cl,stdout=sto,stderr=sto,cwd=self.opts.output_dir) | |
| 154 retval1 = x.wait() | |
| 155 if retval1 == 0: | |
| 156 os.unlink(inpdf) | |
| 157 shutil.move(outpdf,inpdf) | |
| 158 outpng = '%s.%s' % (os.path.splitext(inpdf)[0],thumbformat) | |
| 159 cl2 = ['convert', inpdf, outpng] | |
| 160 x = subprocess.Popen(cl2,stdout=sto,stderr=sto,cwd=self.opts.output_dir) | |
| 161 retval2 = x.wait() | |
| 162 sto.close() | |
| 163 retval = retval1 or retval2 | |
| 164 return retval | |
| 165 | |
| 166 | |
| 167 def getfSize(self,fpath,outpath): | |
| 168 """ | |
| 169 format a nice file size string | |
| 170 """ | |
| 171 size = '' | |
| 172 fp = os.path.join(outpath,fpath) | |
| 173 if os.path.isfile(fp): | |
| 174 n = float(os.path.getsize(fp)) | |
| 175 if n > 2**20: | |
| 176 size = ' (%1.1f MB)' % (n/2**20) | |
| 177 elif n > 2**10: | |
| 178 size = ' (%1.1f KB)' % (n/2**10) | |
| 179 elif n > 0: | |
| 180 size = ' (%d B)' % (int(n)) | |
| 181 return size | |
| 182 | |
| 183 def makeHtml(self): | |
| 184 """ | |
| 185 """ | |
| 186 flist = os.listdir(self.opts.output_dir) | |
| 187 flist = [x for x in flist if x <> 'Rplots.pdf'] | |
| 188 flist.sort() | |
| 189 html = [galhtmlprefix % progname,] | |
| 190 html.append('<h2>Galaxy %s outputs run at %s</h2><br/>\n' % (self.toolname,timenow())) | |
| 191 fhtml = [] | |
| 192 if len(flist) > 0: | |
| 193 html.append('<table cellpadding="3" cellspacing="3">\n') | |
| 194 for fname in flist: | |
| 195 dname,e = os.path.splitext(fname) | |
| 196 sfsize = self.getfSize(fname,self.opts.output_dir) | |
| 197 if e.lower() == '.pdf' : # compress and make a thumbnail | |
| 198 thumb = '%s.%s' % (dname,self.thumbformat) | |
| 199 pdff = os.path.join(self.opts.output_dir,fname) | |
| 200 retval = self.compressPDF(inpdf=pdff,thumbformat=self.thumbformat) | |
| 201 if retval == 0: | |
| 202 s= '<tr><td><a href="%s"><img src="%s" title="Click to download a PDF of %s" hspace="10" width="600"></a></td></tr>\n' % (fname,thumb,fname) | |
| 203 html.append(s) | |
| 204 fhtml.append('<li><a href="%s">%s %s</a></li>' % (fname,fname,sfsize)) | |
| 205 else: | |
| 206 fhtml.append('<li><a href="%s">%s %s</a></li>' % (fname,fname,sfsize)) | |
| 207 html.append('</table>\n') | |
| 208 if len(fhtml) > 0: | |
| 209 fhtml.insert(0,'<ul>') | |
| 210 fhtml.append('</ul>') | |
| 211 html += fhtml # add all non-pdf files to the end of the display | |
| 212 else: | |
| 213 html.append('<h2>### Error - %s returned no files - please confirm that parameters are sane</h1>' % self.opts.interpreter) | |
| 214 html.append('<h3>%s log follows below</h3><hr><pre>\n' % self.opts.interpreter) | |
| 215 rlog = open(self.tlog,'r').readlines() | |
| 216 html += rlog | |
| 217 html.append('%s CL = %s</br>\n' % (self.toolname,' '.join(sys.argv))) | |
| 218 html.append('CL = %s</br>\n' % (' '.join(self.cl))) | |
| 219 html.append('</pre>\n') | |
| 220 html.append(galhtmlattr % (progname,timenow())) | |
| 221 html.append(galhtmlpostfix) | |
| 222 htmlf = file(self.opts.output_html,'w') | |
| 223 htmlf.write('\n'.join(html)) | |
| 224 htmlf.write('\n') | |
| 225 htmlf.close() | |
| 226 self.html = html | |
| 227 | |
| 228 | |
| 229 def run(self): | |
| 230 """ | |
| 231 """ | |
| 232 if self.opts.output_dir or self.opts.makeTool: | |
| 233 sto = open(self.tlog,'w') | |
| 234 p = subprocess.Popen(' '.join(self.cl),shell=True,stdout=sto,stderr=sto,stdin=subprocess.PIPE,cwd=self.opts.output_dir) | |
| 235 else: | |
| 236 p = subprocess.Popen(' '.join(self.cl),shell=True,stdin=subprocess.PIPE) | |
| 237 p.stdin.write(self.script) | |
| 238 p.stdin.close() | |
| 239 retval = p.wait() | |
| 240 if self.opts.output_dir or self.opts.makeTool: | |
| 241 sto.close() | |
| 242 self.makeHtml() | |
| 243 return retval | |
| 244 | |
| 245 | |
| 246 def main(): | |
| 247 u = """ | |
| 248 This is a Galaxy wrapper. It expects to be called by a special purpose tool.xml as: | |
| 249 <command interpreter="python">rgBaseScriptWrapper.py --script_path "$scriptPath" --tool_name "foo" --interpreter "Rscript" | |
| 250 </command> | |
| 251 """ | |
| 252 op = optparse.OptionParser() | |
| 253 a = op.add_option | |
| 254 a('--script_path',default=None) | |
| 255 a('--tool_name',default=None) | |
| 256 a('--interpreter',default=None) | |
| 257 a('--output_dir',default=None) | |
| 258 a('--output_html',default=None) | |
| 259 a('--input_tab',default='NONE') | |
| 260 a('--output_tab',default='NONE') | |
| 261 a('--user_email',default=None) | |
| 262 a('--bad_user',default=None) | |
| 263 a('--makeTool',default=None) | |
| 264 opts, args = op.parse_args() | |
| 265 assert not opts.bad_user,'%s is NOT authorized to use this tool. Please ask your friendly admin' % opts.bad_user | |
| 266 assert opts.tool_name,'## Tool Factory expects a tool name - eg --tool_name=DESeq' | |
| 267 assert opts.interpreter,'## Tool Factory wrapper expects an interpreter - eg --interpreter=Rscript' | |
| 268 assert os.path.isfile(opts.script_path),'## Tool Factory wrapper expects a script path - eg --script_path=foo.R' | |
| 269 if opts.output_dir: | |
| 270 try: | |
| 271 os.makedirs(opts.output_dir) | |
| 272 except: | |
| 273 pass | |
| 274 r = ScriptRunner(opts) | |
| 275 if opts.makeTool: | |
| 276 retcode = r.makeTooltar() | |
| 277 else: | |
| 278 retcode = r.run() | |
| 279 if retcode: | |
| 280 sys.exit(retcode) # indicate failure to job runner | |
| 281 | |
| 282 | |
| 283 if __name__ == "__main__": | |
| 284 main() | |
| 285 | |
| 286 |