Mercurial > repos > saketkc > condel
view tools/condel/condel_web.py @ 0:742c7ee37f8b
push
author | Saket Choudhary <saketkc@gmail.com> |
---|---|
date | Wed, 20 Nov 2013 00:48:30 +0530 |
parents | |
children |
line wrap: on
line source
import sys sys.path.insert(0, '/home/saket/requests-new-urllib3-api/requests/packages/') sys.path.insert(0, '/home/saket/requests-new-urllib3-api') import requests import pycurl import os from os.path import getsize import argparse import sys import cStringIO from functools import wraps import tempfile, shutil,time __url__="http://bg.upf.edu/condel/taskService" def stop_err( msg ): sys.stderr.write( '%s\n' % msg ) sys.exit() def retry(ExceptionToCheck, tries=10, delay=3, backoff=2, logger=None): """Retry calling the decorated function using an exponential backoff. http://www.saltycrane.com/blog/2009/11/trying-out-retry-decorator-python/ original from: http://wiki.python.org/moin/PythonDecoratorLibrary#Retry :param ExceptionToCheck: the exception to check. may be a tuple of exceptions to check :type ExceptionToCheck: Exception or tuple :param tries: number of times to try (not retry) before giving up :type tries: int :param delay: initial delay between retries in seconds :type delay: int :param backoff: backoff multiplier e.g. value of 2 will double the delay each retry :type backoff: int :param logger: logger to use. If None, print :type logger: logging.Logger instance """ def deco_retry(f): @wraps(f) def f_retry(*args, **kwargs): mtries, mdelay = tries, delay while mtries > 1: try: return f(*args, **kwargs) except ExceptionToCheck, e: #msg = "%s, Retrying in %d seconds..." % (str(e), mdelay) msg = "Retrying in %d seconds..." % (mdelay) if logger: logger.warning(msg) else: #print msg pass time.sleep(mdelay) mtries -= 1 mdelay *= backoff return f(*args, **kwargs) return f_retry # true decorator return deco_retry class TransficUploader: def __init__(self): self.c = pycurl.Curl() self.c.setopt(pycurl.URL, __url__) self.c.setopt(pycurl.UPLOAD, 1) #c.setopt(pycurl.USERPWD, 'saket.kumar:whatsinaname.') self.c.setopt(pycurl.PROXY, 'http://saket.kumar:whatsinaname.@netmon.iitb.ac.in:80/') #c.setopt(pycurl.PROXYPORT, 80) #c.setopt(pycurl.PROXYTYPE_HTTP,1) #self.c.setopt(pycurl.VERBOSE, 1) self.c.setopt(pycurl.HTTPHEADER, ['Expect:']) #c.setopt(pycurl.HTTPPROXYTUNNEL, 1) self.c.setopt(pycurl.UPLOAD, 1) #c.perform() self.c.setopt(pycurl.NOPROGRESS, 1); #curl_easy_setopt(hnd, CURLOPT_UPLOAD, 1L); self.c.setopt(pycurl.USERAGENT, "curl/7.27.0"); #curl_easy_setopt(hnd, CURLOPT_HTTPHEADER, slist1); #curl_easy_setopt(hnd, CURLOPT_MAXREDIRS, 50L); self.c.setopt(pycurl.SSL_VERIFYPEER, 1); self.c.setopt(pycurl.CUSTOMREQUEST, "PUT") #curl_easy_setopt(hnd, CURLOPT_VERBOSE, 1L); self.c.setopt(pycurl.TCP_NODELAY, 1); self.buf = cStringIO.StringIO() self.c.setopt(self.c.WRITEFUNCTION, self.buf.write) def upload_file(self,filepath): f = open(filepath) self.c.setopt(pycurl.INFILE, f) self.c.setopt(pycurl.INFILESIZE, getsize(filepath)) def run(self): self.c.perform() def get_url(self): return self.buf.getvalue().strip() @retry(requests.exceptions.HTTPError) def result_exists(self,url ): #url="http://www.cravat.us/results/%s/%s.zip" %(job_id,job_id) download_request = requests.request("GET", url) if download_request.status_code==404 or download_request==500: raise requests.HTTPError() else: return url @retry(requests.exceptions.HTTPError) def download_result(self, url,outpath): tmp_dir = tempfile.mkdtemp() r = requests.get( url, stream=True ) if r.status_code == 500: raise requests.HTTPError() else: path = os.path.join( tmp_dir,"results.csv") with open(path, 'wb') as f: for chunk in r.iter_content(128): f.write(chunk) shutil.move(path,outpath) shutil.rmtree(tmp_dir) def main(params): parser = argparse.ArgumentParser() parser.add_argument("--input",type=str,required=True) parser.add_argument("--output",type=str,required=True) args = parser.parse_args(params) uploader = TransficUploader(); uploader.upload_file(args.input) uploader.run() url = uploader.get_url() url = uploader.result_exists(url) download = uploader.download_result(url,args.output) if __name__=="__main__": main(sys.argv[1:])