diff tools/condel/condel_web.py @ 0:742c7ee37f8b

push

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tools/condel/condel_web.py	Wed Nov 20 00:48:30 2013 +0530
@@ -0,0 +1,140 @@
+import sys
+sys.path.insert(0, '/home/saket/requests-new-urllib3-api/requests/packages/')
+sys.path.insert(0, '/home/saket/requests-new-urllib3-api')
+
+
+import requests
+import pycurl
+import os
+from os.path import getsize
+import argparse
+import sys
+import cStringIO
+from functools import wraps
+import tempfile, shutil,time
+
+__url__="http://bg.upf.edu/condel/taskService"
+def stop_err( msg ):
+    sys.stderr.write( '%s\n' % msg )
+    sys.exit()
+
+def retry(ExceptionToCheck, tries=10, delay=3, backoff=2, logger=None):
+    """Retry calling the decorated function using an exponential backoff.
+
+    http://www.saltycrane.com/blog/2009/11/trying-out-retry-decorator-python/
+    original from: http://wiki.python.org/moin/PythonDecoratorLibrary#Retry
+
+    :param ExceptionToCheck: the exception to check. may be a tuple of
+        exceptions to check
+    :type ExceptionToCheck: Exception or tuple
+    :param tries: number of times to try (not retry) before giving up
+    :type tries: int
+    :param delay: initial delay between retries in seconds
+    :type delay: int
+    :param backoff: backoff multiplier e.g. value of 2 will double the delay
+        each retry
+    :type backoff: int
+    :param logger: logger to use. If None, print
+    :type logger: logging.Logger instance
+    """
+    def deco_retry(f):
+
+        @wraps(f)
+        def f_retry(*args, **kwargs):
+            mtries, mdelay = tries, delay
+            while mtries > 1:
+                try:
+                    return f(*args, **kwargs)
+                except ExceptionToCheck, e:
+                    #msg = "%s, Retrying in %d seconds..." % (str(e), mdelay)
+                    msg = "Retrying in %d seconds..." %  (mdelay)
+                    if logger:
+                        logger.warning(msg)
+                    else:
+                        #print msg
+                        pass
+                    time.sleep(mdelay)
+                    mtries -= 1
+                    mdelay *= backoff
+            return f(*args, **kwargs)
+
+        return f_retry  # true decorator
+
+    return deco_retry
+
+class TransficUploader:
+    def __init__(self):
+
+        self.c = pycurl.Curl()
+        self.c.setopt(pycurl.URL, __url__)
+        self.c.setopt(pycurl.UPLOAD, 1)
+        #c.setopt(pycurl.USERPWD, 'saket.kumar:whatsinaname.')
+        self.c.setopt(pycurl.PROXY, 'http://saket.kumar:[e-mail]:80/')
+        #c.setopt(pycurl.PROXYPORT, 80)
+        #c.setopt(pycurl.PROXYTYPE_HTTP,1)
+        #self.c.setopt(pycurl.VERBOSE, 1)
+        self.c.setopt(pycurl.HTTPHEADER, ['Expect:'])
+        #c.setopt(pycurl.HTTPPROXYTUNNEL, 1)
+        self.c.setopt(pycurl.UPLOAD, 1)
+        #c.perform()
+        self.c.setopt(pycurl.NOPROGRESS, 1);
+        #curl_easy_setopt(hnd, CURLOPT_UPLOAD, 1L);
+        self.c.setopt(pycurl.USERAGENT, "curl/7.27.0");
+        #curl_easy_setopt(hnd, CURLOPT_HTTPHEADER, slist1);
+        #curl_easy_setopt(hnd, CURLOPT_MAXREDIRS, 50L);
+        self.c.setopt(pycurl.SSL_VERIFYPEER, 1);
+        self.c.setopt(pycurl.CUSTOMREQUEST, "PUT")
+        #curl_easy_setopt(hnd, CURLOPT_VERBOSE, 1L);
+        self.c.setopt(pycurl.TCP_NODELAY, 1);
+        self.buf = cStringIO.StringIO()
+        self.c.setopt(self.c.WRITEFUNCTION, self.buf.write)
+
+
+    def upload_file(self,filepath):
+        f = open(filepath)
+        self.c.setopt(pycurl.INFILE, f)
+        self.c.setopt(pycurl.INFILESIZE, getsize(filepath))
+
+    def run(self):
+        self.c.perform()
+
+    def get_url(self):
+        return self.buf.getvalue().strip()
+
+    @retry(requests.exceptions.HTTPError)
+    def result_exists(self,url ):
+        #url="http://www.cravat.us/results/%s/%s.zip" %(job_id,job_id)
+        download_request = requests.request("GET", url)
+        if download_request.status_code==404 or download_request==500:
+            raise requests.HTTPError()
+        else:
+            return url
+    @retry(requests.exceptions.HTTPError)
+    def download_result(self, url,outpath):
+        tmp_dir = tempfile.mkdtemp()
+        r = requests.get( url, stream=True )
+        if r.status_code == 500:
+            raise requests.HTTPError()
+        else:
+            path = os.path.join( tmp_dir,"results.csv")
+            with open(path, 'wb') as f:
+                for chunk in r.iter_content(128):
+                    f.write(chunk)
+        shutil.move(path,outpath)
+        shutil.rmtree(tmp_dir)
+
+
+
+def main(params):
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--input",type=str,required=True)
+    parser.add_argument("--output",type=str,required=True)
+    args = parser.parse_args(params)
+    uploader = TransficUploader();
+    uploader.upload_file(args.input)
+    uploader.run()
+    url = uploader.get_url()
+    url = uploader.result_exists(url)
+    download = uploader.download_result(url,args.output)
+if __name__=="__main__":
+    main(sys.argv[1:])