|
3
|
1 import sys
|
|
|
2 import requests
|
|
|
3 import pycurl
|
|
|
4 import os
|
|
|
5 from os.path import getsize
|
|
|
6 import argparse
|
|
|
7 import cStringIO
|
|
|
8 from functools import wraps
|
|
|
9 import tempfile
|
|
|
10 import shutil
|
|
|
11 import time
|
|
|
12
|
|
|
13 __url__ = "http://bg.upf.edu/condel/taskService"
|
|
|
14
|
|
|
15
|
|
|
16 def stop_err(msg):
|
|
|
17 sys.stderr.write('%s\n' % msg)
|
|
|
18 sys.exit()
|
|
|
19
|
|
|
20
|
|
|
21 def retry(ExceptionToCheck, tries=10, delay=3, backoff=2, logger=None):
|
|
|
22 """Retry calling the decorated function using an exponential backoff.
|
|
|
23
|
|
|
24 http://www.saltycrane.com/blog/2009/11/trying-out-retry-decorator-python/
|
|
|
25 original from: http://wiki.python.org/moin/PythonDecoratorLibrary#Retry
|
|
|
26
|
|
|
27 :param ExceptionToCheck: the exception to check. may be a tuple of
|
|
|
28 exceptions to check
|
|
|
29 :type ExceptionToCheck: Exception or tuple
|
|
|
30 :param tries: number of times to try (not retry) before giving up
|
|
|
31 :type tries: int
|
|
|
32 :param delay: initial delay between retries in seconds
|
|
|
33 :type delay: int
|
|
|
34 :param backoff: backoff multiplier e.g. value of 2 will double the delay
|
|
|
35 each retry
|
|
|
36 :type backoff: int
|
|
|
37 :param logger: logger to use. If None, print
|
|
|
38 :type logger: logging.Logger instance
|
|
|
39 """
|
|
|
40 def deco_retry(f):
|
|
|
41 @wraps(f)
|
|
|
42 def f_retry(*args, **kwargs):
|
|
|
43 mtries, mdelay = tries, delay
|
|
|
44 while mtries > 1:
|
|
|
45 try:
|
|
|
46 return f(*args, **kwargs)
|
|
|
47 except ExceptionToCheck, e:
|
|
|
48 #msg = "%s, Retrying in %d seconds..." % (str(e), mdelay)
|
|
|
49 msg = "Retrying in %d seconds..." % (mdelay)
|
|
|
50 if logger:
|
|
|
51 logger.warning(msg)
|
|
|
52 else:
|
|
|
53 # print msg
|
|
|
54 pass
|
|
|
55 time.sleep(mdelay)
|
|
|
56 mtries -= 1
|
|
|
57 mdelay *= backoff
|
|
|
58 return f(*args, **kwargs)
|
|
|
59
|
|
|
60 return f_retry # true decorator
|
|
|
61
|
|
|
62 return deco_retry
|
|
|
63
|
|
|
64
|
|
|
65 class TransficUploader:
|
|
|
66
|
|
|
67 def __init__(self):
|
|
|
68
|
|
|
69 self.c = pycurl.Curl()
|
|
|
70 self.c.setopt(pycurl.URL, __url__)
|
|
|
71 self.c.setopt(pycurl.UPLOAD, 1)
|
|
|
72 self.c.setopt(pycurl.PROXY,
|
|
|
73 os.env['http_proxy'])
|
|
|
74 #'http://saket.kumar:uzfmTjX9839.1314@netmon.iitb.ac.in:80/')
|
|
|
75 self.c.setopt(pycurl.HTTPHEADER, ['Expect:'])
|
|
|
76 self.c.setopt(pycurl.UPLOAD, 1)
|
|
|
77 self.c.setopt(pycurl.NOPROGRESS, 1)
|
|
|
78 self.c.setopt(pycurl.USERAGENT, "curl/7.27.0")
|
|
|
79 self.c.setopt(pycurl.SSL_VERIFYPEER, 1)
|
|
|
80 self.c.setopt(pycurl.CUSTOMREQUEST, "PUT")
|
|
|
81 self.c.setopt(pycurl.TCP_NODELAY, 1)
|
|
|
82 self.buf = cStringIO.StringIO()
|
|
|
83 self.c.setopt(self.c.WRITEFUNCTION, self.buf.write)
|
|
|
84
|
|
|
85 def upload_file(self, filepath):
|
|
|
86 f = open(filepath)
|
|
|
87 self.c.setopt(pycurl.INFILE, f)
|
|
|
88 self.c.setopt(pycurl.INFILESIZE, getsize(filepath))
|
|
|
89
|
|
|
90 def run(self):
|
|
|
91 self.c.perform()
|
|
|
92
|
|
|
93 def get_url(self):
|
|
|
94 return self.buf.getvalue().strip()
|
|
|
95
|
|
|
96 @retry(requests.exceptions.HTTPError)
|
|
|
97 def result_exists(self, url):
|
|
|
98 #url="http://www.cravat.us/results/%s/%s.zip" %(job_id,job_id)
|
|
|
99 download_request = requests.request("GET", url)
|
|
|
100 if download_request.status_code == 404 or download_request == 500:
|
|
|
101 raise requests.HTTPError()
|
|
|
102 else:
|
|
|
103 return url
|
|
|
104
|
|
|
105 @retry(requests.exceptions.HTTPError)
|
|
|
106 def download_result(self, url, outpath):
|
|
|
107 tmp_dir = tempfile.mkdtemp()
|
|
|
108 r = requests.get(url, stream=True)
|
|
|
109 if r.status_code == 500:
|
|
|
110 raise requests.HTTPError()
|
|
|
111 else:
|
|
|
112 path = os.path.join(tmp_dir, "results.csv")
|
|
|
113 with open(path, 'wb') as f:
|
|
|
114 for chunk in r.iter_content(128):
|
|
|
115 f.write(chunk)
|
|
|
116 shutil.move(path, outpath)
|
|
|
117 shutil.rmtree(tmp_dir)
|
|
|
118
|
|
|
119
|
|
|
120 def main(params):
|
|
|
121 parser = argparse.ArgumentParser()
|
|
|
122 parser.add_argument("--input", type=str, required=True)
|
|
|
123 parser.add_argument("--output", type=str, required=True)
|
|
|
124 args = parser.parse_args(params)
|
|
|
125 uploader = TransficUploader()
|
|
|
126 uploader.upload_file(args.input)
|
|
|
127 uploader.run()
|
|
|
128 url = uploader.get_url()
|
|
|
129 url = uploader.result_exists(url)
|
|
|
130 uploader.download_result(url, args.output)
|
|
|
131
|
|
|
132
|
|
|
133 if __name__ == "__main__":
|
|
|
134 main(sys.argv[1:])
|
