|
3
|
1 import sys
|
|
|
2 sys.path.insert(0, '/home/saket/requests-new-urllib3-api/requests/packages/')
|
|
|
3 sys.path.insert(0, '/home/saket/requests-new-urllib3-api')
|
|
|
4
|
|
|
5
|
|
|
6 import requests
|
|
|
7 import pycurl
|
|
|
8 import os
|
|
|
9 from os.path import getsize
|
|
|
10 import argparse
|
|
|
11 import sys
|
|
|
12 import cStringIO
|
|
|
13 from functools import wraps
|
|
|
14 import tempfile
|
|
|
15 import shutil
|
|
|
16 import time
|
|
|
17
|
|
|
18 __url__ = "http://bg.upf.edu/transfic/taskService"
|
|
|
19
|
|
|
20
|
|
|
21 def stop_err(msg, err=1):
|
|
|
22 sys.stderr.write('%s\n' % msg)
|
|
|
23 sys.exit(err)
|
|
|
24
|
|
|
25
|
|
|
26 def retry(ExceptionToCheck, tries=14, delay=3, backoff=2, logger=None):
|
|
|
27 """Retry calling the decorated function using an exponential backoff.
|
|
|
28
|
|
|
29 http://www.saltycrane.com/blog/2009/11/trying-out-retry-decorator-python/
|
|
|
30 original from: http://wiki.python.org/moin/PythonDecoratorLibrary#Retry
|
|
|
31
|
|
|
32 :param ExceptionToCheck: the exception to check. may be a tuple of
|
|
|
33 exceptions to check
|
|
|
34 :type ExceptionToCheck: Exception or tuple
|
|
|
35 :param tries: number of times to try (not retry) before giving up
|
|
|
36 :type tries: int
|
|
|
37 :param delay: initial delay between retries in seconds
|
|
|
38 :type delay: int
|
|
|
39 :param backoff: backoff multiplier e.g. value of 2 will double the delay
|
|
|
40 each retry
|
|
|
41 :type backoff: int
|
|
|
42 :param logger: logger to use. If None, print
|
|
|
43 :type logger: logging.Logger instance
|
|
|
44 """
|
|
|
45 def deco_retry(f):
|
|
|
46
|
|
|
47 @wraps(f)
|
|
|
48 def f_retry(*args, **kwargs):
|
|
|
49 mtries, mdelay = tries, delay
|
|
|
50 while mtries > 1:
|
|
|
51 try:
|
|
|
52 return f(*args, **kwargs)
|
|
|
53 except ExceptionToCheck, e:
|
|
|
54 #msg = "%s, Retrying in %d seconds..." % (str(e), mdelay)
|
|
|
55 msg = "Retrying in %d seconds..." % (mdelay)
|
|
|
56 if logger:
|
|
|
57 logger.warning(msg)
|
|
|
58 else:
|
|
|
59 # print msg
|
|
|
60 pass
|
|
|
61 time.sleep(mdelay)
|
|
|
62 mtries -= 1
|
|
|
63 mdelay *= backoff
|
|
|
64 return f(*args, **kwargs)
|
|
|
65
|
|
|
66 return f_retry # true decorator
|
|
|
67
|
|
|
68 return deco_retry
|
|
|
69
|
|
|
70
|
|
|
71 class TransficUploader:
|
|
|
72
|
|
|
73 def __init__(self):
|
|
|
74
|
|
|
75 self.c = pycurl.Curl()
|
|
|
76 self.c.setopt(pycurl.URL, __url__)
|
|
|
77 self.c.setopt(pycurl.UPLOAD, 1)
|
|
|
78 self.c.setopt(
|
|
|
79 pycurl.PROXY, os.environ['http_proxy'])
|
|
|
80 #'http://saket.kumar:uzfmTjX9839.1314@netmon.iitb.ac.in:80/')
|
|
|
81 self.c.setopt(pycurl.HTTPHEADER, ['Expect:'])
|
|
|
82 self.c.setopt(pycurl.UPLOAD, 1)
|
|
|
83 self.c.setopt(pycurl.NOPROGRESS, 1)
|
|
|
84 self.c.setopt(pycurl.USERAGENT, "curl/7.27.0")
|
|
|
85 self.c.setopt(pycurl.SSL_VERIFYPEER, 1)
|
|
|
86 self.c.setopt(pycurl.CUSTOMREQUEST, "PUT")
|
|
|
87 self.c.setopt(pycurl.TCP_NODELAY, 1)
|
|
|
88 self.buf = cStringIO.StringIO()
|
|
|
89 self.c.setopt(self.c.WRITEFUNCTION, self.buf.write)
|
|
|
90
|
|
|
91 def upload_file(self, filepath):
|
|
|
92 f = open(filepath)
|
|
|
93 self.c.setopt(pycurl.INFILE, f)
|
|
|
94 self.c.setopt(pycurl.INFILESIZE, getsize(filepath))
|
|
|
95
|
|
|
96 def run(self):
|
|
|
97 self.c.perform()
|
|
|
98
|
|
|
99 def get_url(self):
|
|
|
100 return self.buf.getvalue().strip()
|
|
|
101
|
|
|
102 @retry(requests.exceptions.HTTPError)
|
|
|
103 def result_exists(self, url):
|
|
|
104 download_request = requests.request("GET", url)
|
|
|
105 if download_request.status_code == 404 or download_request == 500:
|
|
|
106 raise requests.HTTPError()
|
|
|
107 elif "Task status is : error" in download_request.text:
|
|
|
108 stop_err("No SNVs found!")
|
|
|
109 else:
|
|
|
110 return url
|
|
|
111
|
|
|
112 @retry(requests.exceptions.HTTPError)
|
|
|
113 def download_result(self, url, outpath):
|
|
|
114 tmp_dir = tempfile.mkdtemp()
|
|
|
115 r = requests.get(url, stream=True)
|
|
|
116 if r.status_code == 500:
|
|
|
117 raise requests.HTTPError()
|
|
|
118 else:
|
|
|
119 path = os.path.join(tmp_dir, "results.csv")
|
|
|
120 with open(path, 'wb') as f:
|
|
|
121 for chunk in r.iter_content(128):
|
|
|
122 f.write(chunk)
|
|
|
123 shutil.move(path, outpath)
|
|
|
124 shutil.rmtree(tmp_dir)
|
|
|
125
|
|
|
126
|
|
|
127 def main(params):
|
|
|
128 parser = argparse.ArgumentParser()
|
|
|
129 parser.add_argument("--input", type=str, required=True)
|
|
|
130 parser.add_argument("--output", type=str, required=True)
|
|
|
131 args = parser.parse_args(params)
|
|
|
132 uploader = TransficUploader()
|
|
|
133 uploader.upload_file(args.input)
|
|
|
134 uploader.run()
|
|
|
135 url = uploader.get_url()
|
|
|
136 url = uploader.result_exists(url)
|
|
|
137 uploader.download_result(url, args.output)
|
|
|
138
|
|
|
139
|
|
|
140 if __name__ == "__main__":
|
|
|
141 main(sys.argv[1:])
|
