Mercurial > repos > cathywise > truststore_browse_testing

comparison TrustStoreGalaxyBrowse.py @ 9:0f2a5664c9eb

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Actually update downloader (probably not working).
author Catherine Wise <catherine.wise@csiro.au>
date Tue, 12 May 2015 11:35:04 +1000
parents 2a996fea49a3
children 16b0ceecf1f7
comparison
equal deleted inserted replaced
8:96847f227d20 9:0f2a5664c9eb
1 """TrustStore downloaded for Galaxy."""
1 from __future__ import division, absolute_import, print_function, unicode_literals 2 from __future__ import division, absolute_import, print_function, unicode_literals
2 import sys 3 import sys
3 import shutil 4 import shutil
4 import gzip 5 import gzip
5 import tempfile 6 import tempfile
6 import os 7 import os
8 import json
9 import operator
7 from py_ts import TrustStoreClient, utils 10 from py_ts import TrustStoreClient, utils
8 # from galaxy.datatypes.checkers import * 11 from galaxy.datatypes.checkers import util
9 12
10 def printNice(elem, f, depth): 13 CLIENT_KEY = "desktop"
14 CLIENT_SECRET = "cpU92F1PT7VOCANjSknuCDp4DrubmujoBaF6b0miz8OpKNokEbGMHCaSFK5/lISbBmaaGVCgeADI2A39F3Hkeg=="
15 CHUNK_SIZE = 2**20 # 1Mb
16 SAFE_CHARS = '.-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ '
17
18 def print_nice(elem, f, depth):
19 """Print the file name."""
11 try: 20 try:
12 f.write('\t'*depth + elem.name + " (" + str(len(elem.fragments)) + " parts)\n") 21 f.write('\t'*depth + elem.name + " (" + str(len(elem.fragments)) + " parts)\n")
13 except AttributeError: 22 except AttributeError:
14 f.write('\t'*depth + elem.name + "\n") 23 f.write('\t'*depth + elem.name + "\n")
15 for child in elem.children: 24 for child in elem.children:
16 printNice(child, f, depth+1) 25 print_nice(child, f, depth+1)
17 26
18 def check_gzip(file_path): 27 def check_gzip(file_path):
28 """Check if file is gziped."""
19 try: 29 try:
20 temp = open( file_path, "U" ) 30 temp = open(file_path, "U")
21 magic_check = temp.read( 2 ) 31 magic_check = temp.read(2)
22 temp.close() 32 temp.close()
23 if magic_check != util.gzip_magic: 33 if magic_check != util.gzip_magic:
24 return ( False, False ) 34 return (False, False)
25 except: 35 except Exception:
26 return ( False, False ) 36 return (False, False)
27 return ( True, True ) 37 return (True, True)
28 38
29 def ungzip(download, outputFile): 39 def ungzip(download, outputFile):
40 """Uncompress file."""
30 is_gzipped, is_valid = check_gzip(download) 41 is_gzipped, is_valid = check_gzip(download)
31 42
32 if is_gzipped and not is_valid: 43 if is_gzipped and not is_valid:
33 print "File is compressed (gzip) but not valid." 44 print("File is compressed (gzip) but not valid.")
34 sys.exit(4) 45 sys.exit(4)
35 elif is_gzipped and is_valid: 46 elif is_gzipped and is_valid:
36 # We need to uncompress the temp_name file, but BAM files must remain compressed in the BGZF format 47 # We need to uncompress the temp_name file, but BAM files must
37 CHUNK_SIZE = 2**20 # 1Mb 48 # remain compressed in the BGZF format
38 fd, uncompressed = tempfile.mkstemp(prefix='data_id_upload_gunzip_', dir=os.path.dirname(outputFile), text=False ) 49 file_handle, uncompressed = tempfile.mkstemp(prefix='data_id_upload_gunzip_', dir=os.path.dirname(outputFile), text=False )
39 gzipped_file = gzip.GzipFile(download, 'rb') 50 gzipped_file = gzip.GzipFile(download, 'rb')
40 while 1: 51 while 1:
41 try: 52 try:
42 chunk = gzipped_file.read(CHUNK_SIZE) 53 chunk = gzipped_file.read(CHUNK_SIZE)
43 except IOError: 54 except IOError:
44 os.close(fd) 55 os.close(file_handle)
45 os.remove(uncompressed) 56 os.remove(uncompressed)
46 print 'Problem decompressing gzipped data', dataset, json_file 57 print('Problem decompressing gzipped data %s %s' % (download, outputFile))
47 sys.exit(4) 58 sys.exit(4)
48 if not chunk: 59 if not chunk:
49 break 60 break
50 os.write(fd, chunk) 61 os.write(file_handle, chunk)
51 os.close(fd) 62 os.close(file_handle)
52 gzipped_file.close() 63 gzipped_file.close()
53 64
54 shutil.copy(uncompressed, outputFile) 65 shutil.copy(uncompressed, outputFile)
55 try: 66 try:
56 os.remove(uncompressed) 67 os.remove(uncompressed)
57 os.remove(download) 68 os.remove(download)
58 except OSError: 69 except OSError:
59 pass 70 pass
60 else: 71 else:
61 shutil.copy(download, outputFile) 72 shutil.copy(download, outputFile)
62 73
74 def construct_multi_filename(id, name, file_type):
75 """ Implementation of *Number of Output datasets cannot be determined until
76 tool run* from documentation_.
77 .. _documentation: http://wiki.galaxyproject.org/Admin/Tools/Multiple%20Output%20Files
78 From https://github.com/mdshw5/galaxy-json-data-source/blob/master/json_data_source.py
79 """
80 filename = "%s_%s_%s_%s_%s" % ('primary', id, name, 'visible', file_type)
81 return filename
82
83 def metadata_to_json(dataset_id, filename, name, extesion, ds_type='dataset', primary=False):
84 """ Return line separated JSON
85 From https://github.com/mdshw5/galaxy-json-data-source/blob/master/json_data_source.py
86 """
87 meta_dict = dict(type=ds_type,
88 ext=extesion,
89 filename=filename,
90 name=name,
91 metadata={})
92 if primary:
93 meta_dict['base_dataset_id'] = dataset_id
94 else:
95 meta_dict['dataset_id'] = dataset_id
96 return "%s\n" % json.dumps(meta_dict)
97
98 def main():
99 properties_file = sys.argv[1]
100 ouput_root = sys.argv[2]
101 json_params = None
102 metadata_path = None
103 with open(properties_file, 'r') as file_:
104 all_params = json.loads(file_.read())
105 json_params = all_params.get("param_dict")
106 metadata_path = all_params["job_config"]["TOOL_PROVIDED_JOB_METADATA_FILE"]
107
108 output_filename = json_params.get('output', None)
109 output_data = json_params.get('output_data')
110 extra_files_path, file_name, ext, out_data_name, hda_id, dataset_id = \
111 operator.itemgetter('extra_files_path', 'file_name', 'ext', 'out_data_name', 'hda_id', 'dataset_id')(output_data[0])
112
113 url_params = json_params['url'].split(";")
114 if len(url_params) < 3:
115 print("The url we got back is malformed: "+ url_params)
116 sys.exit(5)
117 short_url = url_params[0]
118 username = url_params[1]
119 password = url_params[2]
120 if "/short" not in short_url:
121 print("The url we got back is malformed: " + url_params)
122 sys.exit(5)
123 kms_url = short_url.split("/short")[0]
124
125 config = TrustStoreClient.Config(
126 None, kms_url, CLIENT_KEY, CLIENT_SECRET, tmpDir='/mnt/galaxy/tmp')
127 truststore = TrustStoreClient.TrustStoreClient(headless=False, config=config)
128 try:
129 truststore.authenticate(username, password)
130 except TrustStoreClient.TrustStoreClientAuthenticationException as err:
131 print(err)
132 sys.exit(5)
133 truststore.getPrivateKey('privkey.pem')
134
135 path_texts = truststore.lengthenPath(short_url)
136 if len(path_texts) < 2:
137 print("The path we got was malformed.")
138 sys.exit(3)
139 paths = path_texts[1:]
140 store_id = path_texts[0]
141
142 store = truststore.getStoreByID(store_id)
143 if store is None:
144 print("Coudn't find store with that ID, or don't have access.")
145 sys.exit(2)
146 root = truststore.listDirectory(store)
147
148 with open(metadata_path, 'wb') as metadata_file:
149 for path in paths:
150 locations = utils.Navigation.files_at_path(root, path)
151 for location in locations:
152 if not locations:
153 print("Path not found: " + path)
154 continue
155 filename = "".join(c in SAFE_CHARS and c or '-' for c in location.name)
156 extension = os.path.splitext(filename)[1]
157 name = construct_multi_filename(hda_id, filename, extension)
158 target_output_filename = os.path.normpath("/".join([ouput_root, name]))
159 metadata_file.write(
160 metadata_to_json(dataset_id, filename, name, extension, target_output_filename))
161 download = truststore.getFile(store, location)
162 if download is None:
163 print("File %s not found." % location.name)
164 sys.exit(4)
165 ungzip(download, target_output_filename)
166
167
63 if __name__ == '__main__': 168 if __name__ == '__main__':
64 169 main()
65 kms_url = "https://tstest-kms.it.csiro.au/kmscolab_3_0"
66 ims_url = "https://tstest-ims.it.csiro.au/ims_3_0/services/IMS"
67 username = sys.argv[1]
68 password = sys.argv[2]
69 client_key = "desktop"
70 client_secret = "cpU92F1PT7VOCANjSknuCDp4DrubmujoBaF6b0miz8OpKNokEbGMHCaSFK5/lISbBmaaGVCgeADI2A39F3Hkeg=="
71 storename = sys.argv[3]
72 path = sys.argv[4]
73 fileType = sys.argv[5]
74 filename = ""
75 outputFile = ""
76 if len(sys.argv) > 9:
77 filename = sys.argv[6]
78 outputFile = sys.argv[7]
79 outputFileId = sys.argv[8]
80 otherFilesDir = sys.argv[9]
81 else:
82 outputFile = sys.argv[6]
83
84 config = TrustStoreClient.Config(ims_url, kms_url, client_key, client_secret, tmpDir='/mnt/galaxy/tmp')
85 ts = TrustStoreClient.TrustStoreClient(headless=False, config=config)
86 try:
87 ts.authenticate(username, password)
88 except TrustStoreClient.TrustStoreClientAuthenticationException as e:
89 print e
90 sys.exit(5)
91 ts.getPrivateKey('privkey.pem')
92 listing = ts.listStores()
93 found = False
94 for store in listing:
95 if store.friendly_name == storename:
96 found = True
97 root = ts.listDirectory(store)
98 location = None
99 if path != "/":
100 location = ts_utils.ts_utils.dirAtPath(root, path)
101 if not location:
102 print "Path not found"
103 sys.exit(3)
104 else:
105 location = root
106 if filename and filename != "":
107 outputFileList = [outputFile]
108 inputFileList = None
109 if "," in filename: # we have multiple files guys.
110 inputFileList = filename.split(",")
111 for inputFile in inputFileList[1:]: # First file will be sent to outputFile.
112 outName = "%s_%s_%s_%s_%s" % ('primary', outputFileId, inputFile.replace(".","-"), 'visible', fileType)
113 outputFileList.append(os.path.join(otherFilesDir, outName))
114 else:
115 inputFileList = [filename]
116 for inFile, outFile in zip(inputFileList, outputFileList):
117 downloadMe = ts_utils.ts_utils.recurseToChildNamed(location, inFile)
118 if downloadMe:
119 download = ts.getFile(store, downloadMe)
120 ungzip(download, outFile)
121 else:
122 print "File %s not found" % inFile
123 sys.exit(4)
124 else:
125 with open(outputFile, 'w+') as f:
126 try:
127 for child in root.children:
128 printNice(child, f, 0)
129 except AttributeError as e:
130 print e
131 print root
132 if not found:
133 print "Store %s not found" % storename
134 sys.exit(2)