|
0
|
1 from __future__ import division, absolute_import, print_function, unicode_literals
|
|
|
2 import sys
|
|
|
3 import shutil
|
|
|
4 import gzip
|
|
|
5 import tempfile
|
|
|
6 import os
|
|
|
7 from py_ts import TrustStoreClient, utils
|
|
|
8 # from galaxy.datatypes.checkers import *
|
|
|
9
|
|
|
10 def printNice(elem, f, depth):
|
|
|
11 try:
|
|
|
12 f.write('\t'*depth + elem.name + " (" + str(len(elem.fragments)) + " parts)\n")
|
|
|
13 except AttributeError:
|
|
|
14 f.write('\t'*depth + elem.name + "\n")
|
|
|
15 for child in elem.children:
|
|
|
16 printNice(child, f, depth+1)
|
|
|
17
|
|
|
18 def check_gzip(file_path):
|
|
|
19 try:
|
|
|
20 temp = open( file_path, "U" )
|
|
|
21 magic_check = temp.read( 2 )
|
|
|
22 temp.close()
|
|
|
23 if magic_check != util.gzip_magic:
|
|
|
24 return ( False, False )
|
|
|
25 except:
|
|
|
26 return ( False, False )
|
|
|
27 return ( True, True )
|
|
|
28
|
|
|
29 def ungzip(download, outputFile):
|
|
|
30 is_gzipped, is_valid = check_gzip(download)
|
|
|
31
|
|
|
32 if is_gzipped and not is_valid:
|
|
|
33 print "File is compressed (gzip) but not valid."
|
|
|
34 sys.exit(4)
|
|
|
35 elif is_gzipped and is_valid:
|
|
|
36 # We need to uncompress the temp_name file, but BAM files must remain compressed in the BGZF format
|
|
|
37 CHUNK_SIZE = 2**20 # 1Mb
|
|
|
38 fd, uncompressed = tempfile.mkstemp(prefix='data_id_upload_gunzip_', dir=os.path.dirname(outputFile), text=False )
|
|
|
39 gzipped_file = gzip.GzipFile(download, 'rb')
|
|
|
40 while 1:
|
|
|
41 try:
|
|
|
42 chunk = gzipped_file.read(CHUNK_SIZE)
|
|
|
43 except IOError:
|
|
|
44 os.close(fd)
|
|
|
45 os.remove(uncompressed)
|
|
|
46 print 'Problem decompressing gzipped data', dataset, json_file
|
|
|
47 sys.exit(4)
|
|
|
48 if not chunk:
|
|
|
49 break
|
|
|
50 os.write(fd, chunk)
|
|
|
51 os.close(fd)
|
|
|
52 gzipped_file.close()
|
|
|
53
|
|
|
54 shutil.copy(uncompressed, outputFile)
|
|
|
55 try:
|
|
|
56 os.remove(uncompressed)
|
|
|
57 os.remove(download)
|
|
|
58 except OSError:
|
|
|
59 pass
|
|
|
60 else:
|
|
|
61 shutil.copy(download, outputFile)
|
|
|
62
|
|
|
63 if __name__ == '__main__':
|
|
|
64
|
|
|
65 kms_url = "https://tstest-kms.it.csiro.au/kmscolab_3_0"
|
|
|
66 ims_url = "https://tstest-ims.it.csiro.au/ims_3_0/services/IMS"
|
|
|
67 username = sys.argv[1]
|
|
|
68 password = sys.argv[2]
|
|
|
69 client_key = "desktop"
|
|
|
70 client_secret = "cpU92F1PT7VOCANjSknuCDp4DrubmujoBaF6b0miz8OpKNokEbGMHCaSFK5/lISbBmaaGVCgeADI2A39F3Hkeg=="
|
|
|
71 storename = sys.argv[3]
|
|
|
72 path = sys.argv[4]
|
|
|
73 fileType = sys.argv[5]
|
|
|
74 filename = ""
|
|
|
75 outputFile = ""
|
|
|
76 if len(sys.argv) > 9:
|
|
|
77 filename = sys.argv[6]
|
|
|
78 outputFile = sys.argv[7]
|
|
|
79 outputFileId = sys.argv[8]
|
|
|
80 otherFilesDir = sys.argv[9]
|
|
|
81 else:
|
|
|
82 outputFile = sys.argv[6]
|
|
|
83
|
|
|
84 config = TrustStoreClient.Config(ims_url, kms_url, client_key, client_secret, tmpDir='/mnt/galaxy/tmp')
|
|
|
85 ts = TrustStoreClient.TrustStoreClient(headless=False, config=config)
|
|
|
86 try:
|
|
|
87 ts.authenticate(username, password)
|
|
|
88 except TrustStoreClient.TrustStoreClientAuthenticationException as e:
|
|
|
89 print e
|
|
|
90 sys.exit(5)
|
|
|
91 ts.getPrivateKey('privkey.pem')
|
|
|
92 listing = ts.listStores()
|
|
|
93 found = False
|
|
|
94 for store in listing:
|
|
|
95 if store.friendly_name == storename:
|
|
|
96 found = True
|
|
|
97 root = ts.listDirectory(store)
|
|
|
98 location = None
|
|
|
99 if path != "/":
|
|
|
100 location = ts_utils.ts_utils.dirAtPath(root, path)
|
|
|
101 if not location:
|
|
|
102 print "Path not found"
|
|
|
103 sys.exit(3)
|
|
|
104 else:
|
|
|
105 location = root
|
|
|
106 if filename and filename != "":
|
|
|
107 outputFileList = [outputFile]
|
|
|
108 inputFileList = None
|
|
|
109 if "," in filename: # we have multiple files guys.
|
|
|
110 inputFileList = filename.split(",")
|
|
|
111 for inputFile in inputFileList[1:]: # First file will be sent to outputFile.
|
|
|
112 outName = "%s_%s_%s_%s_%s" % ('primary', outputFileId, inputFile.replace(".","-"), 'visible', fileType)
|
|
|
113 outputFileList.append(os.path.join(otherFilesDir, outName))
|
|
|
114 else:
|
|
|
115 inputFileList = [filename]
|
|
|
116 for inFile, outFile in zip(inputFileList, outputFileList):
|
|
|
117 downloadMe = ts_utils.ts_utils.recurseToChildNamed(location, inFile)
|
|
|
118 if downloadMe:
|
|
|
119 download = ts.getFile(store, downloadMe)
|
|
|
120 ungzip(download, outFile)
|
|
|
121 else:
|
|
|
122 print "File %s not found" % inFile
|
|
|
123 sys.exit(4)
|
|
|
124 else:
|
|
|
125 with open(outputFile, 'w+') as f:
|
|
|
126 try:
|
|
|
127 for child in root.children:
|
|
|
128 printNice(child, f, 0)
|
|
|
129 except AttributeError as e:
|
|
|
130 print e
|
|
|
131 print root
|
|
|
132 if not found:
|
|
|
133 print "Store %s not found" % storename
|
|
|
134 sys.exit(2)
|
