Mercurial > repos > cathywise > truststore_import

comparison TrustStoreGalaxyImport.py @ 30:c659bd6342ae

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Fix deps.
author Catherine Wise <catherine.wise@csiro.au>
date Thu, 23 Jan 2014 10:36:03 +1100
parents 20dc961b7949
children 6e2dba73eebd
comparison
equal deleted inserted replaced
29:4c25b2ce827e 30:c659bd6342ae
12 except AttributeError: 12 except AttributeError:
13 f.write('\t'*depth + elem.name + "\n") 13 f.write('\t'*depth + elem.name + "\n")
14 for child in elem.children: 14 for child in elem.children:
15 printNice(child, f, depth+1) 15 printNice(child, f, depth+1)
16 16
17 def ungzip(download, outputFile):
18 is_gzipped, is_valid = check_gzip(download)
19
20 if is_gzipped and not is_valid:
21 print "File is compressed (gzip) but not valid."
22 sys.exit(4)
23 elif is_gzipped and is_valid:
24 # We need to uncompress the temp_name file, but BAM files must remain compressed in the BGZF format
25 CHUNK_SIZE = 2**20 # 1Mb
26 fd, uncompressed = tempfile.mkstemp(prefix='data_id_upload_gunzip_', dir=os.path.dirname(outputFile), text=False )
27 gzipped_file = gzip.GzipFile(download, 'rb')
28 while 1:
29 try:
30 chunk = gzipped_file.read(CHUNK_SIZE)
31 except IOError:
32 os.close(fd)
33 os.remove(uncompressed)
34 print 'Problem decompressing gzipped data', dataset, json_file
35 sys.exit(4)
36 if not chunk:
37 break
38 os.write(fd, chunk)
39 os.close(fd)
40 gzipped_file.close()
41
42 shutil.copy(uncompressed, outputFile)
43 try:
44 os.remove(uncompressed)
45 os.remove(download)
46 except OSError:
47 pass
48 else:
49 shutil.copy(download, outputFile)
50
17 if __name__ == '__main__': 51 if __name__ == '__main__':
18 52
19 kms_url = sys.argv[1] 53 kms_url = sys.argv[1]
20 ims_url = sys.argv[2] 54 ims_url = sys.argv[2]
21 username = sys.argv[3] 55 username = sys.argv[3]
27 filename = "" 61 filename = ""
28 outputFile = "" 62 outputFile = ""
29 if len(sys.argv) > 10: 63 if len(sys.argv) > 10:
30 filename = sys.argv[9] 64 filename = sys.argv[9]
31 outputFile = sys.argv[10] 65 outputFile = sys.argv[10]
66 outputFileId = sys.argv[11]
67 outputFileType = sys.argv[12]
68 otherFilesDir = sys.argv[13]
32 else: 69 else:
33 outputFile = sys.argv[9] 70 outputFile = sys.argv[9]
34 71
35 config = TrustStoreClient.Config(ims_url, kms_url, client_key, client_secret) 72 config = TrustStoreClient.Config(ims_url, kms_url, client_key, client_secret)
36 ts = TrustStoreClient.TrustStoreClient(headless=True, config=config) 73 ts = TrustStoreClient.TrustStoreClient(headless=True, config=config)
53 print "Path not found" 90 print "Path not found"
54 sys.exit(3) 91 sys.exit(3)
55 else: 92 else:
56 location = root 93 location = root
57 if filename and filename != "": 94 if filename and filename != "":
58 downloadMe = ts_utils.ts_utils.recurseToChildNamed(location, filename) 95 outputFileList = [outputFile]
59 if downloadMe: 96 inputFileList = None
60 download = ts.getFile(store, downloadMe) 97 if "," in filename: # we have multiple files guys.
61 is_gzipped, is_valid = check_gzip(download) 98 inputFileList = filename.split(",")
62 99 for inputFile in inputFileList:
63 if is_gzipped and not is_valid: 100 outName = "%s_%s_%s_%s_%s" % ('primary', outputFileId, inputFile.replace(".","-"), 'visible', outputFileType)
64 print "File is compressed (gzip) but not valid." 101 outputFileList.append(os.path.join(otherFilesDir, outName))
102 else:
103 inputFileList = [filename]
104 for inFile, outFile in zip(inputFileList, outputFileList):
105 downloadMe = ts_utils.ts_utils.recurseToChildNamed(location, inFile)
106 if downloadMe:
107 download = ts.getFile(store, downloadMe)
108 ungzip(download, outFile)
109 else:
110 print "File %s not found" % inFile
65 sys.exit(4) 111 sys.exit(4)
66 elif is_gzipped and is_valid:
67 # We need to uncompress the temp_name file, but BAM files must remain compressed in the BGZF format
68 CHUNK_SIZE = 2**20 # 1Mb
69 fd, uncompressed = tempfile.mkstemp(prefix='data_id_upload_gunzip_', dir=os.path.dirname(outputFile), text=False )
70 gzipped_file = gzip.GzipFile(download, 'rb')
71 while 1:
72 try:
73 chunk = gzipped_file.read(CHUNK_SIZE)
74 except IOError:
75 os.close(fd)
76 os.remove(uncompressed)
77 print 'Problem decompressing gzipped data', dataset, json_file
78 sys.exit(4)
79 if not chunk:
80 break
81 os.write(fd, chunk)
82 os.close(fd)
83 gzipped_file.close()
84
85 shutil.copy(uncompressed, outputFile)
86 try:
87 os.remove(uncompressed)
88 os.remove(download)
89 except OSError:
90 pass
91 else:
92 shutil.copy(download, outputFile)
93 else:
94 print "File not found"
95 sys.exit(4)
96 else: 112 else:
97 with open(outputFile, 'w+') as f: 113 with open(outputFile, 'w+') as f:
98 try: 114 try:
99 for child in root.children: 115 for child in root.children:
100 printNice(child, f, 0) 116 printNice(child, f, 0)