Mercurial > repos > cathywise > truststore_import
annotate TrustStoreGalaxyImport.py @ 35:3bed3ea2c474
UGH
| author | Catherine Wise <catherine.wise@csiro.au> |
|---|---|
| date | Thu, 23 Jan 2014 13:56:51 +1100 |
| parents | 6e2dba73eebd |
| children | 2ced2ca1c758 |
| rev | line source |
|---|---|
| 4 | 1 import sys |
| 2 import shutil | |
| 20 | 3 import gzip |
| 22 | 4 import tempfile |
| 24 | 5 import os |
| 4 | 6 from py_ts import TrustStoreClient, ts_utils |
| 20 | 7 from galaxy.datatypes.checkers import * |
| 4 | 8 |
| 5 | 9 def printNice(elem, f, depth): |
| 10 try: | |
| 11 f.write('\t'*depth + elem.name + " (" + str(len(elem.fragments)) + " parts)\n") | |
| 12 except AttributeError: | |
| 13 f.write('\t'*depth + elem.name + "\n") | |
| 14 for child in elem.children: | |
| 15 printNice(child, f, depth+1) | |
| 16 | |
| 30 | 17 def ungzip(download, outputFile): |
| 18 is_gzipped, is_valid = check_gzip(download) | |
| 19 | |
| 20 if is_gzipped and not is_valid: | |
| 21 print "File is compressed (gzip) but not valid." | |
| 22 sys.exit(4) | |
| 23 elif is_gzipped and is_valid: | |
| 24 # We need to uncompress the temp_name file, but BAM files must remain compressed in the BGZF format | |
| 25 CHUNK_SIZE = 2**20 # 1Mb | |
| 26 fd, uncompressed = tempfile.mkstemp(prefix='data_id_upload_gunzip_', dir=os.path.dirname(outputFile), text=False ) | |
| 27 gzipped_file = gzip.GzipFile(download, 'rb') | |
| 28 while 1: | |
| 29 try: | |
| 30 chunk = gzipped_file.read(CHUNK_SIZE) | |
| 31 except IOError: | |
| 32 os.close(fd) | |
| 33 os.remove(uncompressed) | |
| 34 print 'Problem decompressing gzipped data', dataset, json_file | |
| 35 sys.exit(4) | |
| 36 if not chunk: | |
| 37 break | |
| 38 os.write(fd, chunk) | |
| 39 os.close(fd) | |
| 40 gzipped_file.close() | |
| 41 | |
| 42 shutil.copy(uncompressed, outputFile) | |
| 43 try: | |
| 44 os.remove(uncompressed) | |
| 45 os.remove(download) | |
| 46 except OSError: | |
| 47 pass | |
| 48 else: | |
| 49 shutil.copy(download, outputFile) | |
| 50 | |
| 4 | 51 if __name__ == '__main__': |
| 52 | |
| 53 kms_url = sys.argv[1] | |
| 54 ims_url = sys.argv[2] | |
| 55 username = sys.argv[3] | |
| 56 password = sys.argv[4] | |
| 57 client_key = sys.argv[5] | |
| 58 client_secret = sys.argv[6] | |
| 59 storename = sys.argv[7] | |
| 60 path = sys.argv[8] | |
| 5 | 61 filename = "" |
| 62 outputFile = "" | |
| 63 if len(sys.argv) > 10: | |
| 64 filename = sys.argv[9] | |
| 65 outputFile = sys.argv[10] | |
|
31
6e2dba73eebd
Undo multi file ugh.
Catherine Wise <catherine.wise@csiro.au>
parents:
30
diff
changeset
|
66 # outputFileId = sys.argv[11] |
|
6e2dba73eebd
Undo multi file ugh.
Catherine Wise <catherine.wise@csiro.au>
parents:
30
diff
changeset
|
67 # outputFileType = sys.argv[12] |
|
6e2dba73eebd
Undo multi file ugh.
Catherine Wise <catherine.wise@csiro.au>
parents:
30
diff
changeset
|
68 # otherFilesDir = sys.argv[13] |
| 5 | 69 else: |
| 70 outputFile = sys.argv[9] | |
| 4 | 71 |
| 72 config = TrustStoreClient.Config(ims_url, kms_url, client_key, client_secret) | |
| 73 ts = TrustStoreClient.TrustStoreClient(headless=True, config=config) | |
| 74 try: | |
| 75 ts.authenticate(username, password) | |
| 76 except TrustStoreClient.TrustStoreClientAuthenticationException as e: | |
| 77 print e | |
| 78 sys.exit(5) | |
| 79 ts.getPrivateKey('privkey.pem') | |
| 80 listing = ts.listStores() | |
| 81 found = False | |
| 82 for store in listing: | |
| 83 if store.friendly_name == storename: | |
| 84 found = True | |
| 85 root = ts.listDirectory(store) | |
| 86 location = None | |
| 87 if path != "/": | |
| 88 location = ts_utils.ts_utils.dirAtPath(root, path) | |
| 89 if not location: | |
| 90 print "Path not found" | |
| 91 sys.exit(3) | |
| 92 else: | |
| 93 location = root | |
| 5 | 94 if filename and filename != "": |
| 30 | 95 outputFileList = [outputFile] |
| 96 inputFileList = None | |
|
31
6e2dba73eebd
Undo multi file ugh.
Catherine Wise <catherine.wise@csiro.au>
parents:
30
diff
changeset
|
97 # if "," in filename: # we have multiple files guys. |
|
6e2dba73eebd
Undo multi file ugh.
Catherine Wise <catherine.wise@csiro.au>
parents:
30
diff
changeset
|
98 # inputFileList = filename.split(",") |
|
6e2dba73eebd
Undo multi file ugh.
Catherine Wise <catherine.wise@csiro.au>
parents:
30
diff
changeset
|
99 # for inputFile in inputFileList: |
|
6e2dba73eebd
Undo multi file ugh.
Catherine Wise <catherine.wise@csiro.au>
parents:
30
diff
changeset
|
100 # outName = "%s_%s_%s_%s_%s" % ('primary', outputFileId, inputFile.replace(".","-"), 'visible', outputFileType) |
|
6e2dba73eebd
Undo multi file ugh.
Catherine Wise <catherine.wise@csiro.au>
parents:
30
diff
changeset
|
101 # outputFileList.append(os.path.join(otherFilesDir, outName)) |
|
6e2dba73eebd
Undo multi file ugh.
Catherine Wise <catherine.wise@csiro.au>
parents:
30
diff
changeset
|
102 # else: |
|
6e2dba73eebd
Undo multi file ugh.
Catherine Wise <catherine.wise@csiro.au>
parents:
30
diff
changeset
|
103 inputFileList = [filename] |
| 30 | 104 for inFile, outFile in zip(inputFileList, outputFileList): |
| 105 downloadMe = ts_utils.ts_utils.recurseToChildNamed(location, inFile) | |
| 106 if downloadMe: | |
| 107 download = ts.getFile(store, downloadMe) | |
| 108 ungzip(download, outFile) | |
| 109 else: | |
| 110 print "File %s not found" % inFile | |
| 20 | 111 sys.exit(4) |
| 4 | 112 else: |
| 5 | 113 with open(outputFile, 'w+') as f: |
| 114 try: | |
| 115 for child in root.children: | |
| 116 printNice(child, f, 0) | |
| 117 except AttributeError as e: | |
| 118 print e | |
| 119 print root | |
| 4 | 120 if not found: |
| 121 print "Store not found" | |
| 122 sys.exit(2) |