Mercurial > repos > cathywise > truststore_import
annotate TrustStoreGalaxyImport.py @ 25:20dc961b7949
FILES DELETE THEM UGH
| author | Catherine Wise <catherine.wise@csiro.au> |
|---|---|
| date | Tue, 14 Jan 2014 15:51:07 +1100 |
| parents | 7b1720b6acf2 |
| children | c659bd6342ae |
| rev | line source |
|---|---|
| 4 | 1 import sys |
| 2 import shutil | |
| 20 | 3 import gzip |
| 22 | 4 import tempfile |
| 24 | 5 import os |
| 4 | 6 from py_ts import TrustStoreClient, ts_utils |
| 20 | 7 from galaxy.datatypes.checkers import * |
| 4 | 8 |
| 5 | 9 def printNice(elem, f, depth): |
| 10 try: | |
| 11 f.write('\t'*depth + elem.name + " (" + str(len(elem.fragments)) + " parts)\n") | |
| 12 except AttributeError: | |
| 13 f.write('\t'*depth + elem.name + "\n") | |
| 14 for child in elem.children: | |
| 15 printNice(child, f, depth+1) | |
| 16 | |
| 4 | 17 if __name__ == '__main__': |
| 18 | |
| 19 kms_url = sys.argv[1] | |
| 20 ims_url = sys.argv[2] | |
| 21 username = sys.argv[3] | |
| 22 password = sys.argv[4] | |
| 23 client_key = sys.argv[5] | |
| 24 client_secret = sys.argv[6] | |
| 25 storename = sys.argv[7] | |
| 26 path = sys.argv[8] | |
| 5 | 27 filename = "" |
| 28 outputFile = "" | |
| 29 if len(sys.argv) > 10: | |
| 30 filename = sys.argv[9] | |
| 31 outputFile = sys.argv[10] | |
| 32 else: | |
| 33 outputFile = sys.argv[9] | |
| 4 | 34 |
| 35 config = TrustStoreClient.Config(ims_url, kms_url, client_key, client_secret) | |
| 36 ts = TrustStoreClient.TrustStoreClient(headless=True, config=config) | |
| 37 try: | |
| 38 ts.authenticate(username, password) | |
| 39 except TrustStoreClient.TrustStoreClientAuthenticationException as e: | |
| 40 print e | |
| 41 sys.exit(5) | |
| 42 ts.getPrivateKey('privkey.pem') | |
| 43 listing = ts.listStores() | |
| 44 found = False | |
| 45 for store in listing: | |
| 46 if store.friendly_name == storename: | |
| 47 found = True | |
| 48 root = ts.listDirectory(store) | |
| 49 location = None | |
| 50 if path != "/": | |
| 51 location = ts_utils.ts_utils.dirAtPath(root, path) | |
| 52 if not location: | |
| 53 print "Path not found" | |
| 54 sys.exit(3) | |
| 55 else: | |
| 56 location = root | |
| 5 | 57 if filename and filename != "": |
| 58 downloadMe = ts_utils.ts_utils.recurseToChildNamed(location, filename) | |
| 59 if downloadMe: | |
| 60 download = ts.getFile(store, downloadMe) | |
| 20 | 61 is_gzipped, is_valid = check_gzip(download) |
| 62 | |
| 63 if is_gzipped and not is_valid: | |
| 64 print "File is compressed (gzip) but not valid." | |
| 65 sys.exit(4) | |
| 66 elif is_gzipped and is_valid: | |
| 21 | 67 # We need to uncompress the temp_name file, but BAM files must remain compressed in the BGZF format |
| 68 CHUNK_SIZE = 2**20 # 1Mb | |
| 23 | 69 fd, uncompressed = tempfile.mkstemp(prefix='data_id_upload_gunzip_', dir=os.path.dirname(outputFile), text=False ) |
| 21 | 70 gzipped_file = gzip.GzipFile(download, 'rb') |
| 71 while 1: | |
| 72 try: | |
| 73 chunk = gzipped_file.read(CHUNK_SIZE) | |
| 74 except IOError: | |
| 75 os.close(fd) | |
| 76 os.remove(uncompressed) | |
| 77 print 'Problem decompressing gzipped data', dataset, json_file | |
| 78 sys.exit(4) | |
| 79 if not chunk: | |
| 80 break | |
| 81 os.write(fd, chunk) | |
| 82 os.close(fd) | |
| 83 gzipped_file.close() | |
| 20 | 84 |
| 85 shutil.copy(uncompressed, outputFile) | |
| 24 | 86 try: |
| 87 os.remove(uncompressed) | |
|
25
20dc961b7949
FILES DELETE THEM UGH
Catherine Wise <catherine.wise@csiro.au>
parents:
24
diff
changeset
|
88 os.remove(download) |
| 24 | 89 except OSError: |
| 90 pass | |
| 20 | 91 else: |
| 92 shutil.copy(download, outputFile) | |
| 5 | 93 else: |
| 94 print "File not found" | |
| 95 sys.exit(4) | |
| 4 | 96 else: |
| 5 | 97 with open(outputFile, 'w+') as f: |
| 98 try: | |
| 99 for child in root.children: | |
| 100 printNice(child, f, 0) | |
| 101 except AttributeError as e: | |
| 102 print e | |
| 103 print root | |
| 4 | 104 if not found: |
| 105 print "Store not found" | |
| 106 sys.exit(2) |