Mercurial > repos > cathywise > truststore_import
annotate TrustStoreGalaxyImport.py @ 39:64e06297da39 default tip
Update version.
| author | Catherine Wise <catherine.wise@csiro.au> |
|---|---|
| date | Mon, 24 Feb 2014 15:00:07 +1100 |
| parents | 2ced2ca1c758 |
| children |
| rev | line source |
|---|---|
| 4 | 1 import sys |
| 2 import shutil | |
| 20 | 3 import gzip |
| 22 | 4 import tempfile |
| 24 | 5 import os |
| 4 | 6 from py_ts import TrustStoreClient, ts_utils |
| 20 | 7 from galaxy.datatypes.checkers import * |
| 4 | 8 |
| 5 | 9 def printNice(elem, f, depth): |
| 10 try: | |
| 11 f.write('\t'*depth + elem.name + " (" + str(len(elem.fragments)) + " parts)\n") | |
| 12 except AttributeError: | |
| 13 f.write('\t'*depth + elem.name + "\n") | |
| 14 for child in elem.children: | |
| 15 printNice(child, f, depth+1) | |
| 16 | |
| 30 | 17 def ungzip(download, outputFile): |
| 18 is_gzipped, is_valid = check_gzip(download) | |
| 19 | |
| 20 if is_gzipped and not is_valid: | |
| 21 print "File is compressed (gzip) but not valid." | |
| 22 sys.exit(4) | |
| 23 elif is_gzipped and is_valid: | |
| 24 # We need to uncompress the temp_name file, but BAM files must remain compressed in the BGZF format | |
| 25 CHUNK_SIZE = 2**20 # 1Mb | |
| 26 fd, uncompressed = tempfile.mkstemp(prefix='data_id_upload_gunzip_', dir=os.path.dirname(outputFile), text=False ) | |
| 27 gzipped_file = gzip.GzipFile(download, 'rb') | |
| 28 while 1: | |
| 29 try: | |
| 30 chunk = gzipped_file.read(CHUNK_SIZE) | |
| 31 except IOError: | |
| 32 os.close(fd) | |
| 33 os.remove(uncompressed) | |
| 34 print 'Problem decompressing gzipped data', dataset, json_file | |
| 35 sys.exit(4) | |
| 36 if not chunk: | |
| 37 break | |
| 38 os.write(fd, chunk) | |
| 39 os.close(fd) | |
| 40 gzipped_file.close() | |
| 41 | |
| 42 shutil.copy(uncompressed, outputFile) | |
| 43 try: | |
| 44 os.remove(uncompressed) | |
| 45 os.remove(download) | |
| 46 except OSError: | |
| 47 pass | |
| 48 else: | |
| 49 shutil.copy(download, outputFile) | |
| 50 | |
| 4 | 51 if __name__ == '__main__': |
| 52 | |
|
37
2ced2ca1c758
Multiple downloads.
Catherine Wise <catherine.wise@csiro.au>
parents:
31
diff
changeset
|
53 kms_url = "https://tstest-kms.it.csiro.au/kmscolab_3_0" |
|
2ced2ca1c758
Multiple downloads.
Catherine Wise <catherine.wise@csiro.au>
parents:
31
diff
changeset
|
54 ims_url = "https://tstest-ims.it.csiro.au/ims_3_0/services/IMS" |
|
2ced2ca1c758
Multiple downloads.
Catherine Wise <catherine.wise@csiro.au>
parents:
31
diff
changeset
|
55 username = sys.argv[1] |
|
2ced2ca1c758
Multiple downloads.
Catherine Wise <catherine.wise@csiro.au>
parents:
31
diff
changeset
|
56 password = sys.argv[2] |
|
2ced2ca1c758
Multiple downloads.
Catherine Wise <catherine.wise@csiro.au>
parents:
31
diff
changeset
|
57 client_key = "desktop" |
|
2ced2ca1c758
Multiple downloads.
Catherine Wise <catherine.wise@csiro.au>
parents:
31
diff
changeset
|
58 client_secret = "cpU92F1PT7VOCANjSknuCDp4DrubmujoBaF6b0miz8OpKNokEbGMHCaSFK5/lISbBmaaGVCgeADI2A39F3Hkeg==" |
|
2ced2ca1c758
Multiple downloads.
Catherine Wise <catherine.wise@csiro.au>
parents:
31
diff
changeset
|
59 storename = sys.argv[3] |
|
2ced2ca1c758
Multiple downloads.
Catherine Wise <catherine.wise@csiro.au>
parents:
31
diff
changeset
|
60 path = sys.argv[4] |
|
2ced2ca1c758
Multiple downloads.
Catherine Wise <catherine.wise@csiro.au>
parents:
31
diff
changeset
|
61 fileType = sys.argv[5] |
| 5 | 62 filename = "" |
| 63 outputFile = "" | |
|
37
2ced2ca1c758
Multiple downloads.
Catherine Wise <catherine.wise@csiro.au>
parents:
31
diff
changeset
|
64 if len(sys.argv) > 9: |
|
2ced2ca1c758
Multiple downloads.
Catherine Wise <catherine.wise@csiro.au>
parents:
31
diff
changeset
|
65 filename = sys.argv[6] |
|
2ced2ca1c758
Multiple downloads.
Catherine Wise <catherine.wise@csiro.au>
parents:
31
diff
changeset
|
66 outputFile = sys.argv[7] |
|
2ced2ca1c758
Multiple downloads.
Catherine Wise <catherine.wise@csiro.au>
parents:
31
diff
changeset
|
67 outputFileId = sys.argv[8] |
|
2ced2ca1c758
Multiple downloads.
Catherine Wise <catherine.wise@csiro.au>
parents:
31
diff
changeset
|
68 otherFilesDir = sys.argv[9] |
| 5 | 69 else: |
|
37
2ced2ca1c758
Multiple downloads.
Catherine Wise <catherine.wise@csiro.au>
parents:
31
diff
changeset
|
70 outputFile = sys.argv[6] |
| 4 | 71 |
| 72 config = TrustStoreClient.Config(ims_url, kms_url, client_key, client_secret) | |
| 73 ts = TrustStoreClient.TrustStoreClient(headless=True, config=config) | |
| 74 try: | |
| 75 ts.authenticate(username, password) | |
| 76 except TrustStoreClient.TrustStoreClientAuthenticationException as e: | |
| 77 print e | |
| 78 sys.exit(5) | |
| 79 ts.getPrivateKey('privkey.pem') | |
| 80 listing = ts.listStores() | |
| 81 found = False | |
| 82 for store in listing: | |
| 83 if store.friendly_name == storename: | |
| 84 found = True | |
| 85 root = ts.listDirectory(store) | |
| 86 location = None | |
| 87 if path != "/": | |
| 88 location = ts_utils.ts_utils.dirAtPath(root, path) | |
| 89 if not location: | |
| 90 print "Path not found" | |
| 91 sys.exit(3) | |
| 92 else: | |
| 93 location = root | |
| 5 | 94 if filename and filename != "": |
| 30 | 95 outputFileList = [outputFile] |
| 96 inputFileList = None | |
|
37
2ced2ca1c758
Multiple downloads.
Catherine Wise <catherine.wise@csiro.au>
parents:
31
diff
changeset
|
97 if "," in filename: # we have multiple files guys. |
|
2ced2ca1c758
Multiple downloads.
Catherine Wise <catherine.wise@csiro.au>
parents:
31
diff
changeset
|
98 inputFileList = filename.split(",") |
|
2ced2ca1c758
Multiple downloads.
Catherine Wise <catherine.wise@csiro.au>
parents:
31
diff
changeset
|
99 for inputFile in inputFileList[1:]: # First file will be sent to outputFile. |
|
2ced2ca1c758
Multiple downloads.
Catherine Wise <catherine.wise@csiro.au>
parents:
31
diff
changeset
|
100 outName = "%s_%s_%s_%s_%s" % ('primary', outputFileId, inputFile.replace(".","-"), 'visible', fileType) |
|
2ced2ca1c758
Multiple downloads.
Catherine Wise <catherine.wise@csiro.au>
parents:
31
diff
changeset
|
101 outputFileList.append(os.path.join(otherFilesDir, outName)) |
|
2ced2ca1c758
Multiple downloads.
Catherine Wise <catherine.wise@csiro.au>
parents:
31
diff
changeset
|
102 else: |
|
2ced2ca1c758
Multiple downloads.
Catherine Wise <catherine.wise@csiro.au>
parents:
31
diff
changeset
|
103 inputFileList = [filename] |
| 30 | 104 for inFile, outFile in zip(inputFileList, outputFileList): |
| 105 downloadMe = ts_utils.ts_utils.recurseToChildNamed(location, inFile) | |
| 106 if downloadMe: | |
| 107 download = ts.getFile(store, downloadMe) | |
| 108 ungzip(download, outFile) | |
| 109 else: | |
| 110 print "File %s not found" % inFile | |
| 20 | 111 sys.exit(4) |
| 4 | 112 else: |
| 5 | 113 with open(outputFile, 'w+') as f: |
| 114 try: | |
| 115 for child in root.children: | |
| 116 printNice(child, f, 0) | |
| 117 except AttributeError as e: | |
| 118 print e | |
| 119 print root | |
| 4 | 120 if not found: |
|
37
2ced2ca1c758
Multiple downloads.
Catherine Wise <catherine.wise@csiro.au>
parents:
31
diff
changeset
|
121 print "Store %s not found" % storename |
| 4 | 122 sys.exit(2) |