Mercurial > repos > cathywise > truststore_browse
annotate TrustStoreGalaxyBrowse.py @ 19:bae7471a4741
Again.
| author | Wise, Catherine (Digital, Acton) <Catherine.Wise@csiro.au> |
|---|---|
| date | Thu, 25 Jun 2015 13:02:16 +1000 |
| parents | dd8e0066e9f5 |
| children | 91bad9ac39c6 |
| rev | line source |
|---|---|
| 0 | 1 """TrustStore downloaded for Galaxy.""" |
| 2 from __future__ import division, absolute_import, print_function, unicode_literals | |
| 3 import sys | |
| 4 import shutil | |
| 5 import gzip | |
| 6 import tempfile | |
| 7 import os | |
| 8 import json | |
| 9 import operator | |
| 10 import urlparse | |
| 11 from py_ts import TrustStoreClient, utils | |
|
14
c7287129f37f
And again.
Wise, Catherine (Digital, Acton) <Catherine.Wise@csiro.au>
parents:
13
diff
changeset
|
12 # import galaxy.model # need to import model before sniff to resolve a circular import dependency |
| 0 | 13 from galaxy.datatypes.checkers import util |
|
14
c7287129f37f
And again.
Wise, Catherine (Digital, Acton) <Catherine.Wise@csiro.au>
parents:
13
diff
changeset
|
14 # from galaxy.datatypes import sniff |
|
c7287129f37f
And again.
Wise, Catherine (Digital, Acton) <Catherine.Wise@csiro.au>
parents:
13
diff
changeset
|
15 # from galaxy.datatypes.registry import Registry |
| 0 | 16 |
| 17 # Tell urllib3 to use pyOpenSSL because we are on old Python stdlib. | |
| 18 # import urllib3.contrib.pyopenssl | |
| 19 # urllib3.contrib.pyopenssl.inject_into_urllib3() | |
| 20 # | |
| 21 os.environ["REQUESTS_CA_BUNDLE"] = "/etc/ssl/certs/ca-certificates.crt" | |
| 22 | |
| 23 CLIENT_KEY = "desktop" | |
| 24 CLIENT_SECRET = "cpU92F1PT7VOCANjSknuCDp4DrubmujoBaF6b0miz8OpKNokEbGMHCaSFK5/lISbBmaaGVCgeADI2A39F3Hkeg==" | |
| 25 CHUNK_SIZE = 2**20 # 1Mb | |
| 26 SAFE_CHARS = '.-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ ' | |
| 27 | |
| 28 def print_nice(elem, f, depth): | |
| 29 """Print the file name.""" | |
| 30 try: | |
| 31 f.write('\t'*depth + elem.name + " (" + str(len(elem.fragments)) + " parts)\n") | |
| 32 except AttributeError: | |
| 33 f.write('\t'*depth + elem.name + "\n") | |
| 34 for child in elem.children: | |
| 35 print_nice(child, f, depth+1) | |
| 36 | |
| 37 def check_gzip(file_path): | |
| 38 """Check if file is gziped.""" | |
| 39 try: | |
| 40 temp = open(file_path, "U") | |
| 41 magic_check = temp.read(2) | |
| 42 temp.close() | |
| 43 if magic_check != util.gzip_magic: | |
| 44 return (False, False) | |
| 45 except Exception: | |
| 46 return (False, False) | |
| 47 return (True, True) | |
| 48 | |
| 49 def ungzip(download, outputFile): | |
| 50 """Uncompress file.""" | |
| 51 is_gzipped, is_valid = check_gzip(download) | |
| 52 | |
| 53 if is_gzipped and not is_valid: | |
| 54 print("File is compressed (gzip) but not valid.") | |
| 55 sys.exit(4) | |
| 56 elif is_gzipped and is_valid: | |
| 57 # We need to uncompress the temp_name file, but BAM files must | |
| 58 # remain compressed in the BGZF format | |
| 59 file_handle, uncompressed = tempfile.mkstemp(prefix='data_id_upload_gunzip_', dir=os.path.dirname(outputFile), text=False ) | |
| 60 gzipped_file = gzip.GzipFile(download, 'rb') | |
| 61 while 1: | |
| 62 try: | |
| 63 chunk = gzipped_file.read(CHUNK_SIZE) | |
| 64 except IOError: | |
| 65 os.close(file_handle) | |
| 66 os.remove(uncompressed) | |
| 67 print('Problem decompressing gzipped data %s %s' % (download, outputFile)) | |
| 68 sys.exit(4) | |
| 69 if not chunk: | |
| 70 break | |
| 71 os.write(file_handle, chunk) | |
| 72 os.close(file_handle) | |
| 73 gzipped_file.close() | |
| 74 | |
|
18
dd8e0066e9f5
I wish there was an easier way to test this. Ugh.
Wise, Catherine (Digital, Acton) <Catherine.Wise@csiro.au>
parents:
17
diff
changeset
|
75 try: |
|
dd8e0066e9f5
I wish there was an easier way to test this. Ugh.
Wise, Catherine (Digital, Acton) <Catherine.Wise@csiro.au>
parents:
17
diff
changeset
|
76 shutil.copy(uncompressed, outputFile) |
|
dd8e0066e9f5
I wish there was an easier way to test this. Ugh.
Wise, Catherine (Digital, Acton) <Catherine.Wise@csiro.au>
parents:
17
diff
changeset
|
77 except shutil.Error: |
|
dd8e0066e9f5
I wish there was an easier way to test this. Ugh.
Wise, Catherine (Digital, Acton) <Catherine.Wise@csiro.au>
parents:
17
diff
changeset
|
78 pass # If the file is already in the right location, move along. |
| 0 | 79 try: |
| 80 os.remove(uncompressed) | |
| 81 os.remove(download) | |
| 82 except OSError: | |
| 83 pass | |
| 84 else: | |
|
19
bae7471a4741
Again.
Wise, Catherine (Digital, Acton) <Catherine.Wise@csiro.au>
parents:
18
diff
changeset
|
85 try: |
|
bae7471a4741
Again.
Wise, Catherine (Digital, Acton) <Catherine.Wise@csiro.au>
parents:
18
diff
changeset
|
86 shutil.copy(download, outputFile) |
|
bae7471a4741
Again.
Wise, Catherine (Digital, Acton) <Catherine.Wise@csiro.au>
parents:
18
diff
changeset
|
87 except shutil.Error: |
|
bae7471a4741
Again.
Wise, Catherine (Digital, Acton) <Catherine.Wise@csiro.au>
parents:
18
diff
changeset
|
88 pass # If the file is already in the right location, move along. |
| 0 | 89 |
| 90 def construct_multi_filename(id, name, file_type): | |
| 91 """ Implementation of *Number of Output datasets cannot be determined until | |
| 92 tool run* from documentation_. | |
| 93 .. _documentation: http://wiki.galaxyproject.org/Admin/Tools/Multiple%20Output%20Files | |
| 94 From https://github.com/mdshw5/galaxy-json-data-source/blob/master/json_data_source.py | |
| 95 """ | |
| 96 filename = "%s_%s_%s_%s_%s" % ('primary', id, name, 'visible', file_type) | |
| 97 return filename | |
| 98 | |
| 99 def metadata_to_json(dataset_id, filename, name, extesion, ds_type='dataset', primary=False): | |
| 100 """ Return line separated JSON | |
| 101 From https://github.com/mdshw5/galaxy-json-data-source/blob/master/json_data_source.py | |
| 102 """ | |
| 103 meta_dict = dict(type=ds_type, | |
| 104 ext=extesion, | |
| 105 filename=filename, | |
| 106 name=name, | |
| 107 metadata={}) | |
| 108 if primary: | |
| 109 meta_dict['base_dataset_id'] = dataset_id | |
| 110 else: | |
| 111 meta_dict['dataset_id'] = dataset_id | |
| 112 return "%s\n" % json.dumps(meta_dict) | |
| 113 | |
| 114 def main(): | |
| 115 properties_file = sys.argv[1] | |
| 116 json_params = None | |
| 117 metadata_path = None | |
| 118 all_params = None | |
| 119 with open(properties_file, 'r') as file_: | |
| 120 settings = file_.read() | |
| 121 all_params = json.loads(settings) | |
| 122 json_params = all_params.get("param_dict") | |
| 123 metadata_path = all_params["job_config"]["TOOL_PROVIDED_JOB_METADATA_FILE"] | |
| 124 | |
| 125 output_filename = json_params.get('output', None) | |
| 126 output_data = all_params.get('output_data') | |
| 127 extra_files_path, file_name, ext, out_data_name, hda_id, dataset_id = \ | |
| 128 operator.itemgetter('extra_files_path', 'file_name', 'ext', 'out_data_name', 'hda_id', 'dataset_id')(output_data[0]) | |
|
4
2ca750b9083c
Fix extra files directory.
Catherine Wise <catherine.wise@csiro.au>
parents:
3
diff
changeset
|
129 extra_files_path = json_params['__new_file_path__'] |
| 0 | 130 |
|
14
c7287129f37f
And again.
Wise, Catherine (Digital, Acton) <Catherine.Wise@csiro.au>
parents:
13
diff
changeset
|
131 # datatypes_registry = Registry() |
|
c7287129f37f
And again.
Wise, Catherine (Digital, Acton) <Catherine.Wise@csiro.au>
parents:
13
diff
changeset
|
132 # datatypes_registry.load_datatypes( |
|
c7287129f37f
And again.
Wise, Catherine (Digital, Acton) <Catherine.Wise@csiro.au>
parents:
13
diff
changeset
|
133 # root_dir=all_params['job_config']['GALAXY_ROOT_DIR'], |
|
c7287129f37f
And again.
Wise, Catherine (Digital, Acton) <Catherine.Wise@csiro.au>
parents:
13
diff
changeset
|
134 # config=all_params['job_config']['GALAXY_DATATYPES_CONF_FILE'] |
|
c7287129f37f
And again.
Wise, Catherine (Digital, Acton) <Catherine.Wise@csiro.au>
parents:
13
diff
changeset
|
135 # ) |
|
9
3e8bd0d01725
Attempt to guess data type.
Wise, Catherine (Digital, Acton) <Catherine.Wise@csiro.au>
parents:
4
diff
changeset
|
136 |
| 0 | 137 url_params = urlparse.unquote(json_params['URL']).split(";") |
| 138 if len(url_params) < 3: | |
| 139 print("The url we got back is malformed: "+ json_params['URL']) | |
| 140 sys.exit(5) | |
| 141 short_url = url_params[0] | |
| 142 username = url_params[1] | |
| 143 password = url_params[2] | |
| 144 if "/short" not in short_url: | |
| 145 print("The url we got back is malformed: " + json_params['URL']) | |
| 146 sys.exit(5) | |
| 147 kms_url = short_url.split("/short")[0] | |
| 148 | |
| 149 tmp_dir = '/mnt/galaxy/tmp' | |
| 150 tmp = None | |
| 151 if os.path.exists(tmp_dir): | |
| 152 tmp = tmp_dir | |
| 153 | |
| 154 config = TrustStoreClient.Config( | |
| 155 None, kms_url, CLIENT_KEY, CLIENT_SECRET, tmpDir=tmp) | |
| 156 truststore = TrustStoreClient.TrustStoreClient(headless=False, config=config) | |
| 157 try: | |
| 158 truststore.authenticate(username, password) | |
| 159 except TrustStoreClient.TrustStoreClientAuthenticationException as err: | |
| 160 print(err) | |
| 161 sys.exit(5) | |
| 162 truststore.getPrivateKey('privkey.pem') | |
| 163 | |
| 164 path_texts = truststore.lengthenPath(short_url) | |
| 165 if len(path_texts) < 2: | |
| 166 print("The path we got was malformed: " + str(path_texts)) | |
| 167 sys.exit(3) | |
| 168 paths = path_texts[1:] | |
| 169 store_id = path_texts[0] | |
| 170 | |
| 171 store = truststore.getStoreByID(store_id) | |
| 172 if store is None: | |
| 173 print("Coudn't find store with that ID, or don't have access.") | |
| 174 sys.exit(2) | |
| 175 root = truststore.listDirectory(store) | |
| 176 | |
| 177 first = True | |
| 178 | |
| 179 print("Preparing the following for downloading: " + str(paths)) | |
| 180 | |
|
16
bc3e29a97709
Make first file empty.
Wise, Catherine (Digital, Acton) <Catherine.Wise@csiro.au>
parents:
15
diff
changeset
|
181 # Empty parameter file. |
|
bc3e29a97709
Make first file empty.
Wise, Catherine (Digital, Acton) <Catherine.Wise@csiro.au>
parents:
15
diff
changeset
|
182 open(file_name, 'w' ).close() |
|
bc3e29a97709
Make first file empty.
Wise, Catherine (Digital, Acton) <Catherine.Wise@csiro.au>
parents:
15
diff
changeset
|
183 |
| 0 | 184 if root is not None: |
| 185 with open(metadata_path, 'wb') as metadata_file: | |
| 186 for path in paths: | |
| 187 locations = utils.Navigation.files_at_path(root, path) | |
| 188 if not locations or locations == []: | |
| 189 print("Path not found: " + path) | |
| 190 print("In root: " + str(root)) | |
| 191 else: | |
| 192 print("Downloading file..." + ", ".join([loc.name for loc in locations])) | |
| 193 for location in locations: | |
| 194 filename = "".join(c in SAFE_CHARS and c or '-' for c in location.name) | |
|
3
34bdad74ec64
Fix json output again.
Catherine Wise <catherine.wise@csiro.au>
parents:
2
diff
changeset
|
195 extension = os.path.splitext(filename)[1].strip(".") |
| 0 | 196 name = construct_multi_filename(hda_id, filename, extension) |
| 197 target_output_filename = None | |
|
3
34bdad74ec64
Fix json output again.
Catherine Wise <catherine.wise@csiro.au>
parents:
2
diff
changeset
|
198 data_type = "new_primary_dataset" |
|
15
8b88de25dd2c
More.
Wise, Catherine (Digital, Acton) <Catherine.Wise@csiro.au>
parents:
14
diff
changeset
|
199 target_output_filename = os.path.normpath(os.path.join(extra_files_path, name)) |
| 0 | 200 download = truststore.getFile(store, location) |
| 201 if download is None: | |
| 202 print("File %s not found." % location.name) | |
| 203 sys.exit(4) | |
|
17
eefb47f4f438
Try again.
Wise, Catherine (Digital, Acton) <Catherine.Wise@csiro.au>
parents:
16
diff
changeset
|
204 if first: |
|
eefb47f4f438
Try again.
Wise, Catherine (Digital, Acton) <Catherine.Wise@csiro.au>
parents:
16
diff
changeset
|
205 first = False |
|
eefb47f4f438
Try again.
Wise, Catherine (Digital, Acton) <Catherine.Wise@csiro.au>
parents:
16
diff
changeset
|
206 target_output_filename = download |
| 0 | 207 ungzip(download, target_output_filename) |
|
14
c7287129f37f
And again.
Wise, Catherine (Digital, Acton) <Catherine.Wise@csiro.au>
parents:
13
diff
changeset
|
208 # ext = sniff.handle_uploaded_dataset_file(target_output_filename, datatypes_registry, ext=ext) |
|
c7287129f37f
And again.
Wise, Catherine (Digital, Acton) <Catherine.Wise@csiro.au>
parents:
13
diff
changeset
|
209 # print("Guessed file type: " + ext) |
|
c7287129f37f
And again.
Wise, Catherine (Digital, Acton) <Catherine.Wise@csiro.au>
parents:
13
diff
changeset
|
210 print("in file: " + properties_file) |
|
11
9ada74e8915e
Try again.
Wise, Catherine (Digital, Acton) <Catherine.Wise@csiro.au>
parents:
10
diff
changeset
|
211 metadata_file.write( |
|
14
c7287129f37f
And again.
Wise, Catherine (Digital, Acton) <Catherine.Wise@csiro.au>
parents:
13
diff
changeset
|
212 metadata_to_json(dataset_id, target_output_filename, name, extension, data_type)) |
| 0 | 213 else: |
| 214 print("Store is damaged or we don't have sufficient access.") | |
| 215 sys.exit(4) | |
| 216 | |
| 217 | |
| 218 if __name__ == '__main__': | |
| 219 main() |