Mercurial > repos > cathywise > truststore_browse
annotate TrustStoreGalaxyBrowse.py @ 9:3e8bd0d01725
Attempt to guess data type.
| author | Wise, Catherine (Digital, Acton) <Catherine.Wise@csiro.au> |
|---|---|
| date | Thu, 25 Jun 2015 08:26:34 +1000 |
| parents | 2ca750b9083c |
| children | 7301c2e96fce |
| rev | line source |
|---|---|
| 0 | 1 """TrustStore downloaded for Galaxy.""" |
| 2 from __future__ import division, absolute_import, print_function, unicode_literals | |
| 3 import sys | |
| 4 import shutil | |
| 5 import gzip | |
| 6 import tempfile | |
| 7 import os | |
| 8 import json | |
| 9 import operator | |
| 10 import urlparse | |
| 11 from py_ts import TrustStoreClient, utils | |
| 12 from galaxy.datatypes.checkers import util | |
|
9
3e8bd0d01725
Attempt to guess data type.
Wise, Catherine (Digital, Acton) <Catherine.Wise@csiro.au>
parents:
4
diff
changeset
|
13 from galaxy.datatypes import sniff |
|
3e8bd0d01725
Attempt to guess data type.
Wise, Catherine (Digital, Acton) <Catherine.Wise@csiro.au>
parents:
4
diff
changeset
|
14 from galaxy.datatypes.registry import Registry |
| 0 | 15 |
| 16 # Tell urllib3 to use pyOpenSSL because we are on old Python stdlib. | |
| 17 # import urllib3.contrib.pyopenssl | |
| 18 # urllib3.contrib.pyopenssl.inject_into_urllib3() | |
| 19 # | |
| 20 os.environ["REQUESTS_CA_BUNDLE"] = "/etc/ssl/certs/ca-certificates.crt" | |
| 21 | |
| 22 CLIENT_KEY = "desktop" | |
| 23 CLIENT_SECRET = "cpU92F1PT7VOCANjSknuCDp4DrubmujoBaF6b0miz8OpKNokEbGMHCaSFK5/lISbBmaaGVCgeADI2A39F3Hkeg==" | |
| 24 CHUNK_SIZE = 2**20 # 1Mb | |
| 25 SAFE_CHARS = '.-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ ' | |
| 26 | |
| 27 def print_nice(elem, f, depth): | |
| 28 """Print the file name.""" | |
| 29 try: | |
| 30 f.write('\t'*depth + elem.name + " (" + str(len(elem.fragments)) + " parts)\n") | |
| 31 except AttributeError: | |
| 32 f.write('\t'*depth + elem.name + "\n") | |
| 33 for child in elem.children: | |
| 34 print_nice(child, f, depth+1) | |
| 35 | |
| 36 def check_gzip(file_path): | |
| 37 """Check if file is gziped.""" | |
| 38 try: | |
| 39 temp = open(file_path, "U") | |
| 40 magic_check = temp.read(2) | |
| 41 temp.close() | |
| 42 if magic_check != util.gzip_magic: | |
| 43 return (False, False) | |
| 44 except Exception: | |
| 45 return (False, False) | |
| 46 return (True, True) | |
| 47 | |
| 48 def ungzip(download, outputFile): | |
| 49 """Uncompress file.""" | |
| 50 is_gzipped, is_valid = check_gzip(download) | |
| 51 | |
| 52 if is_gzipped and not is_valid: | |
| 53 print("File is compressed (gzip) but not valid.") | |
| 54 sys.exit(4) | |
| 55 elif is_gzipped and is_valid: | |
| 56 # We need to uncompress the temp_name file, but BAM files must | |
| 57 # remain compressed in the BGZF format | |
| 58 file_handle, uncompressed = tempfile.mkstemp(prefix='data_id_upload_gunzip_', dir=os.path.dirname(outputFile), text=False ) | |
| 59 gzipped_file = gzip.GzipFile(download, 'rb') | |
| 60 while 1: | |
| 61 try: | |
| 62 chunk = gzipped_file.read(CHUNK_SIZE) | |
| 63 except IOError: | |
| 64 os.close(file_handle) | |
| 65 os.remove(uncompressed) | |
| 66 print('Problem decompressing gzipped data %s %s' % (download, outputFile)) | |
| 67 sys.exit(4) | |
| 68 if not chunk: | |
| 69 break | |
| 70 os.write(file_handle, chunk) | |
| 71 os.close(file_handle) | |
| 72 gzipped_file.close() | |
| 73 | |
| 74 shutil.copy(uncompressed, outputFile) | |
| 75 try: | |
| 76 os.remove(uncompressed) | |
| 77 os.remove(download) | |
| 78 except OSError: | |
| 79 pass | |
| 80 else: | |
| 81 shutil.copy(download, outputFile) | |
| 82 | |
| 83 def construct_multi_filename(id, name, file_type): | |
| 84 """ Implementation of *Number of Output datasets cannot be determined until | |
| 85 tool run* from documentation_. | |
| 86 .. _documentation: http://wiki.galaxyproject.org/Admin/Tools/Multiple%20Output%20Files | |
| 87 From https://github.com/mdshw5/galaxy-json-data-source/blob/master/json_data_source.py | |
| 88 """ | |
| 89 filename = "%s_%s_%s_%s_%s" % ('primary', id, name, 'visible', file_type) | |
| 90 return filename | |
| 91 | |
| 92 def metadata_to_json(dataset_id, filename, name, extesion, ds_type='dataset', primary=False): | |
| 93 """ Return line separated JSON | |
| 94 From https://github.com/mdshw5/galaxy-json-data-source/blob/master/json_data_source.py | |
| 95 """ | |
| 96 meta_dict = dict(type=ds_type, | |
| 97 ext=extesion, | |
| 98 filename=filename, | |
| 99 name=name, | |
| 100 metadata={}) | |
| 101 if primary: | |
| 102 meta_dict['base_dataset_id'] = dataset_id | |
| 103 else: | |
| 104 meta_dict['dataset_id'] = dataset_id | |
| 105 return "%s\n" % json.dumps(meta_dict) | |
| 106 | |
| 107 def main(): | |
| 108 properties_file = sys.argv[1] | |
| 109 json_params = None | |
| 110 metadata_path = None | |
| 111 all_params = None | |
| 112 with open(properties_file, 'r') as file_: | |
| 113 settings = file_.read() | |
| 114 all_params = json.loads(settings) | |
| 115 json_params = all_params.get("param_dict") | |
| 116 metadata_path = all_params["job_config"]["TOOL_PROVIDED_JOB_METADATA_FILE"] | |
| 117 | |
| 118 output_filename = json_params.get('output', None) | |
| 119 output_data = all_params.get('output_data') | |
| 120 extra_files_path, file_name, ext, out_data_name, hda_id, dataset_id = \ | |
| 121 operator.itemgetter('extra_files_path', 'file_name', 'ext', 'out_data_name', 'hda_id', 'dataset_id')(output_data[0]) | |
|
4
2ca750b9083c
Fix extra files directory.
Catherine Wise <catherine.wise@csiro.au>
parents:
3
diff
changeset
|
122 extra_files_path = json_params['__new_file_path__'] |
| 0 | 123 |
|
9
3e8bd0d01725
Attempt to guess data type.
Wise, Catherine (Digital, Acton) <Catherine.Wise@csiro.au>
parents:
4
diff
changeset
|
124 datatypes_registry = Registry() |
|
3e8bd0d01725
Attempt to guess data type.
Wise, Catherine (Digital, Acton) <Catherine.Wise@csiro.au>
parents:
4
diff
changeset
|
125 datatypes_registry.load_datatypes( |
|
3e8bd0d01725
Attempt to guess data type.
Wise, Catherine (Digital, Acton) <Catherine.Wise@csiro.au>
parents:
4
diff
changeset
|
126 root_dir=all_params['job_config']['GALAXY_ROOT_DIR'], |
|
3e8bd0d01725
Attempt to guess data type.
Wise, Catherine (Digital, Acton) <Catherine.Wise@csiro.au>
parents:
4
diff
changeset
|
127 config=all_params['job_config']['GALAXY_DATATYPES_CONF_FILE'] |
|
3e8bd0d01725
Attempt to guess data type.
Wise, Catherine (Digital, Acton) <Catherine.Wise@csiro.au>
parents:
4
diff
changeset
|
128 ) |
|
3e8bd0d01725
Attempt to guess data type.
Wise, Catherine (Digital, Acton) <Catherine.Wise@csiro.au>
parents:
4
diff
changeset
|
129 |
| 0 | 130 url_params = urlparse.unquote(json_params['URL']).split(";") |
| 131 if len(url_params) < 3: | |
| 132 print("The url we got back is malformed: "+ json_params['URL']) | |
| 133 sys.exit(5) | |
| 134 short_url = url_params[0] | |
| 135 username = url_params[1] | |
| 136 password = url_params[2] | |
| 137 if "/short" not in short_url: | |
| 138 print("The url we got back is malformed: " + json_params['URL']) | |
| 139 sys.exit(5) | |
| 140 kms_url = short_url.split("/short")[0] | |
| 141 | |
| 142 tmp_dir = '/mnt/galaxy/tmp' | |
| 143 tmp = None | |
| 144 if os.path.exists(tmp_dir): | |
| 145 tmp = tmp_dir | |
| 146 | |
| 147 config = TrustStoreClient.Config( | |
| 148 None, kms_url, CLIENT_KEY, CLIENT_SECRET, tmpDir=tmp) | |
| 149 truststore = TrustStoreClient.TrustStoreClient(headless=False, config=config) | |
| 150 try: | |
| 151 truststore.authenticate(username, password) | |
| 152 except TrustStoreClient.TrustStoreClientAuthenticationException as err: | |
| 153 print(err) | |
| 154 sys.exit(5) | |
| 155 truststore.getPrivateKey('privkey.pem') | |
| 156 | |
| 157 path_texts = truststore.lengthenPath(short_url) | |
| 158 if len(path_texts) < 2: | |
| 159 print("The path we got was malformed: " + str(path_texts)) | |
| 160 sys.exit(3) | |
| 161 paths = path_texts[1:] | |
| 162 store_id = path_texts[0] | |
| 163 | |
| 164 store = truststore.getStoreByID(store_id) | |
| 165 if store is None: | |
| 166 print("Coudn't find store with that ID, or don't have access.") | |
| 167 sys.exit(2) | |
| 168 root = truststore.listDirectory(store) | |
| 169 | |
| 170 first = True | |
| 171 | |
| 172 print("Preparing the following for downloading: " + str(paths)) | |
| 173 | |
| 174 if root is not None: | |
| 175 with open(metadata_path, 'wb') as metadata_file: | |
| 176 for path in paths: | |
| 177 locations = utils.Navigation.files_at_path(root, path) | |
| 178 if not locations or locations == []: | |
| 179 print("Path not found: " + path) | |
| 180 print("In root: " + str(root)) | |
| 181 else: | |
| 182 print("Downloading file..." + ", ".join([loc.name for loc in locations])) | |
| 183 for location in locations: | |
| 184 filename = "".join(c in SAFE_CHARS and c or '-' for c in location.name) | |
|
3
34bdad74ec64
Fix json output again.
Catherine Wise <catherine.wise@csiro.au>
parents:
2
diff
changeset
|
185 extension = os.path.splitext(filename)[1].strip(".") |
| 0 | 186 name = construct_multi_filename(hda_id, filename, extension) |
| 187 target_output_filename = None | |
|
3
34bdad74ec64
Fix json output again.
Catherine Wise <catherine.wise@csiro.au>
parents:
2
diff
changeset
|
188 data_type = "new_primary_dataset" |
| 0 | 189 if first: |
| 190 target_output_filename = file_name | |
|
2
3ff3e9b8794f
Fix output json format.
Catherine Wise <catherine.wise@csiro.au>
parents:
0
diff
changeset
|
191 dataset = "new_primary_dataset" |
| 0 | 192 first = False |
| 193 else: | |
|
3
34bdad74ec64
Fix json output again.
Catherine Wise <catherine.wise@csiro.au>
parents:
2
diff
changeset
|
194 target_output_filename = os.path.normpath(os.path.join(extra_files_path, name)) |
|
9
3e8bd0d01725
Attempt to guess data type.
Wise, Catherine (Digital, Acton) <Catherine.Wise@csiro.au>
parents:
4
diff
changeset
|
195 ext = sniff.handle_uploaded_dataset_file(filename, datatypes_registry, ext=ext) |
| 0 | 196 metadata_file.write( |
|
9
3e8bd0d01725
Attempt to guess data type.
Wise, Catherine (Digital, Acton) <Catherine.Wise@csiro.au>
parents:
4
diff
changeset
|
197 metadata_to_json(dataset_id, target_output_filename, name, ext, data_type)) |
| 0 | 198 download = truststore.getFile(store, location) |
| 199 if download is None: | |
| 200 print("File %s not found." % location.name) | |
| 201 sys.exit(4) | |
| 202 ungzip(download, target_output_filename) | |
| 203 else: | |
| 204 print("Store is damaged or we don't have sufficient access.") | |
| 205 sys.exit(4) | |
| 206 | |
| 207 | |
| 208 if __name__ == '__main__': | |
| 209 main() |