Mercurial > repos > cathywise > truststore_browse_testing
changeset 0:2a996fea49a3
Initial.
| author | Catherine Wise <catherine.wise@csiro.au> |
|---|---|
| date | Thu, 07 May 2015 12:34:06 +1000 |
| parents | |
| children | 4da92580c0a3 |
| files | TrustStoreGalaxyBrowse.py TrustStoreGalaxyBrowse.xml tool_dependencies.xml |
| diffstat | 3 files changed, 197 insertions(+), 0 deletions(-) [+] |
line wrap: on
line diff
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/TrustStoreGalaxyBrowse.py Thu May 07 12:34:06 2015 +1000 @@ -0,0 +1,134 @@ +from __future__ import division, absolute_import, print_function, unicode_literals +import sys +import shutil +import gzip +import tempfile +import os +from py_ts import TrustStoreClient, utils +# from galaxy.datatypes.checkers import * + +def printNice(elem, f, depth): + try: + f.write('\t'*depth + elem.name + " (" + str(len(elem.fragments)) + " parts)\n") + except AttributeError: + f.write('\t'*depth + elem.name + "\n") + for child in elem.children: + printNice(child, f, depth+1) + +def check_gzip(file_path): + try: + temp = open( file_path, "U" ) + magic_check = temp.read( 2 ) + temp.close() + if magic_check != util.gzip_magic: + return ( False, False ) + except: + return ( False, False ) + return ( True, True ) + +def ungzip(download, outputFile): + is_gzipped, is_valid = check_gzip(download) + + if is_gzipped and not is_valid: + print "File is compressed (gzip) but not valid." + sys.exit(4) + elif is_gzipped and is_valid: + # We need to uncompress the temp_name file, but BAM files must remain compressed in the BGZF format + CHUNK_SIZE = 2**20 # 1Mb + fd, uncompressed = tempfile.mkstemp(prefix='data_id_upload_gunzip_', dir=os.path.dirname(outputFile), text=False ) + gzipped_file = gzip.GzipFile(download, 'rb') + while 1: + try: + chunk = gzipped_file.read(CHUNK_SIZE) + except IOError: + os.close(fd) + os.remove(uncompressed) + print 'Problem decompressing gzipped data', dataset, json_file + sys.exit(4) + if not chunk: + break + os.write(fd, chunk) + os.close(fd) + gzipped_file.close() + + shutil.copy(uncompressed, outputFile) + try: + os.remove(uncompressed) + os.remove(download) + except OSError: + pass + else: + shutil.copy(download, outputFile) + +if __name__ == '__main__': + + kms_url = "https://tstest-kms.it.csiro.au/kmscolab_3_0" + ims_url = "https://tstest-ims.it.csiro.au/ims_3_0/services/IMS" + username = sys.argv[1] + password = sys.argv[2] + client_key = "desktop" + client_secret = "cpU92F1PT7VOCANjSknuCDp4DrubmujoBaF6b0miz8OpKNokEbGMHCaSFK5/lISbBmaaGVCgeADI2A39F3Hkeg==" + storename = sys.argv[3] + path = sys.argv[4] + fileType = sys.argv[5] + filename = "" + outputFile = "" + if len(sys.argv) > 9: + filename = sys.argv[6] + outputFile = sys.argv[7] + outputFileId = sys.argv[8] + otherFilesDir = sys.argv[9] + else: + outputFile = sys.argv[6] + + config = TrustStoreClient.Config(ims_url, kms_url, client_key, client_secret, tmpDir='/mnt/galaxy/tmp') + ts = TrustStoreClient.TrustStoreClient(headless=False, config=config) + try: + ts.authenticate(username, password) + except TrustStoreClient.TrustStoreClientAuthenticationException as e: + print e + sys.exit(5) + ts.getPrivateKey('privkey.pem') + listing = ts.listStores() + found = False + for store in listing: + if store.friendly_name == storename: + found = True + root = ts.listDirectory(store) + location = None + if path != "/": + location = ts_utils.ts_utils.dirAtPath(root, path) + if not location: + print "Path not found" + sys.exit(3) + else: + location = root + if filename and filename != "": + outputFileList = [outputFile] + inputFileList = None + if "," in filename: # we have multiple files guys. + inputFileList = filename.split(",") + for inputFile in inputFileList[1:]: # First file will be sent to outputFile. + outName = "%s_%s_%s_%s_%s" % ('primary', outputFileId, inputFile.replace(".","-"), 'visible', fileType) + outputFileList.append(os.path.join(otherFilesDir, outName)) + else: + inputFileList = [filename] + for inFile, outFile in zip(inputFileList, outputFileList): + downloadMe = ts_utils.ts_utils.recurseToChildNamed(location, inFile) + if downloadMe: + download = ts.getFile(store, downloadMe) + ungzip(download, outFile) + else: + print "File %s not found" % inFile + sys.exit(4) + else: + with open(outputFile, 'w+') as f: + try: + for child in root.children: + printNice(child, f, 0) + except AttributeError as e: + print e + print root + if not found: + print "Store %s not found" % storename + sys.exit(2)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/TrustStoreGalaxyBrowse.xml Thu May 07 12:34:06 2015 +1000 @@ -0,0 +1,35 @@ +<tool id="ts_import_2" name="Browse Data from TrustStore" version="0.5" force_history_refresh="True"> <!-- tool_type="data_source"> --> + <description> secure cloud storage.</description> + <stdio> + <exit_code range="2" err_level="fatal" description="Could not find store with supplied name." /> + <exit_code range="3" err_level="fatal" description="Could not find supplied path in store." /> + <exit_code range="4" err_level="fatal" description="Could not find supplied filename in store at path." /> + <exit_code range="5" err_level="fatal" description="Bad credentials supplied. See stdout for more information." /> + </stdio> + <command interpreter="python">TrustStoreGalaxyImport.py $url</command> + <inputs action="http://tstest-kms.it.csiro.au/kmscolab_3_0/galaxy_start" check_values="false" method="get"> + <display>go to TrustStore server</display> + <!--<param name="appCallbackUrl" type="baseurl" value="/tool_runner?tool_id=ts_import_2" />--> + <param name="GALAXY_URL" type="baseurl" value="/tool_runner/ts_import_2" /> + </inputs> + <request_param_translation> + <request_param galaxy_name="url" remote_name="url" missing="" /> + </request_param_translation> + <outputs> + <data name="output1" format="auto" /> + </outputs> + <uihints minwidth="800"/> + <options sanitize="False" refresh="True"/> + + <requirements> + <requirement type="package" version="0.5">py_ts</requirement> + <requirement type="python_module">py_ts</requirement> + <requirement type="binary" version="1.0.0">openssl</requirement> + </requirements> + + <help> +This tool lets users import data they have uploaded to the TrustStore system. + +Upload files to TrustStore using the desktop client. + </help> +</tool> \ No newline at end of file
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tool_dependencies.xml Thu May 07 12:34:06 2015 +1000 @@ -0,0 +1,28 @@ +<?xml version="1.0"?> +<tool_dependency> + <package name="py_ts" version="0.3"> + <readme> + I should write this. + </readme> + <install version="1.0"> + <actions> + <action type="setup_virtualenv"> +requests +requests-oauthlib +boto +simplejson +passlib +pycrypto >= 2.6 +psycopg2 + </action> + <action type="download_file">https://tstest-kms.it.csiro.au/truststore/downloads/PythonTrustStore-0.5.tar.gz</action> + <action type="shell_command">pip install --install-option="--home=$INSTALL_DIR" --install-option="--install-scripts=$INSTALL_DIR/bin" PythonTrustStore-0.5.tar.gz</action> + <action type="make_directory">$INSTALL_DIR/lib/python</action> + <action type="set_environment"> + <environment_variable name="PYTHONPATH" action="prepend_to">$INSTALL_DIR/lib/python</environment_variable> + <environment_variable name="PATH" action="prepend_to">$INSTALL_DIR/bin</environment_variable> + </action> + </actions> + </install> + </package> +</tool_dependency>