Mercurial > repos > eric-rasche > apollo
changeset 5:7610987e0c48 draft
planemo upload for repository https://github.com/TAMU-CPT/galaxy-webapollo commit 29795b77c0d5c7894219b018a92c5ee7818096c3
| author | eric-rasche |
|---|---|
| date | Wed, 01 Mar 2017 22:39:58 -0500 |
| parents | 23ead6905145 |
| children | 8f76685cdfc8 |
| files | LICENSE create_features_from_gff3.py create_features_from_gff3.xml create_or_update_organism.py create_or_update_organism.xml delete_features.py delete_features.xml delete_organism.py delete_organism.xml export.py fetch_organism_jbrowse.xml json2iframe.py list_organisms.py macros.xml webapollo.py |
| diffstat | 15 files changed, 988 insertions(+), 52 deletions(-) [+] |
line wrap: on
line diff
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/LICENSE Wed Mar 01 22:39:58 2017 -0500 @@ -0,0 +1,32 @@ +Based on http://opensource.org/licenses/MIT + +This is a template. Complete and ship as file LICENSE the following 2 +lines (only) + +YEAR: +COPYRIGHT HOLDER: + +and specify as + +License: MIT + file LICENSE + +Copyright (c) <YEAR>, <COPYRIGHT HOLDER> + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/create_features_from_gff3.py Wed Mar 01 22:39:58 2017 -0500 @@ -0,0 +1,99 @@ +#!/usr/bin/env python +import sys +import json +import time +import argparse +from webapollo import WebApolloInstance, featuresToFeatureSchema +from webapollo import WAAuth, OrgOrGuess, GuessOrg, AssertUser +from BCBio import GFF +import logging +logging.basicConfig(level=logging.INFO) +log = logging.getLogger(__name__) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Sample script to add an attribute to a feature via web services') + WAAuth(parser) + parser.add_argument('email', help='User Email') + OrgOrGuess(parser) + + parser.add_argument('gff3', type=argparse.FileType('r'), help='GFF3 file') + args = parser.parse_args() + + wa = WebApolloInstance(args.apollo, args.username, args.password) + # User must have an account + gx_user = AssertUser(wa.users.loadUsers(email=args.email)) + + # Get organism + org_cn = GuessOrg(args, wa) + if isinstance(org_cn, list): + org_cn = org_cn[0] + + # TODO: Check user perms on org. + org = wa.organisms.findOrganismByCn(org_cn) + + bad_quals = ['date_creation', 'source', 'owner', 'date_last_modified', 'Name', 'ID'] + + sys.stdout.write('# ') + sys.stdout.write('\t'.join(['Feature ID', 'Apollo ID', 'Success', 'Messages'])) + sys.stdout.write('\n') + + # print(wa.annotations.getFeatures()) + for rec in GFF.parse(args.gff3): + wa.annotations.setSequence(rec.id, org['id']) + for feature in rec.features: + # We can only handle genes right now + if feature.type != 'gene': + continue + # Convert the feature into a presentation that Apollo will accept + featureData = featuresToFeatureSchema([feature]) + + try: + # We're experiencing a (transient?) problem where gene_001 to + # gene_025 will be rejected. Thus, hardcode to a known working + # gene name and update later. + featureData[0]['name'] = 'gene_000' + # Extract CDS feature from the feature data, this will be used + # to set the CDS location correctly (apollo currently screwing + # this up (2.0.6)) + CDS = featureData[0]['children'][0]['children'] + CDS = [x for x in CDS if x['type']['name'] == 'CDS'][0]['location'] + # Create the new feature + newfeature = wa.annotations.addFeature(featureData, trustme=True) + # Extract the UUIDs that apollo returns to us + mrna_id = newfeature['features'][0]['uniquename'] + gene_id = newfeature['features'][0]['parent_id'] + # Sleep to give it time to actually persist the feature. Apollo + # is terrible about writing + immediately reading back written + # data. + time.sleep(1) + # Correct the translation start, but with strand specific log + if CDS['strand'] == 1: + wa.annotations.setTranslationStart(mrna_id, min(CDS['fmin'], CDS['fmax'])) + else: + wa.annotations.setTranslationStart(mrna_id, max(CDS['fmin'], CDS['fmax']) - 1) + + # Finally we set the name, this should be correct. + wa.annotations.setName(mrna_id, feature.qualifiers.get('product', ["Unknown"])[0]) + wa.annotations.setName(gene_id, feature.qualifiers.get('product', ["Unknown"])[0]) + + for (k, v) in feature.qualifiers.items(): + if k not in bad_quals: + # set qualifier + pass + + sys.stdout.write('\t'.join([ + feature.id, + gene_id, + 'success', + "Dropped qualifiers: %s" % (json.dumps({k: v for (k, v) in feature.qualifiers.items() if k not in bad_quals})), + ])) + except Exception as e: + sys.stdout.write('\t'.join([ + feature.id, + '', + 'ERROR', + str(e) + ])) + + sys.stdout.write('\n')
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/create_features_from_gff3.xml Wed Mar 01 22:39:58 2017 -0500 @@ -0,0 +1,38 @@ +<?xml version="1.0"?> +<tool id="edu.tamu.cpt2.webapollo.feat_from_gff3" name="GFF3 to Apollo Annotations" version="0.9"> + <description></description> + <macros> + <import>macros.xml</import> + </macros> + <code file="webapollo.py"/> + <expand macro="requirements"/> + <command detect_errors="aggressive"><![CDATA[ +python $__tool_directory__/create_features_from_gff3.py +@ADMIN_AUTH@ +@ORG_OR_GUESS@ + +"$__user_email__" +$gff3_data +> $output]]></command> + <inputs> + <expand macro="org_or_guess" /> + <expand macro="gff3_input" /> + </inputs> + <outputs> + <data format="tabular" name="output" label="Process and Error Log"/> + </outputs> + <help><![CDATA[ +**NOTA BENE** + +This is **incredibly, highly experimental** + +DO NOT: + +- Run on gff3 referencing multiple reference sequences/contigs +- Expect it to work well +- Expect it to work at all + +@REFERENCES@ +]]></help> +</tool> +
--- a/create_or_update_organism.py Thu Jan 12 11:53:44 2017 -0500 +++ b/create_or_update_organism.py Wed Mar 01 22:39:58 2017 -0500 @@ -1,4 +1,5 @@ #!/usr/bin/env python +import sys import json import argparse import time @@ -9,7 +10,7 @@ if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Sample script to add an attribute to a feature via web services') + parser = argparse.ArgumentParser(description='Create or update an organism in an Apollo instance') WAAuth(parser) parser.add_argument('jbrowse', help='JBrowse Data Directory') @@ -18,6 +19,7 @@ parser.add_argument('--genus', help='Organism Genus') parser.add_argument('--species', help='Organism Species') parser.add_argument('--public', action='store_true', help='Make organism public') + parser.add_argument('--group', help='Give access to a user group') args = parser.parse_args() wa = WebApolloInstance(args.apollo, args.username, args.password) @@ -35,8 +37,17 @@ except Exception: org = None - # TODO: Check ownership if org: + has_perms = False + for user_owned_organism in gx_user.organismPermissions: + if 'WRITE' in user_owned_organism['permissions']: + has_perms = True + break + + if not has_perms: + print("Naming Conflict. You do not have permissions to access this organism. Either request permission from the owner, or choose a different name for your organism.") + sys.exit(2) + log.info("\tUpdating Organism") data = wa.organisms.updateOrganismInfo( org['id'], @@ -47,6 +58,8 @@ species=args.species, public=args.public ) + time.sleep(2) + data = [wa.organisms.findOrganismById(org['id'])] else: # New organism log.info("\tAdding Organism") @@ -68,5 +81,12 @@ read=True, ) + # Group access + if args.group: + group = wa.groups.loadGroupByName(name=args.group) + res = wa.groups.updateOrganismPermission(group, org_cn, + administrate=False, write=True, read=True, + export=True) + data = [o for o in data if o['commonName'] == org_cn] - print json.dumps(data, indent=2) + print(json.dumps(data, indent=2))
--- a/create_or_update_organism.xml Thu Jan 12 11:53:44 2017 -0500 +++ b/create_or_update_organism.xml Wed Mar 01 22:39:58 2017 -0500 @@ -1,5 +1,5 @@ <?xml version="1.0"?> -<tool id="edu.tamu.cpt2.webapollo.create_or_update" name="Create or Update Organism" version="3.0"> +<tool id="edu.tamu.cpt2.webapollo.create_or_update" name="Create or Update Organism" version="3.1"> <description>will create the organism if it doesn't exist, and update otherwise</description> <macros> <import>macros.xml</import> @@ -15,6 +15,7 @@ --genus "$genus" --species "$species" +--group '${group}' $public @ORG_OR_GUESS@ @@ -30,6 +31,7 @@ <param name="genus" type="text" label="Genus" optional="False" /> <param name="species" type="text" label="Species" optional="True" /> <param name="public" type="boolean" truevalue="--public" falsevalue="" label="Is Organism Public" /> + <param name="group" type="select" dynamic_options="galaxy_list_groups(__trans__)" label="Grant access to a user group" optional="True" /> </inputs> <outputs> <data format="json" name="output"/>
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/delete_features.py Wed Mar 01 22:39:58 2017 -0500 @@ -0,0 +1,39 @@ +#!/usr/bin/env python +import argparse +from webapollo import WebApolloInstance +from webapollo import WAAuth, OrgOrGuess, GuessOrg, AssertUser +import logging +logging.basicConfig(level=logging.INFO) +log = logging.getLogger(__name__) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Sample script to delete all features from an organism') + WAAuth(parser) + parser.add_argument('email', help='User Email') + OrgOrGuess(parser) + + args = parser.parse_args() + + wa = WebApolloInstance(args.apollo, args.username, args.password) + # User must have an account + gx_user = AssertUser(wa.users.loadUsers(email=args.email)) + + # Get organism + org_cn = GuessOrg(args, wa) + if isinstance(org_cn, list): + org_cn = org_cn[0] + + # TODO: Check user perms on org. + org = wa.organisms.findOrganismByCn(org_cn) + + # Call setSequence to tell apollo which organism we're working with + wa.annotations.setSequence(org['commonName'], org['id']) + # Then get a list of features. + features = wa.annotations.getFeatures() + # For each feature in the features + for feature in features['features']: + # We see that deleteFeatures wants a uniqueName, and so we pass + # is the uniquename field in the feature. + print(wa.annotations.deleteFeatures([feature['uniquename']])) +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/delete_features.xml Wed Mar 01 22:39:58 2017 -0500 @@ -0,0 +1,60 @@ +<?xml version="1.0"?> +<tool id="edu.tamu.cpt2.webapollo.delete_features" name="Delete all annotations from an Apollo record" version="1.2"> + <description></description> + <macros> + <import>macros.xml</import> + </macros> + <code file="webapollo.py"/> + <expand macro="requirements"/> + <command detect_errors="aggressive"><![CDATA[ +#if str($ask_one) == "yes": + #if str($ask_two) == "yes": + ## Nope, still don't trust them to not be dumb (or malicious), so we backup first. + python $__tool_directory__/export.py + @ADMIN_AUTH@ + @ORG_OR_GUESS@ + --gff "$gff_out" + --fasta "$fasta_out" + --json "$json_out"; + + ## Now we delete + python $__tool_directory__/delete_features.py + @ADMIN_AUTH@ + @ORG_OR_GUESS@ + "$__user_email__" + > $output; + #else + echo "Nothing to do" > $output; + #end if +#else + echo "Nothing to do" > $output; +#end if + ]]></command> + <inputs> + <expand macro="org_or_guess" /> + <param name="ask_one" type="boolean" truevalue="yes" falsevalue="" label="Are you SURE you want to do this?" help="It will PERMANENTLY delete all of the features on this organism."/> + <param name="ask_two" type="boolean" truevalue="yes" falsevalue="" label="Are you really, really SURE you want to do this?" help="There's NO coming back from this."/> + </inputs> + <outputs> + <data format="tabular" name="output" label="Process and Error Log"/> + + <data format="gff3" name="gff_out" label="Annotations from Apollo" hidden="true"/> + <data format="fasta" name="fasta_out" label="Sequence(s) from Apollo" hidden="true"/> + <data format="json" name="json_out" label="Metadata from Apollo" hidden="true"/> + </outputs> + <help><![CDATA[ +**What it does** + +Deletes every single one of the annotations on an organism. Intentionally. + +**Why?** + +There are legitimate uses for this tool, generally re-opened genomes is a good +one. Needing to transfer annotations from one build of an organism to another +(with the same refseq name). + + +@REFERENCES@ +]]></help> +</tool> +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/delete_organism.py Wed Mar 01 22:39:58 2017 -0500 @@ -0,0 +1,39 @@ +#!/usr/bin/env python +import argparse +from webapollo import WebApolloInstance +from webapollo import WAAuth, OrgOrGuess, GuessOrg, AssertUser +import logging +logging.basicConfig(level=logging.INFO) +log = logging.getLogger(__name__) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Sample script to completely delete an organism') + WAAuth(parser) + parser.add_argument('email', help='User Email') + OrgOrGuess(parser) + + args = parser.parse_args() + + wa = WebApolloInstance(args.apollo, args.username, args.password) + # User must have an account + gx_user = AssertUser(wa.users.loadUsers(email=args.email)) + + # Get organism + org_cn = GuessOrg(args, wa) + if isinstance(org_cn, list): + org_cn = org_cn[0] + + # TODO: Check user perms on org. + org = wa.organisms.findOrganismByCn(org_cn) + + # Call setSequence to tell apollo which organism we're working with + wa.annotations.setSequence(org['commonName'], org['id']) + # Then get a list of features. + features = wa.annotations.getFeatures() + # For each feature in the features + for feature in features['features']: + # We see that deleteFeatures wants a uniqueName, and so we pass + # is the uniquename field in the feature. + print(wa.annotations.deleteFeatures([feature['uniquename']])) +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/delete_organism.xml Wed Mar 01 22:39:58 2017 -0500 @@ -0,0 +1,60 @@ +<?xml version="1.0"?> +<tool id="edu.tamu.cpt2.webapollo.delete_organism" name="Delete an Apollo record" version="1.0"> + <description></description> + <macros> + <import>macros.xml</import> + </macros> + <code file="webapollo.py"/> + <expand macro="requirements"/> + <command detect_errors="aggressive"><![CDATA[ +#if str($ask_one) == "yes": + #if str($ask_two) == "yes": + ## Nope, still don't trust them to not be dumb (or malicious), so we backup first. + python $__tool_directory__/export.py + @ADMIN_AUTH@ + @ORG_OR_GUESS@ + --gff "$gff_out" + --fasta "$fasta_out" + --json "$json_out"; + + ## Now we delete + python $__tool_directory__/delete_organism.py + @ADMIN_AUTH@ + @ORG_OR_GUESS@ + "$__user_email__" + > $output; + #else + echo "Nothing to do" > $output; + #end if +#else + echo "Nothing to do" > $output; +#end if + ]]></command> + <inputs> + <expand macro="org_or_guess" /> + <param name="ask_one" type="boolean" truevalue="yes" falsevalue="" label="Are you SURE you want to do this?" help="It will PERMANENTLY delete all of the features on this organism."/> + <param name="ask_two" type="boolean" truevalue="yes" falsevalue="" label="Are you really, really SURE you want to do this?" help="There's NO coming back from this."/> + </inputs> + <outputs> + <data format="tabular" name="output" label="Process and Error Log"/> + + <data format="gff3" name="gff_out" label="Annotations from Apollo" hidden="true"/> + <data format="fasta" name="fasta_out" label="Sequence(s) from Apollo" hidden="true"/> + <data format="json" name="json_out" label="Metadata from Apollo" hidden="true"/> + </outputs> + <help><![CDATA[ +**What it does** + +Deletes every single one of the annotations on an organism. Intentionally. + +**Why?** + +There are legitimate uses for this tool, generally re-opened genomes is a good +one. Needing to transfer annotations from one build of an organism to another +(with the same refseq name). + + +@REFERENCES@ +]]></help> +</tool> +
--- a/export.py Thu Jan 12 11:53:44 2017 -0500 +++ b/export.py Wed Mar 01 22:39:58 2017 -0500 @@ -1,6 +1,10 @@ #!/usr/bin/env python import sys -import StringIO +try: + import StringIO as io +except ImportError: + import io + import json import argparse from Bio import SeqIO @@ -11,7 +15,7 @@ def export(org_cn, seqs): org_data = wa.organisms.findOrganismByCn(org_cn) - data = StringIO.StringIO() + data = io.StringIO() kwargs = dict( exportType='GFF3', @@ -40,11 +44,12 @@ records = list(GFF.parse(data)) if len(records) == 0: - print "Could not find any sequences or annotations for this organism + reference sequence" + print("Could not find any sequences or annotations for this organism + reference sequence") sys.exit(2) else: for record in records: record.annotations = {} + record.features = sorted(record.features, key=lambda x: x.location.start) if args.gff: GFF.write([record], args.gff) record.description = "" @@ -53,6 +58,7 @@ return org_data + if __name__ == '__main__': parser = argparse.ArgumentParser(description='Sample script to add an attribute to a feature via web services') WAAuth(parser)
--- a/fetch_organism_jbrowse.xml Thu Jan 12 11:53:44 2017 -0500 +++ b/fetch_organism_jbrowse.xml Wed Mar 01 22:39:58 2017 -0500 @@ -56,11 +56,11 @@ Fetches the JBrowse directory from Apollo back into Galaxy. -**WARNINGS** +.. class:: warningmark -- If you provide an Apollo JSON file, only the first organism block will - be used in Common Name determination, as it is improbable you wish to - apply a single JBrowse instance to multiple organisms. +If you provide an Apollo JSON file, only the first organism block will be used +in Common Name determination, as it is improbable you wish to apply a single +JBrowse instance to multiple organisms. @REFERENCES@ ]]></help>
--- a/json2iframe.py Thu Jan 12 11:53:44 2017 -0500 +++ b/json2iframe.py Wed Mar 01 22:39:58 2017 -0500 @@ -6,7 +6,7 @@ if __name__ == '__main__': parser = argparse.ArgumentParser(description='Sample script to add an attribute to a feature via web services') parser.add_argument('apollo', help='Complete Apollo URL') - parser.add_argument('json', type=file, help='JSON Data') + parser.add_argument('json', type=argparse.FileType("r"), help='JSON Data') parser.add_argument('external_apollo_url') args = parser.parse_args() @@ -16,7 +16,15 @@ # This is base64 encoded to get past the toolshed's filters. HTML_TPL = """ -<html><head><title>Embedded Apollo Access</title><style type="text/css">body {{margin: 0;}} iframe {{border: 0;width: 100%;height: 100%}}</style></head><body><iframe src="{base_url}/annotator/loadLink?loc={chrom}&organism={orgId}&tracklist=1"></iframe></body></html> + <html> + <head> + <title>Embedded Apollo Access</title> + <style type="text/css">body {{margin: 0;}} iframe {{border: 0;width: 100%;height: 100%}}</style> + </head> + <body> + <iframe src="{base_url}/annotator/loadLink?loc={chrom}&organism={orgId}&tracklist=1"></iframe> + </body> + </html> """ # HTML_TPL = base64.b64decode(HTML_TPL.replace('\n', ''))
--- a/list_organisms.py Thu Jan 12 11:53:44 2017 -0500 +++ b/list_organisms.py Wed Mar 01 22:39:58 2017 -0500 @@ -4,7 +4,7 @@ from webapollo import WAAuth, WebApolloInstance, AssertUser, accessible_organisms if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Sample script to add an attribute to a feature via web services') + parser = argparse.ArgumentParser(description='List all organisms available in an Apollo instance') WAAuth(parser) parser.add_argument('email', help='User Email') args = parser.parse_args() @@ -16,4 +16,4 @@ orgs = accessible_organisms(gx_user, all_orgs) - print json.dumps(orgs, indent=2) + print(json.dumps(orgs, indent=2))
--- a/macros.xml Thu Jan 12 11:53:44 2017 -0500 +++ b/macros.xml Wed Mar 01 22:39:58 2017 -0500 @@ -5,6 +5,7 @@ <requirement type="package" version="2.7">python</requirement> <requirement type="package" version="1.65">biopython</requirement> <requirement type="package" version="0.6.2">bcbiogff</requirement> + <requirement type="package" version="2.12.4">requests</requirement> <yield/> </requirements> </xml> @@ -15,12 +16,6 @@ <token name="@URL@"> \$GALAXY_WEBAPOLLO_URL </token> - <token name="@USER_AUTH_REMOTE@"> -\$GALAXY_WEBAPOLLO_URL -$__user_email__ -"" ---remote_user REMOTE_USER - </token> <token name="@ADMIN_AUTH@"> \$GALAXY_WEBAPOLLO_URL \$GALAXY_WEBAPOLLO_USER @@ -67,7 +62,7 @@ <param name="org_select" type="select" dynamic_options="galaxy_list_orgs(__trans__)" label="Organism" /> </when> <when value="direct"> - <param name="org_raw" type="text" label="Organism Common Name" /> + <param name="org_raw" type="text" label="Organism Common Name" optional="False" /> </when> <when value="auto_json"> <param name="org_file" type="data" format="json" label="Apollo Organism File" help="Will only fetch first organism" />
--- a/webapollo.py Thu Jan 12 11:53:44 2017 -0500 +++ b/webapollo.py Wed Mar 01 22:39:58 2017 -0500 @@ -2,30 +2,415 @@ import json import os import collections -import StringIO +try: + import StringIO as io +except: + import io import logging +import time +import argparse +from abc import abstractmethod from BCBio import GFF from Bio import SeqIO logging.getLogger("requests").setLevel(logging.CRITICAL) log = logging.getLogger() +############################################# +###### BEGIN IMPORT OF CACHING LIBRARY ###### +############################################# +# This code is licensed under the MIT # +# License and is a copy of code publicly # +# available in rev. # +# e27332bc82f4e327aedaec17c9b656ae719322ed # +# of https://github.com/tkem/cachetools/ # +############################################# +class DefaultMapping(collections.MutableMapping): + + __slots__ = () + + @abstractmethod + def __contains__(self, key): # pragma: nocover + return False + + @abstractmethod + def __getitem__(self, key): # pragma: nocover + if hasattr(self.__class__, '__missing__'): + return self.__class__.__missing__(self, key) + else: + raise KeyError(key) + + def get(self, key, default=None): + if key in self: + return self[key] + else: + return default + + __marker = object() + + def pop(self, key, default=__marker): + if key in self: + value = self[key] + del self[key] + elif default is self.__marker: + raise KeyError(key) + else: + value = default + return value + + def setdefault(self, key, default=None): + if key in self: + value = self[key] + else: + self[key] = value = default + return value + +DefaultMapping.register(dict) + + +class _DefaultSize(object): + def __getitem__(self, _): + return 1 + + def __setitem__(self, _, value): + assert value == 1 + + def pop(self, _): + return 1 + + +class Cache(DefaultMapping): + """Mutable mapping to serve as a simple cache or cache base class.""" + + __size = _DefaultSize() + + def __init__(self, maxsize, missing=None, getsizeof=None): + if missing: + self.__missing = missing + if getsizeof: + self.__getsizeof = getsizeof + self.__size = dict() + self.__data = dict() + self.__currsize = 0 + self.__maxsize = maxsize + + def __repr__(self): + return '%s(%r, maxsize=%r, currsize=%r)' % ( + self.__class__.__name__, + list(self.__data.items()), + self.__maxsize, + self.__currsize, + ) + + def __getitem__(self, key): + try: + return self.__data[key] + except KeyError: + return self.__missing__(key) + + def __setitem__(self, key, value): + maxsize = self.__maxsize + size = self.getsizeof(value) + if size > maxsize: + raise ValueError('value too large') + if key not in self.__data or self.__size[key] < size: + while self.__currsize + size > maxsize: + self.popitem() + if key in self.__data: + diffsize = size - self.__size[key] + else: + diffsize = size + self.__data[key] = value + self.__size[key] = size + self.__currsize += diffsize + + def __delitem__(self, key): + size = self.__size.pop(key) + del self.__data[key] + self.__currsize -= size + + def __contains__(self, key): + return key in self.__data + + def __missing__(self, key): + value = self.__missing(key) + try: + self.__setitem__(key, value) + except ValueError: + pass # value too large + return value + + def __iter__(self): + return iter(self.__data) + + def __len__(self): + return len(self.__data) + + @staticmethod + def __getsizeof(value): + return 1 + + @staticmethod + def __missing(key): + raise KeyError(key) + + @property + def maxsize(self): + """The maximum size of the cache.""" + return self.__maxsize + + @property + def currsize(self): + """The current size of the cache.""" + return self.__currsize + + def getsizeof(self, value): + """Return the size of a cache element's value.""" + return self.__getsizeof(value) + + +class _Link(object): + + __slots__ = ('key', 'expire', 'next', 'prev') + + def __init__(self, key=None, expire=None): + self.key = key + self.expire = expire + + def __reduce__(self): + return _Link, (self.key, self.expire) + + def unlink(self): + next = self.next + prev = self.prev + prev.next = next + next.prev = prev + + +class _Timer(object): + + def __init__(self, timer): + self.__timer = timer + self.__nesting = 0 + + def __call__(self): + if self.__nesting == 0: + return self.__timer() + else: + return self.__time + + def __enter__(self): + if self.__nesting == 0: + self.__time = time = self.__timer() + else: + time = self.__time + self.__nesting += 1 + return time + + def __exit__(self, *exc): + self.__nesting -= 1 + + def __reduce__(self): + return _Timer, (self.__timer,) + + def __getattr__(self, name): + return getattr(self.__timer, name) + + +class TTLCache(Cache): + """LRU Cache implementation with per-item time-to-live (TTL) value.""" + + def __init__(self, maxsize, ttl, timer=time.time, missing=None, + getsizeof=None): + Cache.__init__(self, maxsize, missing, getsizeof) + self.__root = root = _Link() + root.prev = root.next = root + self.__links = collections.OrderedDict() + self.__timer = _Timer(timer) + self.__ttl = ttl + + def __contains__(self, key): + try: + link = self.__links[key] # no reordering + except KeyError: + return False + else: + return not (link.expire < self.__timer()) + + def __getitem__(self, key, cache_getitem=Cache.__getitem__): + try: + link = self.__getlink(key) + except KeyError: + expired = False + else: + expired = link.expire < self.__timer() + if expired: + return self.__missing__(key) + else: + return cache_getitem(self, key) + + def __setitem__(self, key, value, cache_setitem=Cache.__setitem__): + with self.__timer as time: + self.expire(time) + cache_setitem(self, key, value) + try: + link = self.__getlink(key) + except KeyError: + self.__links[key] = link = _Link(key) + else: + link.unlink() + link.expire = time + self.__ttl + link.next = root = self.__root + link.prev = prev = root.prev + prev.next = root.prev = link + + def __delitem__(self, key, cache_delitem=Cache.__delitem__): + cache_delitem(self, key) + link = self.__links.pop(key) + link.unlink() + if link.expire < self.__timer(): + raise KeyError(key) + + def __iter__(self): + root = self.__root + curr = root.next + while curr is not root: + # "freeze" time for iterator access + with self.__timer as time: + if not (curr.expire < time): + yield curr.key + curr = curr.next + + def __len__(self): + root = self.__root + curr = root.next + time = self.__timer() + count = len(self.__links) + while curr is not root and curr.expire < time: + count -= 1 + curr = curr.next + return count + + def __setstate__(self, state): + self.__dict__.update(state) + root = self.__root + root.prev = root.next = root + for link in sorted(self.__links.values(), key=lambda obj: obj.expire): + link.next = root + link.prev = prev = root.prev + prev.next = root.prev = link + self.expire(self.__timer()) + + def __repr__(self, cache_repr=Cache.__repr__): + with self.__timer as time: + self.expire(time) + return cache_repr(self) + + @property + def currsize(self): + with self.__timer as time: + self.expire(time) + return super(TTLCache, self).currsize + + @property + def timer(self): + """The timer function used by the cache.""" + return self.__timer + + @property + def ttl(self): + """The time-to-live value of the cache's items.""" + return self.__ttl + + def expire(self, time=None): + """Remove expired items from the cache.""" + if time is None: + time = self.__timer() + root = self.__root + curr = root.next + links = self.__links + cache_delitem = Cache.__delitem__ + while curr is not root and curr.expire < time: + cache_delitem(self, curr.key) + del links[curr.key] + next = curr.next + curr.unlink() + curr = next + + def clear(self): + with self.__timer as time: + self.expire(time) + Cache.clear(self) + + def get(self, *args, **kwargs): + with self.__timer: + return Cache.get(self, *args, **kwargs) + + def pop(self, *args, **kwargs): + with self.__timer: + return Cache.pop(self, *args, **kwargs) + + def setdefault(self, *args, **kwargs): + with self.__timer: + return Cache.setdefault(self, *args, **kwargs) + + def popitem(self): + """Remove and return the `(key, value)` pair least recently used that + has not already expired. + + """ + with self.__timer as time: + self.expire(time) + try: + key = next(iter(self.__links)) + except StopIteration: + raise KeyError('%s is empty' % self.__class__.__name__) + else: + return (key, self.pop(key)) + + if hasattr(collections.OrderedDict, 'move_to_end'): + def __getlink(self, key): + value = self.__links[key] + self.__links.move_to_end(key) + return value + else: + def __getlink(self, key): + value = self.__links.pop(key) + self.__links[key] = value + return value + + +############################################# +###### END IMPORT OF CACHING LIBRARY ###### +############################################# + +cache = TTLCache( + 100, # Up to 100 items + 5 * 60 # 5 minute cache life +) +userCache = TTLCache( + 2, # Up to 2 items + 60 # 1 minute cache life +) + +class UnknownUserException(Exception): + pass + def WAAuth(parser): parser.add_argument('apollo', help='Complete Apollo URL') parser.add_argument('username', help='WA Username') parser.add_argument('password', help='WA Password') - parser.add_argument('--remote_user', default='', help='If set, ignore password, set the header with the name supplied to this argument to the value of email') def OrgOrGuess(parser): - parser.add_argument('--org_json', type=file, help='Apollo JSON output, source for common name') + parser.add_argument('--org_json', type=argparse.FileType("r"), help='Apollo JSON output, source for common name') parser.add_argument('--org_raw', help='Common Name') parser.add_argument('--org_id', help='Organism ID') def CnOrGuess(parser): OrgOrGuess(parser) - parser.add_argument('--seq_fasta', type=file, help='Fasta file, IDs used as sequence sources') + parser.add_argument('--seq_fasta', type=argparse.FileType("r"), help='Fasta file, IDs used as sequence sources') parser.add_argument('--seq_raw', nargs='*', help='Sequence Names') @@ -63,9 +448,11 @@ def AssertUser(user_list): if len(user_list) == 0: - raise Exception("Unknown user. Please register first") + raise UnknownUserException() + elif len(user_list) == 1: + return user_list[0] else: - return user_list[0] + raise Exception("Too many users!") def AssertAdmin(user): @@ -81,6 +468,8 @@ self.apollo_url = url self.username = username self.password = password + # TODO: Remove after apollo 2.0.6. + self.clientToken = time.time() self.annotations = AnnotationsClient(self) self.groups = GroupsClient(self) @@ -93,6 +482,20 @@ def __str__(self): return '<WebApolloInstance at %s>' % self.apollo_url + def requireUser(self, email): + cacheKey = 'user-list' + try: + # Get the cached value + data = userCache[cacheKey] + except KeyError: + # If we hit a key error above, indicating that + # we couldn't find the key, we'll simply re-request + # the data + data = self.users.loadUsers() + userCache[cacheKey] = data + + return AssertUser([x for x in data if x.username == email]) + class GroupObj(object): def __init__(self, **kwargs): @@ -162,6 +565,7 @@ data.update({ 'username': self._wa.username, 'password': self._wa.password, + 'clientToken': self._wa.clientToken, }) r = requests.post(url, data=json.dumps(data), headers=headers, @@ -266,9 +670,22 @@ data.update(self._extra_data) return self.request('setSymbol', data) - def getComments(self, features): + def getComments(self, feature_id): data = { - 'features': features, + 'features': [{'uniquename': feature_id}], + } + data = self._update_data(data) + return self.request('getComments', data) + + def addComments(self, feature_id, comment): + #TODO: This is probably not great and will delete comments, if I had to guess... + data = { + 'features': [ + { + 'uniquename': feature_id, + 'comments': [comment] + } + ], } data = self._update_data(data) return self.request('getComments', data) @@ -297,8 +714,9 @@ if not trustme: raise NotImplementedError("Waiting on better docs from project. If you know what you are doing, pass trustme=True to this function.") - data = {} - data.update(feature) + data = { + 'features': feature, + } data = self._update_data(data) return self.request('addFeature', data) @@ -465,12 +883,32 @@ } return self.request('getOrganismPermissionsForGroup', data) + def loadGroup(self, group): + return self.loadGroupById(group.groupId) + + def loadGroupById(self, groupId): + res = self.request('loadGroups', {'groupId': groupId}) + if isinstance(res, list): + # We can only match one, right? + return GroupObj(**res[0]) + else: + return res + + def loadGroupByName(self, name): + res = self.request('loadGroups', {'name': name}) + if isinstance(res, list): + # We can only match one, right? + return GroupObj(**res[0]) + else: + return res + def loadGroups(self, group=None): - data = {} + res = self.request('loadGroups', {}) + data = [GroupObj(**x) for x in res] if group is not None: - data['groupId'] = group.groupId + data = [x for x in data if x.name == group] - return self.request('loadGroups', data) + return data def deleteGroup(self, group): data = { @@ -493,11 +931,11 @@ export=False): data = { 'groupId': group.groupId, - 'name': organismName, - 'administrate': administrate, - 'write': write, - 'export': export, - 'read': read, + 'organism': organismName, + 'ADMINISTRATE': administrate, + 'WRITE': write, + 'EXPORT': export, + 'READ': read, } return self.request('updateOrganismPermission', data) @@ -706,7 +1144,7 @@ org = self._wa.organisms.findOrganismByCn(cn) self._wa.annotations.setSequence(org['commonName'], org['id']) - data = StringIO.StringIO(self._wa.io.write( + data = io.StringIO(self._wa.io.write( exportType='GFF3', seqType='genomic', exportAllSequences=False, @@ -816,9 +1254,9 @@ def featuresToFeatureSchema(features): compiled = [] for feature in features: - if feature.type != 'gene': - log.warn("Not able to handle %s features just yet...", feature.type) - continue + # if feature.type != 'gene': + # log.warn("Not able to handle %s features just yet...", feature.type) + # continue for x in _yieldFeatData([feature]): compiled.append(x) @@ -834,6 +1272,10 @@ 'ADMINISTRATE' in x['permissions'] or user.role == 'ADMIN' } + + if 'error' in orgs: + raise Exception("Error received from Apollo server: \"%s\"" % orgs['error']) + return [ (org['commonName'], org['id'], False) for org in sorted(orgs, key=lambda x: x['commonName']) @@ -841,18 +1283,114 @@ ] +def galaxy_list_groups(trans, *args, **kwargs): + email = trans.get_user().email + wa = WebApolloInstance( + os.environ['GALAXY_WEBAPOLLO_URL'], + os.environ['GALAXY_WEBAPOLLO_USER'], + os.environ['GALAXY_WEBAPOLLO_PASSWORD'] + ) + # Assert that the email exists in apollo + try: + gx_user = wa.requireUser(email) + except UnknownUserException: + return [] + + # Key for cached data + cacheKey = 'groups-' + email + # We don't want to trust "if key in cache" because between asking and fetch + # it might through key error. + if cacheKey not in cache: + # However if it ISN'T there, we know we're safe to fetch + put in + # there. + data = _galaxy_list_groups(wa, gx_user, *args, **kwargs) + cache[cacheKey] = data + return data + try: + # The cache key may or may not be in the cache at this point, it + # /likely/ is. However we take no chances that it wasn't evicted between + # when we checked above and now, so we reference the object from the + # cache in preparation to return. + data = cache[cacheKey] + return data + except KeyError: + # If access fails due to eviction, we will fail over and can ensure that + # data is inserted. + data = _galaxy_list_groups(wa, gx_user, *args, **kwargs) + cache[cacheKey] = data + return data + + +def _galaxy_list_groups(wa, gx_user, *args, **kwargs): + # Fetch the groups. + group_data = [] + for group in wa.groups.loadGroups(): + # Reformat + group_data.append((group.name, group.groupId, False)) + return group_data + + def galaxy_list_orgs(trans, *args, **kwargs): email = trans.get_user().email + wa = WebApolloInstance( + os.environ['GALAXY_WEBAPOLLO_URL'], + os.environ['GALAXY_WEBAPOLLO_USER'], + os.environ['GALAXY_WEBAPOLLO_PASSWORD'] + ) + try: + gx_user = wa.requireUser(email) + except UnknownUserException: + return [] - wa = WebApolloInstance( - os.environ.get('GALAXY_WEBAPOLLO_URL', 'https://example.com'), - os.environ.get('GALAXY_WEBAPOLLO_USER', 'admin'), - os.environ.get('GALAXY_WEBAPOLLO_PASSWORD', 'admin') - ) + # Key for cached data + cacheKey = 'orgs-' + email + if cacheKey not in cache: + data = _galaxy_list_orgs(wa, gx_user, *args, **kwargs) + cache[cacheKey] = data + return data + try: + data = cache[cacheKey] + return data + except KeyError: + data = _galaxy_list_orgs(wa, gx_user, *args, **kwargs) + cache[cacheKey] = data + return data + - gx_user = AssertUser(wa.users.loadUsers(email=email)) +def _galaxy_list_orgs(wa, gx_user, *args, **kwargs): + # Fetch all organisms all_orgs = wa.organisms.findAllOrganisms() + # Figure out which are accessible to the user + orgs = accessible_organisms(gx_user, all_orgs) + # Return org list + return orgs + +## This is all for implementing the command line interface for testing. + +class obj(object): + pass + + +class fakeTrans(object): + + def __init__(self, username): + self.un = username - orgs = accessible_organisms(gx_user, all_orgs) + def get_user(self): + o = obj() + o.email = self.un + return o - return orgs +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Test access to apollo server') + parser.add_argument('email', help='Email of user to test') + parser.add_argument('--action', choices=['org', 'group'], default='org', help='Data set to test, fetch a list of groups or users known to the requesting user.') + args = parser.parse_args() + + trans = fakeTrans(args.email) + if args.action == 'org': + for f in galaxy_list_orgs(trans): + print(f) + else: + for f in galaxy_list_groups(trans): + print(f)