Mercurial > repos > eric-rasche > apollo
changeset 7:f9a6e151b3b4 draft
planemo upload for repository https://github.com/TAMU-CPT/galaxy-webapollo commit 52b9e5bf6a6efb09a5cb845ee48703651c644174
| author | eric-rasche |
|---|---|
| date | Tue, 27 Jun 2017 04:05:17 -0400 |
| parents | 8f76685cdfc8 |
| children | df7a90763b3c |
| files | create_account.py create_features_from_gff3.py create_features_from_gff3.xml create_or_update_organism.py create_or_update_organism.xml delete_features.py delete_features.xml delete_organism.py delete_organism.xml export.py fetch_organism_jbrowse.py fetch_organism_jbrowse.xml json2iframe.py list_organisms.py macros.xml webapollo.py |
| diffstat | 16 files changed, 647 insertions(+), 128 deletions(-) [+] |
line wrap: on
line diff
--- a/create_account.py Sat Mar 04 18:00:52 2017 -0500 +++ b/create_account.py Tue Jun 27 04:05:17 2017 -0400 @@ -1,4 +1,7 @@ #!/usr/bin/env python +from __future__ import print_function +from builtins import str +from builtins import range import random import argparse import time @@ -9,6 +12,7 @@ chars = list('qwrtpsdfghjklzxcvbnm') return ''.join(random.choice(chars) for _ in range(length)) + if __name__ == '__main__': parser = argparse.ArgumentParser(description='Sample script to add an account via web services') WAAuth(parser) @@ -30,9 +34,9 @@ # Update name, regen password if the user ran it again userObj = user[0] returnData = wa.users.updateUser(userObj, args.email, args.first, args.last, password) - print 'Updated User\nUsername: %s\nPassword: %s' % (args.email, password) + print('Updated User\nUsername: %s\nPassword: %s' % (args.email, password)) else: returnData = wa.users.createUser(args.email, args.first, args.last, password, role='user') - print 'Created User\nUsername: %s\nPassword: %s' % (args.email, password) + print('Created User\nUsername: %s\nPassword: %s' % (args.email, password)) - print "Return data: " + str(returnData) + print("Return data: " + str(returnData))
--- a/create_features_from_gff3.py Sat Mar 04 18:00:52 2017 -0500 +++ b/create_features_from_gff3.py Tue Jun 27 04:05:17 2017 -0400 @@ -1,10 +1,10 @@ #!/usr/bin/env python +from builtins import str import sys -import json import time import argparse from webapollo import WebApolloInstance, featuresToFeatureSchema -from webapollo import WAAuth, OrgOrGuess, GuessOrg, AssertUser +from webapollo import WAAuth, OrgOrGuess, GuessOrg, AssertUser, retry from BCBio import GFF import logging logging.basicConfig(level=logging.INFO) @@ -15,6 +15,7 @@ parser = argparse.ArgumentParser(description='Sample script to add an attribute to a feature via web services') WAAuth(parser) parser.add_argument('email', help='User Email') + parser.add_argument('--source', help='URL where the input dataset can be found.') OrgOrGuess(parser) parser.add_argument('gff3', type=argparse.FileType('r'), help='GFF3 file') @@ -37,63 +38,147 @@ sys.stdout.write('# ') sys.stdout.write('\t'.join(['Feature ID', 'Apollo ID', 'Success', 'Messages'])) sys.stdout.write('\n') - # print(wa.annotations.getFeatures()) for rec in GFF.parse(args.gff3): wa.annotations.setSequence(rec.id, org['id']) for feature in rec.features: # We can only handle genes right now - if feature.type != 'gene': + if feature.type not in ('gene', 'terminator'): continue # Convert the feature into a presentation that Apollo will accept featureData = featuresToFeatureSchema([feature]) + if 'children' in featureData[0] and any([child['type']['name'] == 'tRNA' for child in featureData[0]['children']]): + # We're experiencing a (transient?) problem where gene_001 to + # gene_025 will be rejected. Thus, hardcode to a known working + # gene name and update later. - try: + featureData[0]['name'] = 'tRNA_000' + tRNA_sf = [child for child in feature.sub_features if child.type == 'tRNA'][0] + tRNA_type = 'tRNA-' + tRNA_sf.qualifiers.get('Codon', ["Unk"])[0] + + if 'Name' in feature.qualifiers: + if feature.qualifiers['Name'][0].startswith('tRNA-'): + tRNA_type = feature.qualifiers['Name'][0] + + newfeature = wa.annotations.addFeature(featureData, trustme=True) + + def func0(): + wa.annotations.setName( + newfeature['features'][0]['uniquename'], + tRNA_type, + ) + retry(func0) + + if args.source: + gene_id = newfeature['features'][0]['parent_id'] + + def setSource(): + wa.annotations.addAttributes(gene_id, {'DatasetSource': [args.source]}) + retry(setSource) + + sys.stdout.write('\t'.join([ + feature.id, + newfeature['features'][0]['uniquename'], + 'success', + ])) + elif featureData[0]['type']['name'] == 'terminator': # We're experiencing a (transient?) problem where gene_001 to # gene_025 will be rejected. Thus, hardcode to a known working # gene name and update later. - featureData[0]['name'] = 'gene_000' - # Extract CDS feature from the feature data, this will be used - # to set the CDS location correctly (apollo currently screwing - # this up (2.0.6)) - CDS = featureData[0]['children'][0]['children'] - CDS = [x for x in CDS if x['type']['name'] == 'CDS'][0]['location'] - # Create the new feature + featureData[0]['name'] = 'terminator_000' newfeature = wa.annotations.addFeature(featureData, trustme=True) - # Extract the UUIDs that apollo returns to us - mrna_id = newfeature['features'][0]['uniquename'] - gene_id = newfeature['features'][0]['parent_id'] - # Sleep to give it time to actually persist the feature. Apollo - # is terrible about writing + immediately reading back written - # data. - time.sleep(1) - # Correct the translation start, but with strand specific log - if CDS['strand'] == 1: - wa.annotations.setTranslationStart(mrna_id, min(CDS['fmin'], CDS['fmax'])) - else: - wa.annotations.setTranslationStart(mrna_id, max(CDS['fmin'], CDS['fmax']) - 1) + + def func0(): + wa.annotations.setName( + newfeature['features'][0]['uniquename'], + 'terminator' + ) - # Finally we set the name, this should be correct. - wa.annotations.setName(mrna_id, feature.qualifiers.get('product', ["Unknown"])[0]) - wa.annotations.setName(gene_id, feature.qualifiers.get('product', ["Unknown"])[0]) + retry(func0) - for (k, v) in feature.qualifiers.items(): - if k not in bad_quals: - # set qualifier - pass + if args.source: + gene_id = newfeature['features'][0]['parent_id'] + + def setSource(): + wa.annotations.addAttributes(gene_id, {'DatasetSource': [args.source]}) + retry(setSource) sys.stdout.write('\t'.join([ feature.id, - gene_id, + newfeature['features'][0]['uniquename'], 'success', - "Dropped qualifiers: %s" % (json.dumps({k: v for (k, v) in feature.qualifiers.items() if k not in bad_quals})), ])) - except Exception as e: - sys.stdout.write('\t'.join([ - feature.id, - '', - 'ERROR', - str(e) - ])) + else: + try: + # We're experiencing a (transient?) problem where gene_001 to + # gene_025 will be rejected. Thus, hardcode to a known working + # gene name and update later. + featureData[0]['name'] = 'gene_000' + # Extract CDS feature from the feature data, this will be used + # to set the CDS location correctly (apollo currently screwing + # this up (2.0.6)) + CDS = featureData[0]['children'][0]['children'] + CDS = [x for x in CDS if x['type']['name'] == 'CDS'][0]['location'] + # Create the new feature + newfeature = wa.annotations.addFeature(featureData, trustme=True) + # Extract the UUIDs that apollo returns to us + mrna_id = newfeature['features'][0]['uniquename'] + gene_id = newfeature['features'][0]['parent_id'] + # Sleep to give it time to actually persist the feature. Apollo + # is terrible about writing + immediately reading back written + # data. + time.sleep(1) + # Correct the translation start, but with strand specific log + if CDS['strand'] == 1: + wa.annotations.setTranslationStart(mrna_id, min(CDS['fmin'], CDS['fmax'])) + else: + wa.annotations.setTranslationStart(mrna_id, max(CDS['fmin'], CDS['fmax']) - 1) + + # Finally we set the name, this should be correct. + time.sleep(0.5) + wa.annotations.setName(mrna_id, feature.qualifiers.get('product', feature.qualifiers.get('Name', ["Unknown"]))[0]) + time.sleep(0.5) + + def func(): + wa.annotations.setName(gene_id, feature.qualifiers.get('product', feature.qualifiers.get('Name', ["Unknown"]))[0]) + retry(func) + if args.source: + gene_id = newfeature['features'][0]['parent_id'] + + def setSource(): + wa.annotations.addAttributes(gene_id, {'DatasetSource': [args.source]}) + retry(setSource) + extra_attr = {} + for (key, values) in feature.qualifiers.items(): + if key in bad_quals: + continue + + if key == 'Note': + def func2(): + wa.annotations.addComments(gene_id, values) + retry(func2) + else: + extra_attr[key] = values + + def func3(): + wa.annotations.addAttributes(gene_id, extra_attr) + retry(func3) + + sys.stdout.write('\t'.join([ + feature.id, + gene_id, + 'success', + ])) + except Exception as e: + msg = str(e) + if '\n' in msg: + msg = msg[0:msg.index('\n')] + sys.stdout.write('\t'.join([ + feature.id, + '', + 'ERROR', + msg + ])) sys.stdout.write('\n') + sys.stdout.flush()
--- a/create_features_from_gff3.xml Sat Mar 04 18:00:52 2017 -0500 +++ b/create_features_from_gff3.xml Tue Jun 27 04:05:17 2017 -0400 @@ -13,6 +13,8 @@ "$__user_email__" $gff3_data +--source "${__app__.config.galaxy_infrastructure_url}history/view/${__app__.security.encode_id($input.history_id)}" + > $output]]></command> <inputs> <expand macro="org_or_guess" /> @@ -35,4 +37,3 @@ @REFERENCES@ ]]></help> </tool> -
--- a/create_or_update_organism.py Sat Mar 04 18:00:52 2017 -0500 +++ b/create_or_update_organism.py Tue Jun 27 04:05:17 2017 -0400 @@ -1,4 +1,5 @@ #!/usr/bin/env python +from __future__ import print_function import sys import json import argparse @@ -85,8 +86,8 @@ if args.group: group = wa.groups.loadGroupByName(name=args.group) res = wa.groups.updateOrganismPermission(group, org_cn, - administrate=False, write=True, read=True, - export=True) + administrate=False, write=True, read=True, + export=True) data = [o for o in data if o['commonName'] == org_cn] print(json.dumps(data, indent=2))
--- a/create_or_update_organism.xml Sat Mar 04 18:00:52 2017 -0500 +++ b/create_or_update_organism.xml Tue Jun 27 04:05:17 2017 -0400 @@ -15,7 +15,7 @@ --genus "$genus" --species "$species" -#if str(${group}) != "None": +#if str($group) != "None": --group '${group}' #end if $public
--- a/delete_features.py Sat Mar 04 18:00:52 2017 -0500 +++ b/delete_features.py Tue Jun 27 04:05:17 2017 -0400 @@ -1,7 +1,9 @@ #!/usr/bin/env python +from __future__ import print_function import argparse +import random from webapollo import WebApolloInstance -from webapollo import WAAuth, OrgOrGuess, GuessOrg, AssertUser +from webapollo import WAAuth, OrgOrGuess, GuessOrg, AssertUser, retry import logging logging.basicConfig(level=logging.INFO) log = logging.getLogger(__name__) @@ -11,6 +13,7 @@ parser = argparse.ArgumentParser(description='Sample script to delete all features from an organism') WAAuth(parser) parser.add_argument('email', help='User Email') + parser.add_argument('--type', help='Feature type filter') OrgOrGuess(parser) args = parser.parse_args() @@ -27,13 +30,36 @@ # TODO: Check user perms on org. org = wa.organisms.findOrganismByCn(org_cn) - # Call setSequence to tell apollo which organism we're working with - wa.annotations.setSequence(org['commonName'], org['id']) - # Then get a list of features. - features = wa.annotations.getFeatures() - # For each feature in the features - for feature in features['features']: - # We see that deleteFeatures wants a uniqueName, and so we pass - # is the uniquename field in the feature. - print(wa.annotations.deleteFeatures([feature['uniquename']])) + sequences = wa.organisms.getSequencesForOrganism(org['id']) + for sequence in sequences['sequences']: + log.info("Processing %s %s", org['commonName'], sequence['name']) + # Call setSequence to tell apollo which organism we're working with + wa.annotations.setSequence(sequence['name'], org['id']) + # Then get a list of features. + features = wa.annotations.getFeatures() + # For each feature in the features + for feature in sorted(features['features'], key=lambda x: random.random()): + if args.type: + if args.type == 'tRNA': + if feature['type']['name'] != 'tRNA': + continue + elif args.type == 'terminator': + if feature['type']['name'] != 'terminator': + continue + + elif args.type == 'mRNA': + if feature['type']['name'] != 'mRNA': + continue + + else: + raise Exception("Unknown type") + + # We see that deleteFeatures wants a uniqueName, and so we pass + # is the uniquename field in the feature. + def fn(): + wa.annotations.deleteFeatures([feature['uniquename']]) + print('Deleted %s [type=%s]' % (feature['uniquename'], feature['type']['name'])) + + if not retry(fn, limit=3): + print('Error %s' % feature['uniquename'])
--- a/delete_features.xml Sat Mar 04 18:00:52 2017 -0500 +++ b/delete_features.xml Tue Jun 27 04:05:17 2017 -0400 @@ -1,5 +1,5 @@ <?xml version="1.0"?> -<tool id="edu.tamu.cpt2.webapollo.delete_features" name="Delete all annotations from an Apollo record" version="1.2"> +<tool id="edu.tamu.cpt2.webapollo.delete_features" name="Delete all annotations from an Apollo record" version="1.5" profile="16.04"> <description></description> <macros> <import>macros.xml</import> @@ -8,32 +8,36 @@ <expand macro="requirements"/> <command detect_errors="aggressive"><![CDATA[ #if str($ask_one) == "yes": - #if str($ask_two) == "yes": - ## Nope, still don't trust them to not be dumb (or malicious), so we backup first. - python $__tool_directory__/export.py - @ADMIN_AUTH@ - @ORG_OR_GUESS@ - --gff "$gff_out" - --fasta "$fasta_out" - --json "$json_out"; + ## Nope, still don't trust them to not be dumb (or malicious), so we backup first. + python $__tool_directory__/export.py + @ADMIN_AUTH@ + @ORG_OR_GUESS@ + --gff "$gff_out" + --fasta "$fasta_out" + --json "$json_out"; - ## Now we delete - python $__tool_directory__/delete_features.py - @ADMIN_AUTH@ - @ORG_OR_GUESS@ - "$__user_email__" - > $output; - #else - echo "Nothing to do" > $output; - #end if + ## Now we delete + python $__tool_directory__/delete_features.py + @ADMIN_AUTH@ + @ORG_OR_GUESS@ + "$__user_email__" + #if str($filter) != "all" + --type $filter + #end if + > $output; #else echo "Nothing to do" > $output; #end if ]]></command> <inputs> <expand macro="org_or_guess" /> + <param name="filter" type="select" label="Feature Type Filter"> + <option value="all">All</option> + <option value="mRNA">Genes</option> + <option value="terminator">Terminators</option> + <option value="tRNA">tRNAs</option> + </param> <param name="ask_one" type="boolean" truevalue="yes" falsevalue="" label="Are you SURE you want to do this?" help="It will PERMANENTLY delete all of the features on this organism."/> - <param name="ask_two" type="boolean" truevalue="yes" falsevalue="" label="Are you really, really SURE you want to do this?" help="There's NO coming back from this."/> </inputs> <outputs> <data format="tabular" name="output" label="Process and Error Log"/> @@ -57,4 +61,3 @@ @REFERENCES@ ]]></help> </tool> -
--- a/delete_organism.py Sat Mar 04 18:00:52 2017 -0500 +++ b/delete_organism.py Tue Jun 27 04:05:17 2017 -0400 @@ -1,4 +1,5 @@ #!/usr/bin/env python +from __future__ import print_function import argparse from webapollo import WebApolloInstance from webapollo import WAAuth, OrgOrGuess, GuessOrg, AssertUser @@ -36,4 +37,3 @@ # We see that deleteFeatures wants a uniqueName, and so we pass # is the uniquename field in the feature. print(wa.annotations.deleteFeatures([feature['uniquename']])) -
--- a/delete_organism.xml Sat Mar 04 18:00:52 2017 -0500 +++ b/delete_organism.xml Tue Jun 27 04:05:17 2017 -0400 @@ -1,5 +1,5 @@ <?xml version="1.0"?> -<tool id="edu.tamu.cpt2.webapollo.delete_organism" name="Delete an Apollo record" version="1.0"> +<tool id="edu.tamu.cpt2.webapollo.delete_organism" name="Delete an Apollo record" version="1.0" profile="16.04"> <description></description> <macros> <import>macros.xml</import> @@ -57,4 +57,3 @@ @REFERENCES@ ]]></help> </tool> -
--- a/export.py Sat Mar 04 18:00:52 2017 -0500 +++ b/export.py Tue Jun 27 04:05:17 2017 -0400 @@ -1,16 +1,18 @@ #!/usr/bin/env python +from __future__ import print_function +import argparse +import json import sys +from Bio import SeqIO +from BCBio import GFF +from webapollo import WAAuth, WebApolloInstance, CnOrGuess, GuessCn +from future import standard_library +standard_library.install_aliases() try: import StringIO as io except ImportError: import io -import json -import argparse -from Bio import SeqIO -from BCBio import GFF -from webapollo import WAAuth, WebApolloInstance, CnOrGuess, GuessCn - def export(org_cn, seqs): org_data = wa.organisms.findOrganismByCn(org_cn)
--- a/fetch_organism_jbrowse.py Sat Mar 04 18:00:52 2017 -0500 +++ b/fetch_organism_jbrowse.py Tue Jun 27 04:05:17 2017 -0400 @@ -1,13 +1,54 @@ #!/usr/bin/env python +from __future__ import print_function import os +import sys +import time import argparse -from webapollo import WAAuth, WebApolloInstance, GuessOrg, OrgOrGuess +import filecmp +import os.path import logging import subprocess +from webapollo import WAAuth, WebApolloInstance, GuessOrg, OrgOrGuess logging.basicConfig(level=logging.INFO) log = logging.getLogger(__name__) +def are_dir_trees_equal(dir1, dir2): + """ + Compare two directories recursively. Files in each directory are + assumed to be equal if their names and contents are equal. + + @param dir1: First directory path + @param dir2: Second directory path + + @return: True if the directory trees are the same and + there were no errors while accessing the directories or files, + False otherwise. + + # http://stackoverflow.com/questions/4187564/recursive-dircmp-compare-two-directories-to-ensure-they-have-the-same-files-and/6681395#6681395 + """ + + dirs_cmp = filecmp.dircmp(dir1, dir2) + if len(dirs_cmp.left_only) > 0 or len(dirs_cmp.right_only) > 0 or \ + len(dirs_cmp.funny_files) > 0: + print(('LEFT', dirs_cmp.left_only)) + print(('RIGHT', dirs_cmp.right_only)) + print(('FUNNY', dirs_cmp.funny_files)) + return False + (_, mismatch, errors) = filecmp.cmpfiles( + dir1, dir2, dirs_cmp.common_files, shallow=False) + if len(mismatch) > 0 or len(errors) > 0: + print(mismatch) + print(errors) + return False + for common_dir in dirs_cmp.common_dirs: + new_dir1 = os.path.join(dir1, common_dir) + new_dir2 = os.path.join(dir2, common_dir) + if not are_dir_trees_equal(new_dir1, new_dir2): + return False + return True + + if __name__ == '__main__': parser = argparse.ArgumentParser(description='Sample script to add an attribute to a feature via web services') WAAuth(parser) @@ -26,9 +67,46 @@ if not os.path.exists(args.target_dir): os.makedirs(args.target_dir) + if not os.path.exists(os.path.join(org['directory'], 'seq')): + sys.stderr.write("Missing seq directory BEFORE copy") + sys.exit(1) + cmd = [ - 'cp', '-R', - org['directory'], + 'rsync', '-avr', + org['directory'].rstrip('/') + '/', + os.path.join(args.target_dir, 'data', '') + ] + # We run this OBSESSIVELY because my org had a hiccup where the origin + # (silent) cp -R failed at one point. This caused MANY HEADACHES. + # + # Our response is to run this 3 times (in case the issue is temporary), + # with delays in between. And ensure that we have the correct number of + # files / folders before and after. + sys.stderr.write(' '.join(cmd)) + sys.stderr.write('\n') + sys.stderr.write(subprocess.check_output(cmd)) + if not are_dir_trees_equal( + os.path.join(org['directory'].rstrip('/')), os.path.join(args.target_dir, 'data') - ] - subprocess.check_call(cmd) + ): + # Not good + time.sleep(5) + sys.stderr.write('\n') + sys.stderr.write(' '.join(cmd)) + sys.stderr.write('\n') + sys.stderr.write(subprocess.check_output(cmd)) + if not are_dir_trees_equal( + os.path.join(org['directory'].rstrip('/'), 'data'), + os.path.join(args.target_dir, 'data') + ): + time.sleep(5) + sys.stderr.write('\n') + sys.stderr.write(' '.join(cmd)) + sys.stderr.write('\n') + sys.stderr.write(subprocess.check_output(cmd)) + if not are_dir_trees_equal( + os.path.join(org['directory'].rstrip('/'), 'data'), + os.path.join(args.target_dir, 'data') + ): + sys.stderr.write('FAILED THREE TIMES TO COPY. SOMETHING IS WRONG WRONG WRONG.') + sys.exit(2)
--- a/fetch_organism_jbrowse.xml Sat Mar 04 18:00:52 2017 -0500 +++ b/fetch_organism_jbrowse.xml Tue Jun 27 04:05:17 2017 -0400 @@ -1,5 +1,5 @@ <?xml version="1.0"?> -<tool id="edu.tamu.cpt2.webapollo.fetch_jbrowse" name="Retrieve JBrowse" version="3.0"> +<tool id="edu.tamu.cpt2.webapollo.fetch_jbrowse" name="Retrieve JBrowse" version="3.2" profile="16.04"> <description>for an organism, from Apollo</description> <macros> <import>macros.xml</import> @@ -13,9 +13,11 @@ @ADMIN_AUTH@ @ORG_OR_GUESS@ -$jbrowse.files_path/; +$jbrowse.files_path/ && -cp $dummyIndex $jbrowse; +cp $dummyIndex $jbrowse && +find $jbrowse.files_path -type f -printf '<li><a href="%P">%P</a></li>\n' | sort >> $jbrowse && +echo '</ul></body></html>' >> $jbrowse; ]]></command> <configfiles> @@ -40,8 +42,11 @@ Convert to Standalone" tool in Galaxy to "upgrade" to a full JBrowse instance. </p> - </body> - </html> + <p> + The following list is provided for your convenience / debugging. + </p> + <h2>Contained Files</h2> + <ul> ]]> </configfile> </configfiles>
--- a/json2iframe.py Sat Mar 04 18:00:52 2017 -0500 +++ b/json2iframe.py Tue Jun 27 04:05:17 2017 -0400 @@ -1,7 +1,8 @@ #!/usr/bin/env python +from __future__ import print_function +import argparse import json -import base64 -import argparse + if __name__ == '__main__': parser = argparse.ArgumentParser(description='Sample script to add an attribute to a feature via web services') @@ -26,6 +27,5 @@ </body> </html> """ - # HTML_TPL = base64.b64decode(HTML_TPL.replace('\n', '')) - print HTML_TPL.format(base_url=args.external_apollo_url, chrom="", orgId=data[0]['id']) + print(HTML_TPL.format(base_url=args.external_apollo_url, chrom="", orgId=data[0]['id']))
--- a/list_organisms.py Sat Mar 04 18:00:52 2017 -0500 +++ b/list_organisms.py Tue Jun 27 04:05:17 2017 -0400 @@ -1,4 +1,5 @@ #!/usr/bin/env python +from __future__ import print_function import json import argparse from webapollo import WAAuth, WebApolloInstance, AssertUser, accessible_organisms
--- a/macros.xml Sat Mar 04 18:00:52 2017 -0500 +++ b/macros.xml Tue Jun 27 04:05:17 2017 -0400 @@ -116,7 +116,7 @@ **Citation** If you use this tool in Galaxy, please cite: -Eric Rasche (2016), `Galaxy wrapper <https://github.com/TAMU-CPT/galaxy-webapollo>`_ +Eric Rasche (2016), `Galaxy Apollo Tools <https://github.com/galaxy-genome-annotation/galaxy-tools/tree/master/tools/apollo>`_ ]]> </token> </macros>
--- a/webapollo.py Sat Mar 04 18:00:52 2017 -0500 +++ b/webapollo.py Tue Jun 27 04:05:17 2017 -0400 @@ -1,23 +1,29 @@ -import requests -import json -import os +from __future__ import print_function +import argparse import collections -try: - import StringIO as io -except: - import io +import json import logging +import os +import requests import time -import argparse +from future import standard_library +from builtins import next +from builtins import str +from builtins import object from abc import abstractmethod from BCBio import GFF from Bio import SeqIO +standard_library.install_aliases() +try: + import StringIO as io +except BaseException: + import io logging.getLogger("requests").setLevel(logging.CRITICAL) log = logging.getLogger() ############################################# -###### BEGIN IMPORT OF CACHING LIBRARY ###### +# BEGIN IMPORT OF CACHING LIBRARY # ############################################# # This code is licensed under the MIT # # License and is a copy of code publicly # @@ -25,6 +31,7 @@ # e27332bc82f4e327aedaec17c9b656ae719322ed # # of https://github.com/tkem/cachetools/ # ############################################# + class DefaultMapping(collections.MutableMapping): __slots__ = () @@ -65,6 +72,7 @@ self[key] = value = default return value + DefaultMapping.register(dict) @@ -381,21 +389,24 @@ ############################################# -###### END IMPORT OF CACHING LIBRARY ###### +# END IMPORT OF CACHING LIBRARY # ############################################# + cache = TTLCache( - 100, # Up to 100 items - 5 * 60 # 5 minute cache life + 100, # Up to 100 items + 5 * 60 # 5 minute cache life ) userCache = TTLCache( - 2, # Up to 2 items - 60 # 1 minute cache life + 2, # Up to 2 items + 60 # 1 minute cache life ) + class UnknownUserException(Exception): pass + def WAAuth(parser): parser.add_argument('apollo', help='Complete Apollo URL') parser.add_argument('username', help='WA Username') @@ -468,8 +479,6 @@ self.apollo_url = url self.username = username self.password = password - # TODO: Remove after apollo 2.0.6. - self.clientToken = time.time() self.annotations = AnnotationsClient(self) self.groups = GroupsClient(self) @@ -478,6 +487,10 @@ self.users = UsersClient(self) self.metrics = MetricsClient(self) self.bio = RemoteRecord(self) + self.status = StatusClient(self) + self.canned_comments = CannedCommentsClient(self) + self.canned_keys = CannedKeysClient(self) + self.canned_values = CannedValuesClient(self) def __str__(self): return '<WebApolloInstance at %s>' % self.apollo_url @@ -539,6 +552,12 @@ data[prop] = getattr(self, prop) return data + def orgPerms(self): + for orgPer in self.organismPermissions: + if len(orgPer['permissions']) > 2: + orgPer['permissions'] = json.loads(orgPer['permissions']) + yield orgPer + def __str__(self): return '<User %s: %s %s <%s>>' % (self.userId, self.firstName, self.lastName, self.username) @@ -565,7 +584,6 @@ data.update({ 'username': self._wa.username, 'password': self._wa.password, - 'clientToken': self._wa.clientToken, }) r = requests.post(url, data=json.dumps(data), headers=headers, @@ -677,22 +695,49 @@ data = self._update_data(data) return self.request('getComments', data) - def addComments(self, feature_id, comment): - #TODO: This is probably not great and will delete comments, if I had to guess... + def addComments(self, feature_id, comments): + # TODO: This is probably not great and will delete comments, if I had to guess... data = { 'features': [ { 'uniquename': feature_id, - 'comments': [comment] + 'comments': comments } ], } data = self._update_data(data) - return self.request('getComments', data) + return self.request('addComments', data) + + def addAttributes(self, feature_id, attributes): + nrps = [] + for (key, values) in attributes.items(): + for value in values: + nrps.append({ + 'tag': key, + 'value': value + }) - def addAttribute(self, features): data = { - 'features': features, + 'features': [ + { + 'uniquename': feature_id, + 'non_reserved_properties': nrps + } + ] + } + data = self._update_data(data) + return self.request('addAttribute', data) + + def deleteAttribute(self, feature_id, key, value): + data = { + 'features': [ + { + 'uniquename': feature_id, + 'non_reserved_properties': [ + {'tag': key, 'value': value} + ] + } + ] } data = self._update_data(data) return self.request('addAttribute', data) @@ -991,6 +1036,198 @@ return self.request('write', data) +class StatusClient(Client): + CLIENT_BASE = '/availableStatus/' + + def addStatus(self, value): + data = { + 'value': value + } + + return self.request('createStatus', data) + + def findAllStatuses(self): + return self.request('showStatus', {}) + + def findStatusByValue(self, value): + statuses = self.findAllStatuses() + statuses = [x for x in statuses if x['value'] == value] + if len(statuses) == 0: + raise Exception("Unknown status value") + else: + return statuses[0] + + def findStatusById(self, id_number): + statuses = self.findAllStatuses() + statuses = [x for x in statuses if str(x['id']) == str(id_number)] + if len(statuses) == 0: + raise Exception("Unknown ID") + else: + return statuses[0] + + def updateStatus(self, id_number, new_value): + data = { + 'id': id_number, + 'new_value': new_value + } + + return self.request('updateStatus', data) + + def deleteStatus(self, id_number): + data = { + 'id': id_number + } + + return self.request('deleteStatus', data) + + +class CannedCommentsClient(Client): + CLIENT_BASE = '/cannedComment/' + + def addComment(self, comment, metadata=""): + data = { + 'comment': comment, + 'metadata': metadata + } + + return self.request('createComment', data) + + def findAllComments(self): + return self.request('showComment', {}) + + def findCommentByValue(self, value): + comments = self.findAllComments() + comments = [x for x in comments if x['comment'] == value] + if len(comments) == 0: + raise Exception("Unknown comment") + else: + return comments[0] + + def findCommentById(self, id_number): + comments = self.findAllComments() + comments = [x for x in comments if str(x['id']) == str(id_number)] + if len(comments) == 0: + raise Exception("Unknown ID") + else: + return comments[0] + + def updateComment(self, id_number, new_value, metadata=None): + data = { + 'id': id_number, + 'new_comment': new_value + } + + if metadata is not None: + data['metadata'] = metadata + + return self.request('updateComment', data) + + def deleteComment(self, id_number): + data = { + 'id': id_number + } + + return self.request('deleteComment', data) + + +class CannedKeysClient(Client): + CLIENT_BASE = '/cannedKey/' + + def addKey(self, key, metadata=""): + data = { + 'key': key, + 'metadata': metadata + } + + return self.request('createKey', data) + + def findAllKeys(self): + return self.request('showKey', {}) + + def findKeyByValue(self, value): + keys = self.findAllKeys() + keys = [x for x in keys if x['label'] == value] + if len(keys) == 0: + raise Exception("Unknown key") + else: + return keys[0] + + def findKeyById(self, id_number): + keys = self.findAllKeys() + keys = [x for x in keys if str(x['id']) == str(id_number)] + if len(keys) == 0: + raise Exception("Unknown ID") + else: + return keys[0] + + def updateKey(self, id_number, new_key, metadata=None): + data = { + 'id': id_number, + 'new_key': new_key + } + + if metadata is not None: + data['metadata'] = metadata + + return self.request('updateKey', data) + + def deleteKey(self, id_number): + data = { + 'id': id_number + } + + return self.request('deleteKey', data) + + +class CannedValuesClient(Client): + CLIENT_BASE = '/cannedValue/' + + def addValue(self, value, metadata=""): + data = { + 'value': value, + 'metadata': metadata + } + + return self.request('createValue', data) + + def findAllValues(self): + return self.request('showValue', {}) + + def findValueByValue(self, value): + values = self.findAllValues() + values = [x for x in values if x['label'] == value] + if len(values) == 0: + raise Exception("Unknown value") + else: + return values[0] + + def findValueById(self, id_number): + values = self.findAllValues() + values = [x for x in values if str(x['id']) == str(id_number)] + if len(values) == 0: + raise Exception("Unknown ID") + else: + return values[0] + + def updateValue(self, id_number, new_value, metadata=None): + data = { + 'id': id_number, + 'new_value': new_value + } + + if metadata is not None: + data['metadata'] = metadata + + return self.request('updateValue', data) + + def deleteValue(self, id_number): + data = { + 'id': id_number + } + + return self.request('deleteValue', data) + + class OrganismsClient(Client): CLIENT_BASE = '/organism/' @@ -1222,7 +1459,7 @@ def _tnType(feature): - if feature.type in ('gene', 'mRNA', 'exon', 'CDS'): + if feature.type in ('gene', 'mRNA', 'exon', 'CDS', 'terminator', 'tRNA'): return feature.type else: return 'exon' @@ -1365,8 +1602,55 @@ # Return org list return orgs -## This is all for implementing the command line interface for testing. + +def galaxy_list_users(trans, *args, **kwargs): + email = trans.get_user().email + wa = WebApolloInstance( + os.environ['GALAXY_WEBAPOLLO_URL'], + os.environ['GALAXY_WEBAPOLLO_USER'], + os.environ['GALAXY_WEBAPOLLO_PASSWORD'] + ) + # Assert that the email exists in apollo + try: + gx_user = wa.requireUser(email) + except UnknownUserException: + return [] + # Key for cached data + cacheKey = 'users-' + email + # We don't want to trust "if key in cache" because between asking and fetch + # it might through key error. + if cacheKey not in cache: + # However if it ISN'T there, we know we're safe to fetch + put in + # there. + data = _galaxy_list_users(wa, gx_user, *args, **kwargs) + cache[cacheKey] = data + return data + try: + # The cache key may or may not be in the cache at this point, it + # /likely/ is. However we take no chances that it wasn't evicted between + # when we checked above and now, so we reference the object from the + # cache in preparation to return. + data = cache[cacheKey] + return data + except KeyError: + # If access fails due to eviction, we will fail over and can ensure that + # data is inserted. + data = _galaxy_list_users(wa, gx_user, *args, **kwargs) + cache[cacheKey] = data + return data + + +def _galaxy_list_users(wa, gx_user, *args, **kwargs): + # Fetch the users. + user_data = [] + for user in wa.users.loadUsers(): + # Reformat + user_data.append((user.username, user.username, False)) + return user_data + + +# This is all for implementing the command line interface for testing. class obj(object): pass @@ -1381,16 +1665,46 @@ o.email = self.un return o + +def retry(closure, sleep=1, limit=5): + """ + Apollo has the bad habit of returning 500 errors if you call APIs + too quickly, largely because of the unholy things that happen in + grails. + + To deal with the fact that we cannot send an addComments call too + quickly after a createFeature call, we have this function that will + keep calling a closure until it works. + """ + count = 0 + while True: + count += 1 + + if count >= limit: + return False + try: + # Try calling it + closure() + # If successful, exit + return True + except Exception as e: + log.info(str(e)[0:100]) + time.sleep(sleep) + + if __name__ == '__main__': parser = argparse.ArgumentParser(description='Test access to apollo server') parser.add_argument('email', help='Email of user to test') - parser.add_argument('--action', choices=['org', 'group'], default='org', help='Data set to test, fetch a list of groups or users known to the requesting user.') + parser.add_argument('--action', choices=['org', 'group', 'users'], default='org', help='Data set to test, fetch a list of groups or users known to the requesting user.') args = parser.parse_args() trans = fakeTrans(args.email) if args.action == 'org': for f in galaxy_list_orgs(trans): print(f) - else: + elif args.action == 'group': for f in galaxy_list_groups(trans): print(f) + else: + for f in galaxy_list_users(trans): + print(f)