Mercurial > repos > stevecassidy > alveoimport
annotate alveo_item_list_downloader.py @ 0:bfe39bd252df draft
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
author | stevecassidy |
---|---|
date | Mon, 18 Jul 2016 23:49:40 -0400 |
parents | |
children | 6fef3489d97c |
rev | line source |
---|---|
0
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
1 from __future__ import print_function |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
2 import json |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
3 import argparse |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
4 import pyalveo |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
5 import sys |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
6 import os |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
7 from fnmatch import fnmatch |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
8 |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
9 API_URL = 'https://app.alveo.edu.au' # TODO: export constants to a separate module |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
10 |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
11 def parser(): |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
12 parser = argparse.ArgumentParser(description="Downloads documents in an Alveo Item List") |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
13 parser.add_argument('--api_key', required=True, action="store", type=str, help="Alveo API key") |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
14 parser.add_argument('--item_list_url', required=True, action="store", type=str, help="Item List to download") |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
15 parser.add_argument('--patterns', required=True, action="store", type=str, help="File patterns to download") |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
16 parser.add_argument('--output_path', required=True, action="store", type=str, help="Path to output file") |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
17 return parser.parse_args() |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
18 |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
19 def get_item_list(api_key, item_list_url): |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
20 client = pyalveo.Client(api_key=api_key, api_url=API_URL) |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
21 return client.get_item_list(item_list_url) |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
22 |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
23 # this file name pattern allows galaxy to discover the dataset designation and type |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
24 FNPAT = "%(designation)s_%(ext)s" |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
25 |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
26 def galaxy_name(fname): |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
27 """construct a filename suitable for Galaxy dataset discovery""" |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
28 |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
29 root, ext = os.path.splitext(fname) |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
30 ext = ext[1:] # remove initial . |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
31 fname = FNPAT % {'designation': fname, 'ext': ext} |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
32 |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
33 return fname |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
34 |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
35 def download_documents(item_list, patterns, output_path): |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
36 """ |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
37 Downloads a list of documents to the directory specificed by output_path. |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
38 |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
39 :type documents: list of pyalveo.Document |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
40 :param documents: Documents to download |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
41 |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
42 :type output_path: String |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
43 :param output_path: directory to download to the documents to |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
44 """ |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
45 if not os.path.exists(output_path): |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
46 os.makedirs(output_path) |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
47 |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
48 downloaded = [] |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
49 |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
50 items = item_list.get_all() |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
51 filtered_documents = [] |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
52 for item in items: |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
53 documents = item.get_documents() |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
54 for doc in documents: |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
55 for pattern in patterns: |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
56 if not pattern == '' and fnmatch(doc.get_filename(), pattern): |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
57 fname = galaxy_name(doc.get_filename()) |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
58 try: |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
59 doc.download_content(dir_path=output_path, filename=fname) |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
60 downloaded.append(doc.get_filename()) |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
61 except: |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
62 # maybe it doesn't exist or we have no access |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
63 # TODO: report this |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
64 pass |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
65 return downloaded |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
66 |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
67 def main(): |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
68 args = parser() |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
69 try: |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
70 api_key = open(args.api_key, 'r').read().strip() |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
71 item_list = get_item_list(api_key, args.item_list_url) |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
72 patterns = args.patterns.split(',') |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
73 downloaded = download_documents(item_list, patterns, args.output_path) |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
74 # write out a list of downloaded files as a result? |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
75 except pyalveo.APIError as e: |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
76 print("ERROR: " + str(e), file=sys.stderr) |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
77 sys.exit(1) |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
78 |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
79 if __name__ == '__main__': |
bfe39bd252df
planemo upload commit 5de43e6a614de2a1b2065bc63823ecc9854ebb32-dirty
stevecassidy
parents:
diff
changeset
|
80 main() |