Mercurial > repos > stevecassidy > alveoimport

changeset 9:2f4907372748 draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
planemo upload commit 7b5663b41b2dc11f9e375b8f386bc31855800bcf-dirty
author stevecassidy
date Wed, 16 Nov 2016 15:01:03 -0500
parents 4d84952bdcb8
children e2989a1d751d
files alveo_api_key.xml alveo_get_item_data.xml alveo_get_item_list.xml alveo_get_primary_text.py alveo_get_primary_text.xml alveo_item_list_importer.xml austalk-select-hVd-words.xml
diffstat 7 files changed, 45 insertions(+), 30 deletions(-) [+]
line wrap: on
line diff
--- a/alveo_api_key.xml	Wed Oct 12 22:08:02 2016 -0400
+++ b/alveo_api_key.xml	Wed Nov 16 15:01:03 2016 -0500
@@ -1,4 +1,4 @@
-<tool id="alveo_api_key_storer" name="Store Alveo API Key" version="0.01" force_history_refresh="True">
+<tool id="alveo_api_key_storer" name="Store Alveo API Key" version="0.01">
     <description>for use with Alveo tools</description>
 
     <requirements>
@@ -14,7 +14,7 @@
     </inputs>
 
     <outputs>
-        <data format="txt" name="output" label="Alveo API key" />
+        <data format="auth_token" name="output" label="Alveo API key" />
     </outputs>
 
     <tests>
--- a/alveo_get_item_data.xml	Wed Oct 12 22:08:02 2016 -0400
+++ b/alveo_get_item_data.xml	Wed Nov 16 15:01:03 2016 -0500
@@ -1,4 +1,4 @@
-<tool id="alveo_get_item_data" name="Get Alveo Data for Items" version="0.01" force_history_refresh="True">
+<tool id="alveo_get_item_data" name="Get Alveo Data for Items" version="0.01">
     <description>Downloads files from the items in an Galaxy list of items</description>
 
     <requirements>
@@ -10,8 +10,8 @@
     </command>
 
     <inputs>
-        <param name="api_key" type="data" format="txt" label="API Key" help="Your Alveo API key"/>
-        <param name="item_list" type="data" format="tabular" label="Item List (table)" help=""/>
+        <param name="api_key" type="data" format="auth_token" label="API Key" help="Your Alveo API key"/>
+        <param name="item_list" type="data" format="item_list" label="Item List (table)" help=""/>
 
         <param name="patternselect" type="select" multiple="true" label="Predefined imports" display="checkboxes">
             <option value='*'>All Files</option>
--- a/alveo_get_item_list.xml	Wed Oct 12 22:08:02 2016 -0400
+++ b/alveo_get_item_list.xml	Wed Nov 16 15:01:03 2016 -0500
@@ -1,4 +1,4 @@
-<tool id="alveo_get_item_list" name="Get Item List from Alveo" version="0.01" force_history_refresh="True">
+<tool id="alveo_get_item_list" name="Get Item URLs for Item List" version="0.01">
     <description>Retrieves Item URLs from an Alveo Item List</description>
 
     <requirements>
@@ -10,7 +10,7 @@
     </command>
 
     <inputs>
-        <param name="api_key" type="data" format="txt" label="API Key" help="Your Alveo API key"/>
+        <param name="api_key" type="data" format="auth_token" label="API Key" help="Your Alveo API key"/>
         <param name="import_list" type="data" format="tabular" label="Imported Alveo Item List" help=""/>
 
         <param name="item_list_url" type="select" label="Alveo Item List" help="The Alveo Item List you wish to import">
@@ -25,7 +25,7 @@
     </inputs>
 
     <outputs>
-        <data format="tabular" name="output" label="${job_name}"/>
+        <data format="item_list" name="output" label="${job_name}"/>
     </outputs>
 
     <tests>
--- a/alveo_get_primary_text.py	Wed Oct 12 22:08:02 2016 -0400
+++ b/alveo_get_primary_text.py	Wed Nov 16 15:01:03 2016 -0500
@@ -5,13 +5,15 @@
 import sys
 import os
 from fnmatch import fnmatch
+import csv
+
 
 API_URL = 'https://app.alveo.edu.au' # TODO: export constants to a separate module
 
 def parser():
     parser = argparse.ArgumentParser(description="Downloads documents in an Alveo Item List")
     parser.add_argument('--api_key', required=True, action="store", type=str, help="Alveo API key")
-    parser.add_argument('--item_list_url', required=True, action="store", type=str, help="Item List to download")
+    parser.add_argument('--item_list', required=True, action="store", type=str, help="File containing list of item URLs")
     parser.add_argument('--output_path', required=True, action="store", type=str, help="Path to output file")
     return parser.parse_args()
 
@@ -28,6 +30,7 @@
     fname = FNPAT % {'designation': fname, 'ext': ext}
 
     return fname
+
 import pprint
 def download_documents(item_list, output_path):
     """
@@ -56,13 +59,31 @@
 
     return downloaded
 
+def read_item_list(filename, client):
+    """Read an item list from a file
+    which should be a tabular formatted file
+    with one column header ItemURL.
+    Return an instance of ItemGroup"""
+
+    with open(filename) as fd:
+        csvreader = csv.DictReader(fd, dialect='excel-tab')
+        if 'ItemURL' not in csvreader.fieldnames:
+            return None
+        itemurls = []
+        for row in csvreader:
+            itemurls.append(row['ItemURL'])
+
+    itemlist = pyalveo.ItemGroup(itemurls, client)
+
+    return itemlist
+
 def main():
     args = parser()
     try:
         api_key = open(args.api_key, 'r').read().strip()
-        item_list = get_item_list(api_key, args.item_list_url)
+        client = pyalveo.Client(api_url=API_URL, api_key=api_key, use_cache=False)
+        item_list = read_item_list(args.item_list, client)
         downloaded = download_documents(item_list, args.output_path)
-        # write out a list of downloaded files as a result?
     except pyalveo.APIError as e:
         print("ERROR: " + str(e), file=sys.stderr)
         sys.exit(1)
--- a/alveo_get_primary_text.xml	Wed Oct 12 22:08:02 2016 -0400
+++ b/alveo_get_primary_text.xml	Wed Nov 16 15:01:03 2016 -0500
@@ -1,4 +1,4 @@
-<tool id="alveo_get_primary_text" name="Get Text from Alveo" version="0.01" force_history_refresh="True">
+<tool id="alveo_get_primary_text" name="Get Text from Alveo" version="0.01">
     <description>Downloads primary text from the items in an Alveo Item List</description>
 
     <requirements>
@@ -10,15 +10,8 @@
     </command>
 
     <inputs>
-        <param name="api_key" type="data" format="txt" label="API Key" help="Your Alveo API key"/>
-        <param name="import_list" type="data" format="tabular" label="Imported Alveo Item List" help=""/>
-
-        <param name="item_list_url" type="select" label="Alveo Item List" help="The Alveo Item List you wish to import">
-            <options from_dataset="import_list">
-                <column name="name" index="0"/>
-                <column name="value" index="1"/>
-            </options>
-        </param>
+        <param name="api_key" type="data" format="auth_token" label="API Key" help="Your Alveo API key"/>
+        <param name="item_list" type="data" format="item_list" label="Item List" help=""/>
 
         <param name="job_name" type="text" size="25"
                label="Supply a name for the outputs to remind you what they contain" value="Item List downloaded from Alveo"/>
--- a/alveo_item_list_importer.xml	Wed Oct 12 22:08:02 2016 -0400
+++ b/alveo_item_list_importer.xml	Wed Nov 16 15:01:03 2016 -0500
@@ -1,5 +1,5 @@
-<tool id="alveo_item_list_importer" name="Get Alveo Item Lists" version="0.01" force_history_refresh="True">
-    <description>Retrieves item list metadata.</description>
+<tool id="alveo_item_list_importer" name="Import Item List Names" version="0.01">
+    <description>Retrieves the list of Item List URLs from Alveo.</description>
 
     <requirements>
         <requirement type="package" version="0.7">pyalveo</requirement>
@@ -11,7 +11,7 @@
 
 
     <inputs>
-        <param name="api_key" type="data" format="txt" label="API Key" help="Your Alveo API key"/>
+        <param name="api_key" type="data" format="auth_token" label="API Key" help="Your Alveo API key"/>
         <param name="job_name" type="text" size="25"
                label="Supply a name for the outputs to remind you what they contain" value="Alveo Item Lists"/>
     </inputs>
@@ -20,8 +20,9 @@
         <data format="tabular" name="item_list" label="${job_name}"/>
     </outputs>
 
-    <help>Import Item Lists from Alveo. This imports the lists, but does not download the individual items.
-        That task is performed by the *Get Files from Alveo* tool.
+    <help>Import Item Lists from Alveo so that you can pass an item list
+    to a subsequent tool.  This only retrieves the list of item list names
+    and URLs that are available to a user, not the item lists themselves.
     </help>
 
     <citations>
--- a/austalk-select-hVd-words.xml	Wed Oct 12 22:08:02 2016 -0400
+++ b/austalk-select-hVd-words.xml	Wed Nov 16 15:01:03 2016 -0500
@@ -1,4 +1,4 @@
-<tool id="austalk-select-hvd-words" name="Find HVD words in Austalk" version="0.01" force_history_refresh="True">
+<tool id="austalk-select-hvd-words" name="Find HVD words in Austalk" version="0.01">
     <description>for a single speaker</description>
 
     <requirements>
@@ -10,8 +10,8 @@
     </command>
 
     <inputs>
-        <param name="api_key" type="data" format="txt" label="API Key" help="Your Alveo API key"/>
-        <param name="speaker" type="text" format="text" label="Speaker ID" help="e.g. 1_123"/>
+        <param name="api_key" type="data" format="auth_token" label="API Key" help="Your Alveo API key"/>
+        <param name="speaker" type="text" format="txt" label="Speaker ID" help="e.g. 1_123"/>
         <param name="words" type="select" multiple="false" label="Word List" display="radio">
             <option value='all'>All hVd words</option>
             <option value='monopthongs'>hVd monopthongs</option>
@@ -22,7 +22,7 @@
     </inputs>
 
     <outputs>
-        <data format="tabular" name="output" label="$job_name" />
+        <data format="item_list" name="output" label="$job_name" />
     </outputs>
 
     <tests>