comparison venv/bin/dynamodb_load @ 0:d67268158946 draft

planemo upload commit a3f181f5f126803c654b3a66dd4e83a48f7e203b

0:d67268158946

#!/Users/bclaywel/workspace/argo-navis/venv/bin/python2.7

import argparse
import os

import boto
from boto.compat import json
from boto.dynamodb.schema import Schema


DESCRIPTION = """Load data into one or more DynamoDB tables.

For each table, data is read from two files:
  - {table_name}.metadata for the table's name, schema and provisioned
    throughput (only required if creating the table).
  - {table_name}.data for the table's actual contents.

Both files are searched for in the current directory. To read them from
somewhere else, use the --in-dir parameter.

This program does not wipe the tables prior to loading data. However, any
items present in the data files will overwrite the table's contents.
"""


def _json_iterload(fd):
    """Lazily load newline-separated JSON objects from a file-like object."""
    buffer = ""
    eof = False
    while not eof:
        try:
            # Add a line to the buffer
            buffer += fd.next()
        except StopIteration:
            # We can't let that exception bubble up, otherwise the last
            # object in the file will never be decoded.
            eof = True
        try:
            # Try to decode a JSON object.
            json_object = json.loads(buffer.strip())

            # Success: clear the buffer (everything was decoded).
            buffer = ""
        except ValueError:
            if eof and buffer.strip():
                # No more lines to load and the buffer contains something other
                # than whitespace: the file is, in fact, malformed.
                raise
            # We couldn't decode a complete JSON object: load more lines.
            continue

        yield json_object


def create_table(metadata_fd):
    """Create a table from a metadata file-like object."""


def load_table(table, in_fd):
    """Load items into a table from a file-like object."""
    for i in _json_iterload(in_fd):
        # Convert lists back to sets.
        data = {}
        for k, v in i.iteritems():
            if isinstance(v, list):
                data[k] = set(v)
            else:
                data[k] = v
        table.new_item(attrs=data).put()


def dynamodb_load(tables, in_dir, create_tables):
    conn = boto.connect_dynamodb()
    for t in tables:
        metadata_file = os.path.join(in_dir, "%s.metadata" % t)
        data_file = os.path.join(in_dir, "%s.data" % t)
        if create_tables:
            with open(metadata_file) as meta_fd:
                metadata = json.load(meta_fd)
            table = conn.create_table(
                name=t,
                schema=Schema(metadata["schema"]),
                read_units=metadata["read_units"],
                write_units=metadata["write_units"],
            )
            table.refresh(wait_for_active=True)
        else:
            table = conn.get_table(t)

        with open(data_file) as in_fd:
            load_table(table, in_fd)


if __name__ == "__main__":
    parser = argparse.ArgumentParser(
        prog="dynamodb_load",
        description=DESCRIPTION
    )
    parser.add_argument(
        "--create-tables",
        action="store_true",
        help="Create the tables if they don't exist already (without this flag, attempts to load data into non-existing tables fail)."
    )
    parser.add_argument("--in-dir", default=".")
    parser.add_argument("tables", metavar="TABLES", nargs="+")

    namespace = parser.parse_args()

    dynamodb_load(namespace.tables, namespace.in_dir, namespace.create_tables)