view PythonTrustStore-0.2.0/py_ts/TrustStoreClient.py @ 1:ff126718bdc5

Uploaded

import requests
from requests_oauth2 import OAuth2
import simplejson as json
import simplejson.scanner
from urllib import quote
from urlparse import parse_qs, urljoin
from functools import wraps
from tempfile import NamedTemporaryFile
# from subprocess import call, check_output, Popen
import subprocess
import os
from urlparse import urlparse
from datetime import datetime
import xmltodict
import uuid
import webbrowser
import base64
import binascii
import hashlib
import textwrap
import re
import boto.exception
import boto.s3.connection as botoConn
from boto.s3.key import Key as botoKey
from distutils.version import LooseVersion
from parts import *
# import threading
from multiprocessing.pool import ThreadPool
# import Queue
from passlib.hash import pbkdf2_sha256
from passlib.utils import ab64_decode
from itertools import izip_longest
import sys

# import cProfile #, pstats


DASHES = "-----"
BEGIN = "BEGIN"
END = "END"
PUBLIC = "PUBLIC"
PRIVATE = "PRIVATE"
KEY = "KEY"
CERTIFICATE = "CERTIFICATE"

class TrustStoreClient(object):
    """A client for connecting to a TrustStore service, and probably a storage provider. Currently supports S3 and S3-like storage providers (such as Nectar) which are supported by the Python boto library. Requires a version of OpenSSL greater than 1.0, i.e. Mac will have to install using homebrew.

    """
    brewOpenSSL = '/usr/local/Cellar/openssl/'
    jsonHeaders = {'Content-type': 'application/json', 'Accept': 'application/json'}
    storesPrefix = "/store"
    storesList = "/all"
    filesPrefix = "/file"
    keysPrefix = "/keys"
    publicKeysPrefix = "/public_key"
    privateKeysPrefix = "/private_key"
    usernamesPrefix = "/username"
    saltPrefix = "/loginsalt"
    passwordResetPrefix = "/credentials"

    providers = []

    pieceSize = 209715

    openSSL = "openssl"

    kmsUrl = u'http://localhost:8080/TSSKeyManagementService-Collaboration'
    imsUrl = u'http://localhost:8080/TSSIntegrityManagementService/services/IMS'
    client_key = u'my-trusted-client-with-secret'
    client_secret = u'somesecret'
    headless = False
    redirect_uri = "oob"
    keyFile = None
    auth = None
    kmsClient = None
    username = None

    def requiresAuth(func):
        """Wraps a function which requires the client be already sucesfully authenticated with the Key Management Service."""
        @wraps(func)
        def withAuth(self, *args, **kwargs):
            if self.kmsClient:
                return func(self, *args, **kwargs)
            else:
                raise TrustStoreClientAuthenticationException("Not authenticated!")
        return withAuth

    def __init__(self, headless, config=None, accessToken=None):
        """Set up the client, checking OpenSSL

        :param headless: Are we running somewhere where opening a browser window would be a bad idea? If we're on a server, this should always be True
        :type headless: boolean
        :param  config: Optional collection of configuration variables, must have values kmsUrl, imsUrl, client_key and client_secret.
        :param accessToken: If being re-run, an accessToken can be supplied and autentication skipped. Note that if you're doing this you're expected to have set the username and password properties manually. In which case you may as well just authenticate.

        """
        if os.path.exists(self.brewOpenSSL):
            versions = os.listdir(self.brewOpenSSL)
            versions.sort(key=LooseVersion)
            self.openSSL = self.brewOpenSSL + versions[-1] + "/bin/openssl"

        sslVersion = subprocess.check_output([self.openSSL, 'version'])
        m = re.search('([0-9]+.[0-9]+.[0-9]+[a-z]*)', sslVersion)
        if LooseVersion(m.group(0)) < LooseVersion("1.0.0"):
            raise OpenSSLVersionException(m.group(0))
        elif not headless:
            print "Working with " + sslVersion

        if config:
            self.kmsUrl = config.kmsUrl
            self.imsUrl = config.imsUrl
            self.client_key = config.client_key
            self.client_secret = config.client_secret
        self.headless = headless
        self.auth = OAuth2(self.client_key, self.client_secret, self.kmsUrl, self.redirect_uri)
        if accessToken:
            self.kmsClient = requests.session(params={'access_token': accessToken})
            self.username = self._getUsername()

    def authenticate(self, username=None, password=None):
        """Talk to KMS and get the required tokens etc. If the username and password are not supplied OAuth-2 token authentication is attempted. In headless mode thiis will cause a url to be returned where the user will have to login and authorize this application. Will throw exceptions if credentials are rejected or KMS cannot be reached.

        :param username: The username already registered with KMS.
        :type username: string/unicode
        :param password: The user's KMS password.
        :type password: string/unicode
        :rtype: None or string (authorisation url)

        """

        if not self.kmsClient:
            response = None
            if not username or not password:
                authorization_url = self.auth.authorize_url(scope='read trust write', response_type='code')
                if authorization_url:
                    if self.headless:
                        return authorization_url
                    else:
                        webbrowser.open(authorization_url)
                        code = raw_input('Code:')
                        response = self.auth.get_token(code, grant_type='authorization_code')
            else:
                self.username = username
                self.password = password
                ans = requests.get("%s%s/%s" %(self.kmsUrl, self.saltPrefix, username))
                salt = base64.b64decode(ans.text)
                # Passlib uses a *custom* base64 encoding because it is arse, so fix to normal.
                sendPassword = base64.b64encode(ab64_decode(pbkdf2_sha256.encrypt(username + password, rounds=1000, salt=salt).split("$")[-1]))
                login_form = {'username': username,
                              'password': sendPassword,
                              'client_id': self.client_key,
                              'client_secret': self.client_secret,
                              'grant_type': 'password'}
                response = requests.post("%s%s" % (self.auth.site, quote(self.auth.token_url)), data=login_form, allow_redirects=True)

                if isinstance(response.content, basestring):
                    try:
                        response = json.loads(response.content)
                    except ValueError:
                        response = parse_qs(response.content)
                else:
                    response = response.content

            if response:
                try:
                    self.kmsClient = requests.session(params={'access_token': response['access_token']})
                    if not self.username:
                        self.username = self._getUsername()
                except KeyError:
                    raise TrustStoreClientAuthenticationException("Credentials not accepted. Response: " + str(response))
            else:
                raise TrustStoreClientAuthenticationException("Communication error with TrustStore server. Is it running?")

    def twoStageAuth(self, code):
        """If using oAuth2, this method will need to be called after :func:`authenticate`

        :param code: The oAuth2 token.
        """

        response = self.auth.get_token(code)
        self.kmsClient = requests.session(params={'access_token': response['access_token']})
        self.username = self._getUsername()

    def clearAuth(self):
        """Clear any authentication (username/password) but preserve other configuration (server locations etc)"""
        self.auth = None
        self.kmsClient = None
        self.username = None

    def addProvider(self, prov):
        """Tell the client about a new provider to use (in addition)

        :param prov: The new provider
        :type prov: `Provider`

        """
        self.providers.append(prov)

    @requiresAuth
    def listStores(self):
        """List all the stores this user has access to

        :rtype: list of Store objects

        """
        stores = []
        r = self.kmsClient.get(self.kmsUrl + self.storesPrefix) # + self.storesList)
        if 'error' not in r.json:
            for store in r.json:
                stores.append(Store(store))
            return stores
        else:
            print r.text
            print r.json

    @requiresAuth
    def getPrivateKey(self, filename):
        """Fetch the user's private key from KMS."""
        getr = self.kmsClient.get(self.kmsUrl + self.privateKeysPrefix + "/" + self.username)
        if getr.json and 'error' not in getr.json:
            key = UserPrivateKey(getr.json)
            if key.certificate:
                with open(filename, 'w+') as f:
                    f.write(key.certificate)
                self.keyFile = filename
            else:
                self.setNewKey(filename)
            return filename
        else:
            print "Couldn't find private key?"
            print getr.text

    @requiresAuth
    def setNewKey(self, filename):
        """Generate a new private key, either because the user doesn't have one yet, or they've requested a new one."""
        print "No private key found!"
        self._generateKeypair(filename)
        keydata = None
        if filename:
            with open(filename, 'r') as f:
                keydata = f.read()
            key = UserPrivateKey(None, self.username, keydata)
            postr = self.kmsClient.post(self.kmsUrl + self.privateKeysPrefix, data=json.dumps(key.dict()), headers=self.jsonHeaders)
            if postr.status_code != requests.codes.ok:
                print "Server refused to save new private key."
                return
        return filename

    @requiresAuth
    def getStore(self, name):
        """Get the first store (that the user has access to) matching the specified name."""
        for store in self.listStores():
            if store.friendly_name == name:
                return store

    @requiresAuth
    def createStore(self, store=None, name=None):
        """Create a new store.

        :param store: Pre-prepared store to save to KMS.
        :type store: Store
        :param name: If no pre-prepared store, create new store with this name. If not supplied, store name will be "default".
        :type name: string/unicode
        :rtype: Store

        """
        if not store:
            if not name:
                name = "default"
            store = Store(owner=self._getUsername(), friendly_name=name)
            store.administrators.append(self._getUsername())

        storejson = json.dumps(store.dict())
        # print storejson
        postr = self.kmsClient.post(self.kmsUrl + self.storesPrefix, data=storejson, headers=self.jsonHeaders)
        if postr.status_code != requests.codes.ok:
            print "Server refused request to save store description."
            print postr.status_code
            print postr.text
            return

        store = Store(postr.json)
        storeFile = StoreProperties()
        username = str(uuid.uuid4())
        password = binascii.b2a_hex(os.urandom(20))
        storeFile.ims_url = self.imsUrl
        storeFile.kms_url = self.kmsUrl
        storeFile.ims_user = {"ident": username, "secret": password}

        storeFile.providers = self.providers
        self._createNewBucket(storeFile)
        self._createAndStoreEmptyDirectory(store, storeFile)

        privateKeyFile = self._generatePKCS1Keypair()
        if privateKeyFile:
            publicKeyFile = self.__publicKeyFromPrivate(privateKeyFile)
            storeFile.private_key_bytes = self.__readPrivateKeyFromFile(privateKeyFile)
            storeFile.public_key_bytes = self.__readPublicKeyFromFile(publicKeyFile)
            # print storeFile.private_key_bytes
            # print storeFile.public_key_bytes
            imsPublicKey = self._imsRegister(username, password, self.__readPublicKeyFromFile(publicKeyFile))
            storeFile.ims_public_key_bytes = imsPublicKey
            # print imsPublicKey
            _json = postr.json
            if self._putStoreFile(store, storeFile):
                putr = self.kmsClient.put(self.kmsUrl + self.storesPrefix, data=json.dumps(store.dict()), headers=self.jsonHeaders)
                if putr.status_code != requests.codes.ok:
                    print "Sever refused request to save store file."
                    print putr.status_code
                    print putr.text
                else:
                    _json = putr.json
            self.__remove(publicKeyFile)
            self.__remove(privateKeyFile)
            return Store(_json)

    @requiresAuth
    def updateStore(self, store):
        """Update a store (where root file or permissions have changed, pressumably)."""
        putr = self.kmsClient.put(self.kmsUrl + self.storesPrefix, data=json.dumps(store.dict()), headers=self.jsonHeaders)
        if putr.status_code != requests.codes.ok:
            print putr.status_code
            print putr.text
        else:
            # update access
            storeFile = self._getStoreFile(store)
            self._putStoreFile(store, storeFile)
            return Store(putr.json)

    @requiresAuth
    def deleteStore(self, store):
        """Delete a store (with the same id as this store, anyway). This is not recoverable!!"""
        url = self.kmsUrl + self.storesPrefix + "/" + str(store.id)
        delr = self.kmsClient.delete(url)
        if delr.status_code != requests.codes.ok:
            print delr.status_code
            print url
            print delr.text

    @requiresAuth
    def changePassword(self, newPassword):
        """Change the user's password to a new one. This requires the old password be known!"""
        newKeyFile = self._changeKeyPassword(self.password, newPassword)
        key = None
        with open(newKeyFile, 'rb') as f:
            key = f.read()
        ans = requests.get("%s%s/%s" %(self.kmsUrl, self.saltPrefix, self.username))
        salt = base64.b64decode(ans.text)
        sendNewPassword = base64.b64encode(ab64_decode(pbkdf2_sha256.encrypt(self.username + newPassword, rounds=1000, salt=salt).split("$")[-1]))
        sendOldPassword = base64.b64encode(ab64_decode(pbkdf2_sha256.encrypt(self.username + self.password, rounds=1000, salt=salt).split("$")[-1]))
        reset = {"password":sendNewPassword, "oldPassword":sendOldPassword, "key":{"username":self.username, "key": key}}
        postr = self.kmsClient.post(self.kmsUrl + self.passwordResetPrefix, data=json.dumps(reset), headers=self.jsonHeaders)
        if postr.status_code != requests.codes.ok:
            print postr.status_code
            print postr.text

    @requiresAuth
    def listDirectory(self, store):
        """List all the files in the given store.

        :rtype: Directory

        """
        directory = None
        makeDirectory = False
        storeFile = self._getStoreFile(store)
        if storeFile and storeFile.bucket:
            provider = self._getCloudService(storeFile)
            tmpRoot = self._getPartFromCloud(store.index_codename, provider, storeFile.bucket)
            if tmpRoot:
                key = self._getKeyForFragment(store.index_codename, store.id)
                if key:
                    tmpRoot = self._decryptPart(tmpRoot, key, store.iv)
                    if tmpRoot:
                        jsonText = ""
                        with open(tmpRoot) as f:
                            jsonText = f.read()
                        directory = Directory(jsonText=jsonText)
                        self.__remove(tmpRoot)
                    else:
                        makeDirectory = True
                else:
                    makeDirectory = True
                self.__remove(tmpRoot)
            else:
                makeDirectory = True
        elif storeFile and len(storeFile.providers) > 0:
            self._createNewBucket(storeFile)
            makeDirectory = True
        else:
            print "has no bucket"
            print storeFile

        if makeDirectory:
            print "creating empty directory"
            directory = self._createAndStoreEmptyDirectory(store, storeFile)
            self.updateStore(store)
        return directory

    @requiresAuth
    def updateDirectory(self, directory, store):
        # Check for conflicts!!
        newName = unicode(uuid.uuid4())
        storeFile = self._getStoreFile(store)
        provider = self._getCloudService(storeFile)
        tmpText = json.dumps(directory.dict())
        tmpRoot = self.__temporaryFileWithBytes(tmpText)
        key = self._generateKey()
        store.iv = self._generateIV()
        tmpRoot = self._encryptPart(tmpRoot, key, store.iv)
        self._putPartInCloud(newName, provider, storeFile.bucket, tmpRoot)
        self._setKeyForFragment(newName, key, store.id)
        store.index_codename = newName
        self.updateStore(store)
        self.__remove(tmpRoot)
        pass

    @requiresAuth
    def getFile(self, store, files, threads=10):
        storeFile = self._getStoreFile(store)
        provider = self._getCloudService(storeFile)
        if files:
            keySets = self._getKeysForFragments(files.fragments, store.id)
            if keySets:
                clearFile = self.__temporaryFile()
                with open(clearFile, 'r+b') as f:
                    junk = b'\x00' * (files.remote_size)
                    f.write(junk)
                # with open(clearFile, 'wb') as f:
                promises = [];
                pool = ThreadPool(processes=threads)
                for fragmentName in keySets:
                    fragment = None
                    for frag in files.fragments:
                        if frag.name == fragmentName:
                            fragment = frag

                    promise = pool.apply_async(self._doFragmentDownload, [clearFile, fragmentName, provider, storeFile, keySets, fragment])
                    promises.append(promise)
                for order, promise in enumerate(promises):
                    self.__graphPrinter(order, len(promises))
                    success = promise.get()
                    self.__graphPrinter(order + 1, len(promises))

                    if not success:
                        print "Error on part!"
                        break
                files.local_path = clearFile
                sys.stdout.write("\n")
                sys.stdout.flush()
                return clearFile

    def __graphPrinter(self, done, total):
        completed = int(((done) * 100.0) / total)
        sys.stdout.write(" |" + "=" * completed + "-" * (100 - completed) + "|  \r")
        sys.stdout.flush()

    @requiresAuth
    def getBytes(self, start, end, store, files):
        storeFile = self._getStoreFile(store)
        provider = self._getCloudService(storeFile)
        fragments = files.fragments
        allData = None
        sizeSoFar = 0
        for fragment, idx in enumerate(fragments):
            if sizeSoFar + fragment.length >= start and sizeSoFar <= end:
                # Get this fragment
                key = self._getKeyForFragment(fragment, store.id)
                isOkay = False
                attempts = 0
                while not isOkay and attempts < 10:
                    tmpPart = self._getPartFromCloud(fragment.name, provider, storeFile.bucket)
                    if tmpPart:
                        isOkay = self._getVerifyPart(tmpPart, fragment.name, storeFile)
                        attempts += 1
                    else:
                        break
                if isOkay:
                    tmpPart2 = self._decryptPart(tmpPart, key, fragment.iv)
                    self.__remove(tmpPart)
                    if tmpPart2:
                        with open(tmpPart2, 'rb') as t:
                            if sizeSoFar <= start:
                                t.seek(start - sizeSoFar)
                            thisMuch = fragment.length
                            if sizeSoFar + fragment.length < end:
                                thisMuch = (sizeSoFar + fragment.length) - end
                            allData += t.read(thisMuch)
                        self.__remove(tmpPart2)
                else:
                    print "File corrupt."
                    break

            sizeSoFar += fragment.length
        return allData

    @requiresAuth
    def updateFile(self, store, file_, path, directory):
        """
        Update a file on TrustStore.

        :type store: Store
        :param store: a store to upload the file to

        :type file_: File
        :param file_: the File object to upload. Must have local_path

        :type path: Directory
        :param path:  the child folder to upload the File to. May be same as directory

        :type directory: Directory
        :param directory: root folder
        """
        # pr = cProfile.Profile()
        # pr.enable()
        print file_.name
        if file_.remote_size <= 0:
            file_.remote_size = os.path.getsize(file_.local_path)
        if store and file_ and path and directory:
            storeFile = self._getStoreFile(store)
            if storeFile:
                provider = self._getCloudService(storeFile)
                fileSize = os.path.getsize(file_.local_path)
                readFragments = 0
                displacement = 0
                optimalSize = self.__optimalPieceSize(fileSize)
                promises = [];
                pool = ThreadPool(processes=10)
                if len(file_.fragments) > 0:
                    self._deleteKeysForFragments(file_.fragments, store.id)
                    count404 = 0
                    for fragment in file_.fragments:
                        if self._deletePartFromCloud(fragment.name, provider, storeFile.bucket) == "404":
                            count404 += 1
                            sys.stdout.write(" Trying to delete old file. Parts not found in cloud: " + str(count404) + " of " + str(len(file_.fragments)) + "\r")
                            sys.stdout.flush()
                    file_.fragments = []
                while fileSize > readFragments * optimalSize:
                    sys.stdout.write(" " + str(readFragments) +  " optimal: " + str(optimalSize) + " file size: " + str(fileSize) + "\r")
                    sys.stdout.flush()
                    promise = pool.apply_async(self._doFragmentUpload, [file_.local_path, optimalSize, readFragments, displacement, storeFile, store.id, provider])
                    readFragments += 1
                        # ) #
                    displacement += optimalSize
                    promises.append(promise)
                for order, promise in enumerate(promises):
                    self.__graphPrinter(order, len(promises))
                    part = promise.get()
                    if part:
                        file_.fragments.append(part)
                    else:
                        print "Part broken?"
                        break
                    self.__graphPrinter(order + 1, len(promises))
                isUpdate = False
                for child in path.children:
                    if file_.name == child.name:
                        # Assume update.
                        child = file_
                        isUpdate = True
                if not isUpdate:
                    path.children.append(file_)
                # print directory
                self.updateDirectory(directory, store)
                sys.stdout.write("\n")
                sys.stdout.flush()
        else:
            print "Hey, you can't upload that! : " + file_.local_path
        # pr.disable()
        # pr.print_stats(1)

    @requiresAuth
    def delFile(self, store, fileName, path, directory):
        if store and fileName and path and directory:
            for child in path.children:
                print child.name
            path.children[:] = [child for child in path.children if self._deleteChildren(fileName, child, store)]
            self.updateDirectory(directory, store)
        elif store and path and directory:
            for child in path.children:
                print child.name
                self._deleteChildren(None, child, store)
            path.children = []
            self.updateDirectory(direcroty, store)
        else:
            print "Not enough information to delete!"

    @requiresAuth
    def addFile(self, store, file_, path, directory):
        if store and file_ and path and directory and os.path.exists(file_.local_path):
            self.updateFile(store, file_, path, directory)
        else:
            print "Hey, you can't upload that! " + file_.local_path

    ## Past here should no user go.

    def _createAndStoreEmptyDirectory(self, store, storeFile):
        directory = Directory()
        newName = unicode(uuid.uuid4())
        store.iv = self._generateIV()
        provider = self._getCloudService(storeFile)
        tmpText = json.dumps(directory.dict())
        tmpRoot = self.__temporaryFileWithString(tmpText)
        key = self._generateKey()
        tmpRoot = self._encryptPart(tmpRoot, key, store.iv)
        self._putPartInCloud(newName, provider, storeFile.bucket, tmpRoot)
        self._setKeyForFragment(newName, key, store.id)
        store.index_codename = newName
        self.__remove(tmpRoot)
        return directory

    def _createNewBucket(self, storeFile):
        storeFile.bucket = "truststore-" + str(uuid.uuid4())
        provider = self._getCloudService(storeFile)
        provider.create_bucket(storeFile.bucket)


    def _deleteChildren(self, fileName, child, store):
        if fileName == child.name or fileName == None:
            storeFile = self._getStoreFile(store)
            if storeFile:
                try:
                    self._deleteKeysForFragments(child.fragments, store.id)
                    provider = self._getCloudService(storeFile)
                    count404 = 0
                    for fragment in child.fragments:
                        if self._deletePartFromCloud(fragment.name, provider, storeFile.bucket) == "404":
                            count404 += 1
                            sys.stdout.write(" Trying to delete old file. Parts not found in cloud: " + str(count404) + " of " + str(len(child.fragments)) + "\r")
                            sys.stdout.flush()
                    child.fragments = []
                    print "\n"
                    print "Deleted file " + fileName
                except AttributeError:
                    for subs in child.children:
                        print subs.name
                        self._deleteChildren(None, subs, store)
            return False
        else:
            return True

    def _doFragmentDownload(self, filename, fragmentName, provider, storeFile, keySets, fragment):
        isOkay = False
        tries = 0
        sucess = False
        tmpPart = None
        while not isOkay and tries < 10:
            tmpPart = self._getPartFromCloud(fragmentName, provider, storeFile.bucket)
            if tmpPart:
                isOkay = self._getVerifyPart(tmpPart, fragmentName, storeFile)
                tries += 1
            else:
                break
        if isOkay:
            tmpPart2 = self._decryptPart(tmpPart, keySets[fragmentName], fragment.iv)
            if tmpPart2:
                with open(tmpPart2, 'rb') as t:
                    currentSize = os.path.getsize(filename)
                    data = t.read()
                    if currentSize <= fragment.offset:
                        with open(filename, 'a+b') as f:
                            if currentSize < fragment.offset:
                                sys.stdout.write("  !! :(")
                                junk = b'\x00' * (fragment.offset - currentSize)
                                f.write(junk)
                            f.write(data)
                            sys.stdout.write(" Piece " + str(fragment.order) + " \t\t\r")
                            sys.stdout.flush()
                            sucess = True
                    else:
                        with open(filename, 'r+b') as f:
                            f.seek(fragment.offset, 0)
                            sys.stdout.write(" Piece " + str(fragment.order) + " \t\t\r")  # + ": " + str(f.tell()) + ": " + str(len(data)) + "\r")
                            sys.stdout.flush()
                            f.write(data)
                            sucess = True
                self.__remove(tmpPart2)
        else:
            print "Part broken!!"
        self.__remove(tmpPart)
        return sucess

    def _doFragmentUpload(self, localPath, optimalSize, readFragments, displacement, storeFile, storeId, provider):
        fragment = Fragment()
        with open(localPath, 'rb') as f:
            fragment.length = optimalSize
            fragment.order = readFragments
            fragment.providers = storeFile.providers
            fragment.offset = displacement
            f.seek(displacement)
            data = f.read(fragment.length)
            # sys.stdout.write(" " + str(fragment.order) + " displacement: " + str(displacement) + " length: " + str(fragment.length)
 # + "\r")
            # sys.stdout.flush()
            if len(data) < fragment.length:
                fragment.length = len(data)
                # sys.stdout.write(" " + str(fragment.order) + " true length: " + str(len(data)) + "                 \r")
                sys.stdout.flush()

            tmpDataFile = self.__temporaryFileWithBytes(data)
            key = self._generateKey()
            fragment.name = unicode(uuid.uuid4())
            fragment.iv = self._generateIV()
            encryptedPart = self._encryptPart(tmpDataFile, key, fragment.iv)
            self._putPartInCloud(fragment.name, provider, storeFile.bucket, encryptedPart)
            self._setKeyForFragment(fragment.name, key, storeId)
            self._storeVerifyPart(encryptedPart, fragment.name, storeFile)
            # sys.stdout.write(" Finished fragment " + unicode(readFragments) + "                  \r")
            sys.stdout.flush()
            self.__remove(tmpDataFile)
            self.__remove(encryptedPart)
        return fragment

    def _getStoreFile(self, store):
        getr = self.kmsClient.get(self.kmsUrl + self.filesPrefix + "/" + unicode(store.id))  # + "/" + unicode(store.filename))
        message = getr.content
        if message:
            clear = self._decryptStoreFileUsingPrivateKeyfile(message, self.keyFile)
            if clear:
                # print clear
                try:
                    storefile = json.loads(clear)
                    storeProps = StoreProperties(eDict=storefile)
                    # print json.dumps(storeProps.dict())
                    return storeProps
                except simplejson.scanner.JSONDecodeError:
                    print "This profile is probably XML."
            else:
                print "Failed to decrypt profile, download status code: " + str(getr.status_code)

    def _putStoreFile(self, store, storeFile):
        success = False
        userList = []
        userList += store.readers
        userList += store.writers
        userList += store.administrators
        userList.append(store.owner)
        userList = list(set(userList))
        storeDict = storeFile.dict()
        messageFile = self._encryptStoreFileForUsers(json.dumps(storeDict), userList)
        # print json.dumps(storeDict)
        if messageFile:
            with open(messageFile, 'rb') as f:
                message = {'filename': ("file", f)} # this filename "file" is never read but it needs to be there to be valid so it's just whatever.
                url = self.kmsUrl + self.filesPrefix + "/" + unicode(store.id)
                postr = self.kmsClient.post(url, files=message)
                # print postr.request.headers
                # print postr.request.data
                self.__remove(messageFile)
                if postr.status_code == requests.codes.ok:
                    success = True
                else:
                    print postr.status_code
                    print postr.text
                    print "Failed to put Store File"
                    print url
                    print store.id
        return success

    def _getCloudService(self, storeFile):
        for provider in storeFile.providers:
            api = provider['api']
            if api == "nectar" or api == "s3":
                calling = botoConn.SubdomainCallingFormat()
                if api == "nectar":
                    calling = botoConn.OrdinaryCallingFormat()
                user = provider['userCredentials']
                endpoint = provider['endpoint']
                connection = botoConn.S3Connection(
                    aws_access_key_id=user['ident'],
                    aws_secret_access_key=user['secret'],
                    port=urlparse(endpoint).port,
                    host=urlparse(endpoint).hostname,
                    is_secure=True,
                    validate_certs=False,
                    calling_format=calling
                )
                return connection

    def _changeKeyPassword(self, old, new):
        newKey = self.keyFile + ".new"
        openssl = [self.openSSL, 'rsa', '-aes128', '-in', self.keyFile, '-out', newKey, '-passin', 'pass:' + old, '-passout', 'pass:' + new]
        okay = subprocess.call(openssl)
        if (okay == 0):
            self.keyFile = newKey
            return newKey
        else:
            return False

    def _getPublicCertFor(self, username):
        getr = self.kmsClient.get(self.kmsUrl + self.publicKeysPrefix + "/" + username)
        return UserPublicCertificate(getr.json)

    def _setPublicCert(self, username):
        publicCertFile = self.__makeRSACertificateFromPrivate(self.keyFile)
        if publicCertFile:
            certificate = self.__readCertificateFromFile(publicCertFile)
            cert = UserPublicCertificate(None, username, certificate)
            headers = {'Content-type': 'application/json', 'Accept': 'application/json'}
            self.kmsClient.post(self.kmsUrl + self.publicKeysPrefix, data=json.dumps(cert.dict()), headers=headers)
            self.__remove(publicCertFile)

    def _checkPublicCert(self, cert):
        print "Checking your certificate..."
        certFile = self.__writeCertificateToFile(cert.certificate)
        openssl = [self.openSSL, 'x509', '-in', certFile, '-modulus', '-noout']
        # certModulus = subprocess.check_output(openssl)
        p = subprocess.Popen(openssl, stdout=subprocess.PIPE)
        certModulus = p.communicate()[0]
        openssl = [self.openSSL, 'rsa', '-in', self.keyFile, '-modulus', '-noout']
        if self.password:
            openssl.append("-passin")
            openssl.append("pass:" + self.password)

        keyModulus = subprocess.check_output(openssl)

        if (certModulus == keyModulus):
            self.__remove(certFile)
            return True
        else:
            print certFile
            print certModulus
            print keyModulus
            return False


    def _getKeyForFragment(self, fragmentName, storeID):
        getr = self.kmsClient.get(self.kmsUrl + self.keysPrefix + "/" + unicode(storeID), params={'codenames': fragmentName})
        if getr.status_code != requests.codes.ok or not getr.json:
            print getr.status_code
            print "Failed to get key for fragment"
            if not getr.json:
                print "No key for that fragment known."
            return
        return base64.b64decode(getr.json[0]['key'])

    def _getKeysForFragments(self, fragments, storeID):
        codenames = [fragment.name for fragment in fragments]
        args = [iter(codenames)] * 20
        batches = izip_longest(fillvalue=None, *args)
        sets = {}
        for batch in batches:
            getr = self.kmsClient.get(self.kmsUrl + self.keysPrefix + "/" + unicode(storeID), params={'codenames': batch})
            if getr.json:
                sets.update(dict([(key['codename'], base64.b64decode(key['key'])) for key in getr.json]))
            elif getr.status_code != requests.codes.ok:
                print "Failed to get keys for " + str(len(batch)) + " fragments."
                print getr
                print getr.text
        return sets

    def _setKeysForFragments(self, storeID, keySets):
        data = json.dumps([{'codename': fragmentName, 'key': base64.b64encode(keySets[fragmentName]), 'expiryDate': None} for fragmentName in keySets])
        headers = {'Content-type': 'application/json', 'Accept': 'application/json'}
        postr = self.kmsClient.post(self.kmsUrl + self.keysPrefix + "/" + unicode(storeID), data=data, headers=headers)
        if postr.status_code != requests.codes.ok:
            print postr.status_code
            print "Failed to set keys for fragments"

    def _deleteKeysForFragments(self, fragments, storeID):
        codenames = [fragment.name for fragment in fragments]
        url = self.kmsUrl + self.keysPrefix + "/" + unicode(storeID)
        args = [iter(codenames)] * 20
        batches = izip_longest(fillvalue=None, *args)
        for batch in batches:
            try:
                delr = self.kmsClient.delete(url, params={'codenames': batch})
                if delr.status_code != requests.codes.ok:
                    print delr.status_code
                    print delr.text
                    print delr.content
                    print "Failed to delete keys."
            except requets.exceptions.ConnectionError as e:
                print (e)
                print url
                print batch


    def _setKeyForFragment(self, fragmentName, key, storeID):
        data = json.dumps([{'codename': fragmentName, 'key': base64.b64encode(key), 'expiryDate': None}])
        headers = {'Content-type': 'application/json', 'Accept': 'application/json'}
        postr = self.kmsClient.post(self.kmsUrl + self.keysPrefix + "/" + unicode(storeID), data=data, headers=headers)
        if postr.status_code != requests.codes.ok:
            print postr.status_code
            print postr.text
            print "Failed to set key for fragment"

    def _decryptStoreFileUsingPrivateKeyfile(self, message, keyFile):

        publicCertificate = self._getPublicCertFor(self.username).certificate

        clearMessage = ""
        messageFileName = self.__temporaryFileWithBytes(message)
        decryptedFileName = self.__temporaryFile()
        certFileName = self.__writeCertificateToFile(publicCertificate)

        openssl = [self.openSSL, 'cms', '-decrypt',
                   '-in', messageFileName,
                   '-out', decryptedFileName,
                   '-recip', certFileName,
                   '-inkey', keyFile,
                   '-inform', 'DER']

        if self.password:
            openssl.append("-passin")
            openssl.append("pass:" + self.password)

        # decrypt
        # openssl smime -decrypt -in encrypted -out decrypted -recip public_cert -inkey private_key
        try:
            okay = subprocess.call(openssl)

            if (okay == 0):
                with open(decryptedFileName) as decryptFile:
                    clearMessage = decryptFile.read()
            else:
                print okay
                print "Failed to decrypt store file "
                # + messageFileName
        except TypeError:
            print "Failed to call OpenSSL properly:"
            print openssl

        # print certFileName
        self.__remove(certFileName)
        # print messageFileName
        self.__remove(messageFileName)
        self.__remove(decryptedFileName)

        return clearMessage

    def _encryptStoreFileForUsers(self, plain, userlist):
        certificateList = [self._getPublicCertFor(username).certificate for username in userlist]
        certificateFileList = [self.__writeCertificateToFile(cert) for cert in certificateList]
        decryptedFileName = self.__temporaryFileWithString(plain)
        encryptFileName = self.__temporaryFile()

        argList = [self.openSSL, 'cms', '-encrypt', '-aes128',
                   '-in', decryptedFileName,
                   '-out', encryptFileName,
                   '-outform', 'DER']
        argList += certificateFileList
        # print argList
        okay = subprocess.call(argList)

        self.__remove(decryptedFileName)
        for cert in certificateFileList:
                self.__remove(cert)

        if okay != 0:
            print okay
            print "Failed to encrypt store file"
            print argList
            return False
        else:
            for cert in certificateFileList:
                self.__remove(cert)
        return encryptFileName

    def _getPartFromCloud(self, codename, provider, bucketName):
        attempts = 0
        while attempts < 20:
            try:
                bucket = provider.get_bucket(bucketName)
                k = botoKey(bucket)
                k.key = codename
                tmpPart = self.__temporaryFile()
                k.get_contents_to_filename(tmpPart)
                return tmpPart
            except (boto.exception.S3ResponseError, boto.exception.BotoServerError) as e:
                if e.status == 403 or e.status == "403":
                    sys.stdout.write(" Error talking to " + str(provider) + ", attempt: " + str(attempts + 1) + " (403)     \r")
                else:
                    sys.stdout.write(" Error talking to " + str(provider) + ", attempt: " + str(attempts + 1) + ": " + str(e) + " \r")
                sys.stdout.flush()
            attempts += 1

    def _putPartInCloud(self, codename, provider, bucketName, part):
        attempts = 0
        while attempts < 20:
            try:
                bucket = provider.get_bucket(bucketName)
                k = botoKey(bucket)
                k.key = codename
                k.set_contents_from_filename(part)
                return
            except boto.exception.S3ResponseError:
                sys.stdout.write(" Encountered a storing error talking to " + str(provider) + ", attempt: " + str(attempts + 1) + "\r")
                sys.stdout.flush()
            attempts += 1

    def _deletePartFromCloud(self, codename, provider, bucketName):
        attempts = 0
        while attempts < 10:
            try:
                bucket = provider.get_bucket(bucketName)
                k = botoKey(bucket)
                k.key = codename
                bucket.delete_key(k)
                return
            except boto.exception.S3ResponseError as e:
                if "404" not in str(e):
                    sys.stdout.write(" Encountered a delete error talking to " + str(provider) + ", attempt: " + str(attempts + 1) + "\r")
                    sys.stdout.flush()
                    attempts += 1
                else:
                    return "404"

    def _decryptPart(self, part, key, iv):
        decryptedPart = self.__temporaryFile()
        mode = '-aes-128-cbc'
        if not iv:
            mode = '-aes-128-ecb'
            iv = b'\x00'
        argList = [self.openSSL, 'enc', mode, '-d', '-in', part, '-out', decryptedPart, '-K', binascii.b2a_hex(key), '-iv', binascii.b2a_hex(iv)]
        okay = subprocess.call(argList)
        # print argList

        if okay != 0:
            print okay
            print "Failed to decrypt part"
            print argList
            self.__remove(decryptedPart)
            return
        return decryptedPart

    def _encryptPart(self, part, key, iv):
        encryptedPart = self.__temporaryFile()
        mode = '-aes-128-cbc'
        if not iv:
            mode = '-aes-128-ecb'
            iv = b'\x00'
        argList = [self.openSSL, 'enc', mode, '-in', part, '-out', encryptedPart, '-K', binascii.b2a_hex(key), '-iv', binascii.b2a_hex(iv)]
        okay = subprocess.call(argList)
        # print argList

        if okay != 0:
            print okay
            print "Failed to encrypt part"
            print argList
        return encryptedPart

    def _imsRegister(self, username, password, publicKey):
        url = urljoin(self.imsUrl, "rest/user/" + username)
        headers = {'passwd': password}
        # postr = requests.post(url, headers=headers, data=base64.b64encode(publicKey))

        postr = requests.post(url, headers=headers, data={'passwd': password, 'publicKey': base64.b64encode(publicKey)})
        if postr.text and postr.status_code == 200:
            text = self.__cleanString(postr.text)
            try:
                # print "IMS Public Key"
                # print text
                return base64.b64decode(text)
            except (UnicodeError, TypeError):
                print "Error getting IMS Public certificate."
                return postr.content
        else:
            print unicode(postr.status_code) + " received while trying to register with IMS."
            print postr.text

    def _getVerifyPart(self, part, partName, storeFile):
        isOkay = False
        url = urljoin(storeFile.ims_url, "rest/hash/" + storeFile.ims_user['ident'])
        headers = {'passwd': storeFile.ims_user['secret']}
        params = {"codename": partName}
        getr = requests.get(url, headers=headers, params=params)
        if getr.text and getr.json:
            # try:
                # resp = xmltodict.parse(getr.text)
                resp = getr.json
                # if "html" not in resp and "hashInfo" in resp:
                if "doubleSignature" in resp:
                    # resp = resp["hashInfo"]
                    keyFile = self.__writePublicKeyToFile(storeFile.public_key_bytes)

                    signature = base64.b64decode(resp["doubleSignature"])
                    sigFile = self.__temporaryFileWithBytes(signature)
                    meta = resp["signableMetaInfo"]
                    plainfile = self.__temporaryFileWithString(meta)

                    argList = [self.openSSL, 'dgst', '-sha1', '-verify', keyFile, '-signature', sigFile, plainfile]
                    okay = subprocess.check_output(argList)
                    verified = (okay and okay == "Verified OK\n")

                    self.__remove(sigFile)
                    self.__remove(keyFile)
                    self.__remove(plainfile)
                    if not verified:
                        print "Unable to verify part."
                        return isOkay

                    sums, imsSum = "", ""
                    with open(part, 'rb') as f:
                        sums = hashlib.md5(f.read()).hexdigest()
                        imsSum = meta[:meta.find("$")]
                    if sums == imsSum:
                        isOkay = True
                    else:
                        print imsSum
                        print sums
                else:
                    print resp
            # except Exception as e:
            #     print url
            #     print getr.text
            #     print e
        return isOkay

    def _storeVerifyPart(self, part, partName, storeFile):
        isOkay = False
        headers = {'passwd': storeFile.ims_user['secret']}
        sums = ""
        with open(part, 'rb') as f:
            sums = hashlib.md5(f.read()).hexdigest()

        imsKeyfile = self.__writePublicKeyToFile(storeFile.ims_public_key_bytes)

        signable = sums + "$" + partName + "$" + datetime.utcnow().isoformat()

        signableFile = self.__temporaryFileWithString(signable)

        signable64 = base64.b64encode(signable)
        url = urljoin(storeFile.ims_url, "rest/sign/" + signable64)
        getr = requests.get(url, headers=headers)
        if getr.text: # and "string" in getr.text:
            resp = getr.text # xmltodict.parse(getr.text)
            resp = resp.replace("\\r\\n", "\n")
            signature = base64.b64decode(resp) #["string"])
            # print resp
            # print len(signature)
            # print len(storeFile.ims_public_key_bytes)

            imsSignatureFile = self.__temporaryFileWithBytes(signature)

            argList = [self.openSSL, 'dgst', '-sha1', '-verify', imsKeyfile, '-signature', imsSignatureFile, signableFile]
            okay = subprocess.check_output(argList)
            verified = (okay == "Verified OK\n")
            self.__remove(imsSignatureFile)
            if not verified:
                print "Unable to make digest."
                print argList
            else:
                nonce = os.urandom(4)
                nonceFile = self.__temporaryFileWithBytes(nonce)
                encryptedNonceFile = self.__temporaryFile()
                argList = [self.openSSL, 'rsautl', '-encrypt', '-pubin', '-inkey', imsKeyfile, '-in', nonceFile, '-out', encryptedNonceFile]
                okay = subprocess.call(argList)
                if okay != 0:
                    print "Unable to encrypt test."
                    print argList
                else:
                    encryptedNonce = ""
                    with open(encryptedNonceFile, 'rb') as f:
                        encryptedNonce = f.read()

                    signatureFile = self.__temporaryFile()
                    privKeyFile = self.__writePrivateKeyToFile(storeFile.private_key_bytes)
                    argList = [self.openSSL, 'dgst', '-sha1', '-sign', privKeyFile, '-out', signatureFile, signableFile]
                    okay = subprocess.call(argList)
                    if okay == 0:

                        signature = ""
                        with open(signatureFile, 'rb') as f:
                            signature = f.read()

                        url = urljoin(storeFile.ims_url, "rest/check/" + storeFile.ims_user['ident'])
                        params = {
                            "nonce": base64.b64encode(encryptedNonce),
                            "doubleSig": base64.b64encode(signature),
                            "xml": signable
                        }
                        getr = requests.get(url, headers=headers, params=params)
                        # print url
                        # print headers
                        # print params
                        if getr.text: # and "string" in getr.text and xmltodict.parse(getr.text)["string"]:
                            resp = self.__cleanString(getr.text)
                            resp = base64.b64decode(resp) #xmltodict.parse(getr.text)["string"])

                            imsEncryptedNonceFile = self.__temporaryFileWithBytes(resp)

                            imsNonceFile = self.__temporaryFile()
                            argList = [self.openSSL, 'rsautl', '-decrypt', '-inkey', privKeyFile, '-out', imsNonceFile, '-in', imsEncryptedNonceFile]
                            okay = subprocess.call(argList)
                            if okay != 0:
                                return isOkay

                            imsNonce = ""
                            with open(imsNonceFile, 'r') as f:
                                imsNonce = f.read()

                            if imsNonce != nonce:
                                return isOkay

                            url = urljoin(storeFile.ims_url, "rest/storehash/" + storeFile.ims_user['ident'])
                            payload = {"xml": signable, "doublesign": base64.b64encode(signature), "codename": partName}
                            postr = requests.post(url, headers=headers, params=params, data=payload)
                            if postr.text and "OK" in postr.text: # and "string" in postr.text
                                isOkay = True
                            else:
                                print postr.text

                            self.__remove(imsEncryptedNonceFile)
                            self.__remove(imsNonceFile)
                        else:
                            print getr.text
                    else:
                        print "Unable to sign digest."
                    self.__remove(signatureFile)
                    self.__remove(privKeyFile)
                self.__remove(nonceFile)
                self.__remove(encryptedNonceFile)
            self.__remove(imsSignatureFile)
        else:
            print getr.text
        self.__remove(imsKeyfile)
        self.__remove(signableFile)
        return isOkay

    def __temporaryFile(self):
        fileName = ""
        with NamedTemporaryFile(delete=False) as aFile:
            fileName = aFile.name
        return fileName

    def __temporaryFileWithString(self, string):
        fileName = self.__temporaryFile()
        with open(fileName, 'w') as f:
            if string:
                f.write(string)
        return fileName

    def __temporaryFileWithBytes(self, bytes):
        fileName = self.__temporaryFile()
        with open(fileName, 'wb') as f:
            if bytes:
                f.write(bytes)
        return fileName

    def __writeCertificateToFile(self, cert):
        certFile = self.__temporaryFile()
        header = DASHES + BEGIN + " " + CERTIFICATE + DASHES
        footer = DASHES + END + " " + CERTIFICATE + DASHES
        with open(certFile, 'w') as f:
            if not cert.startswith(header):
                f.write(header + "\n")
            formatted = "\n".join([line for line in cert.splitlines() if line])
            f.write(formatted)
            if not cert.endswith(footer):
                f.write("\n" + footer)
        return certFile

    def __readCertificateFromFile(self, file_):
        cert = None
        with open(file_, 'r') as f:
            first = f.readline()
            if first == DASHES + BEGIN + " " + CERTIFICATE + DASHES + "\n":
                cert = f.readlines()
                cert = [line for line in cert if line]
                cert = cert[:-1]
                cert = "\n".join(cert)
            else:
                cert = first + f.read()
        return cert

    def __writePublicKeyToFile(self, key):
        return self.__writeKeyToFile(key, PUBLIC)

    def __readPublicKeyFromFile(self, keyFile):
        return self.__readKeyFromFile(keyFile, PUBLIC)

    def __readPrivateKeyFromFile(self, keyFile):
        return self.__readKeyFromFile(keyFile, PRIVATE)

    def __writePrivateKeyToFile(self, key):
        return self.__writeKeyToFile(key, PRIVATE)

    def __writeKeyToFile(self, key, keyname):
        keyFile = self.__temporaryFile()
        if not self.__probablyBase64(key):
            key = base64.b64encode(key)
        publicKey = textwrap.fill(key, 63)
        # print "KEY"
        # print publicKey
        with open(keyFile, 'w') as f:
            f.write(DASHES + BEGIN + " " + keyname + " " + KEY + DASHES + "\n")
            f.write(publicKey)
            f.write("\n" + DASHES + END + " " + keyname + " " + KEY + DASHES)
        return keyFile

    def __readKeyFromFile(self, keyFile, keyname):
        key = None
        with open(keyFile) as f:
            key = f.read()
            if key.startswith(DASHES + BEGIN + " " + keyname + " " + KEY + DASHES + "\n"):
                key = "".join(key.splitlines()[1:-1])
            if "\\r\\n" in key:
                key = key.replace("\\r\\n", "")
            if self.__probablyBase64(key):
                key = base64.b64decode(key)
        return key

    def _getUsername(self):
        getr = self.kmsClient.get(self.kmsUrl + self.usernamesPrefix)
        username = None
        if getr.json:
            try:
                username = getr.json[0]
            except KeyError:
                print getr.json
        if username:
            myCert = self._getPublicCertFor(username)
            if not myCert.certificate:
                self._setPublicCert(username)
            elif not self._checkPublicCert(myCert):
                print "Woah, your certificate isn't valid!"
                if self.headless:
                    sys.exit(1)
                else:
                    reset = raw_input("A new certificate can be created, but all other users will need to re-add you to any shared stores, and you will lose access to any stores you own. Do you want to make a new certificate? (Y for yes, all other input means no): ")
                    if reset == 'Y' or reset == 'y' or reset == 'yes' or reset == "YES":
                        self._setPublicCert(username)
                    else: sys.exit(0)

        return username

    def _generateKey(self):
        """128-bit Pseudo-Random Key"""
        return os.urandom(16)

    def _generateIV(self):
        """128-bit IV"""
        return os.urandom(16)

    def _generateKeypair(self, filename=None, encrypt=True):
        if not filename:
            filename = self.__temporaryFile()
        unencrypted = self.__temporaryFile()
        if self.password or not encrypt:
            # argList = [self.openSSL, 'genrsa', '-aes128', '-out', filename, '-passout', 'pass:' + self.password, '4096']
            argList = [self.openSSL, 'genrsa', '-out', unencrypted, '4096']
            try:
                okay = subprocess.call(argList)
                if okay != 0:
                    self.__remove(unencrypted)
                    unencrypted = False
                    filename = False
                elif encrypt:
                    argList = [self.openSSL, 'pkcs8', '-topk8', '-v2', 'aes128', '-in', unencrypted, '-out', filename, '-passout', 'pass:' + self.password]
                    try:
                        okay = subprocess.call(argList)
                        if okay != 0:
                            filename = False
                    except TypeError:
                        filename = False
                else:
                    self.__remove(filename)
                    filename = unencrypted
            except TypeError:
                self.__remove(filename)
                filename = False
        else:
            self.__remove(filename)
            filename = False
        if encrypt:
            self.__remove(unencrypted)
        return filename

    def _generatePKCS1Keypair(self):
        filename = self.__temporaryFile()
        certFile = self.__temporaryFile()
        confFile = os.path.join(os.path.realpath(os.path.dirname(__file__)), 'certs.conf')
        days = 365*20
        argList = [self.openSSL, 'req', '-x509', '-out', certFile, '-newkey', 'rsa:4096', '-keyout', filename, '-days', unicode(days), '-nodes', '-config', confFile]
        try:
            okay = subprocess.call(argList)
            if okay != 0:
                self.__remove(filename)
                filename = False
        except TypeError:
            self.__remove(filename)
            filename = False
        self.__remove(certFile)
        return filename

    def __makeRSACertificateFromPrivate(self, privateKeyFile):
        certificateFile = self.__temporaryFile()
        confFile = os.path.join(os.path.realpath(os.path.dirname(__file__)), 'certs.conf')
        days = 365*20
        argList = [self.openSSL, 'req', '-new', '-x509', '-key', privateKeyFile, '-out', certificateFile, '-days', unicode(days), '-nodes', '-config', confFile]
        if self.password:
            argList.append("-passin")
            argList.append("pass:" + self.password)
        try:
            okay = subprocess.call(argList)
            if okay != 0:
                self.__remove(certificateFile)
                return False
            return certificateFile
        except TypeError:
            print "Error making certficate."
            print argList
            self.__remove(certificateFile)

    def __publicKeyFromPrivate(self, privateKeyFile):
        publicKeyFile = self.__temporaryFile()
        argList = [self.openSSL, 'rsa', '-pubout', '-outform', 'DER', '-in', privateKeyFile, '-out', publicKeyFile]
        if self.password:
            argList.append("-passin")
            argList.append("pass:" + self.password)
        okay = subprocess.call(argList)
        if okay == 0:
            return publicKeyFile
        self.__remove(publicKeyFile)
        return False

    def __probablyBase64(self, s):
        return (len(''.join(s.split())) % 4 == 0) and re.match('^[A-Za-z0-9+/]+[=]{0,2}$', s)

    def __cleanString(self, s):
        openTag = "<string>"
        closeTag = "</string>"

        if s.startswith(openTag):
            return s[len(openTag):-len(closeTag)]
        s = s.replace("\\r\\n", "\n")
        return s

    def __optimalPieceSize(self, totalSize):
        Dpieces = totalSize / 100
        if Dpieces < self.pieceSize:
            return self.pieceSize
        return Dpieces

    def __remove(self, path):
        try:
            os.remove(path)
        except OSError:
                pass

class Store(object):
    id = None
    index_codename = None
    friendly_name = None
    owner = None
    iv = None
    # readers = []
    # writers = []
    # administrators = []

    def __init__(self, props=None, index_codename="", friendly_name="", filename="", owner="", readers=None, writers=None, administrators=None, iv=None):
        self.readers = []
        self.writers = []
        self.administrators = []
        if props:
            self.id = props['id']
            self.index_codename = props['indexCodename']
            self.friendly_name = props['friendly_name']
            self.owner = props['owner']
            self.readers = props['readers']
            self.writers = props['writers']
            self.administrators = props['administrators']
            if props['iv']:
                self.iv = base64.b64decode(props['iv'])
        else:
            self.index_codename = index_codename
            self.friendly_name = friendly_name
            # self.filename = filename
            self.owner = owner
        if readers:
            self.readers = readers
        if writers:
            self.writers = writers
        if administrators:
            self.administrators = administrators
            self.iv = iv

    def __str__(self):
        message = u'\n' + unicode(self.friendly_name)
        message += u'\n\tOwner: ' + unicode(self.owner)
        message += u'\n\tReaders: ' + unicode(self.readers)
        message += u'\n\tWriters: ' + unicode(self.writers)
        message += u'\n\tAdmins: ' + unicode(self.administrators)
        return message

    def __repr__(self):
        return str(self)

    def dict(self):
        this = {
            "id": self.id,
            "indexCodename": self.index_codename,
            "friendly_name": self.friendly_name,
            # "fileName": self.filename,
            "owner": self.owner,
            "readers": self.readers,
            "writers": self.writers,
            "administrators": self.administrators
        }
        if self.iv:
            this["iv"] = base64.b64encode(self.iv)
        return this


class StoreProperties(object):
    sas_url = ""
    providers = []
    kms_url = ""
    kms_user = None
    ims_url = ""
    ims_user = None
    bucket = None
    public_key_bytes = None
    private_key_bytes = None
    ims_public_key_bytes = None

    def __init__(self, eDict=None):
        self.providers = []
        if eDict:
            self.kms_url = eDict['kmsServiceUrl']
            self.private_key_bytes = base64.b64decode(eDict['privateKeyBytes'])
            self.sas_url = eDict['sasServiceUrl']
            self.ims_user = eDict['imsUser']
            self.providers = eDict['storageProviders']
            self.public_key_bytes = base64.b64decode(eDict['publicKeyBytes'])
            self.ims_url = eDict['imsServiceUrl']
            self.kms_user = eDict['kmsUser']
            self.bucket = eDict['workspace']
            self.ims_public_key_bytes = base64.b64decode(eDict['imsPublicKeyBytes'])

    def dict(self):
        return {'kmsServiceUrl': self.kms_url,
                'privateKeyBytes': binascii.b2a_base64(self.private_key_bytes),
                'sasServiceUrl': self.sas_url,
                'imsUser': self.ims_user,
                'storageProviders': self.providers,
                'publicKeyBytes': binascii.b2a_base64(self.public_key_bytes),
                'imsServiceUrl': self.ims_url,
                'kmsUser': self.kms_user,
                'workspace': self.bucket,
                'imsPublicKeyBytes': binascii.b2a_base64(self.ims_public_key_bytes)}

    def __repr__(self):
        return json.dumps(self.dict(), sort_keys=True, indent=4)


class UserPublicCertificate(object):
    username = None
    certificate = None

    def __init__(self, props=None, username="", certificate=None):
        if props:
            self.username = props['username']
            self.certificate = props['key']
        else:
            self.username = username
            self.certificate = certificate

    def dict(self):
        return {
            'username': self.username,
            'key': self.certificate
            }

    def __repr__(self):
        return json.dumps(self.dict(), sort_keys=True, indent=4)

class UserPrivateKey(UserPublicCertificate):
    pass


class Config(object):
    def __init__(self, ims, kms, key, secret):
        self.kmsUrl = kms
        self.imsUrl = ims
        self.client_key = key
        self.client_secret = secret


class TrustStoreClientAuthenticationException(Exception):
    def __init__(self, message, isConfigured=False):
        self.message = message
        self.isConfigured = isConfigured

    def __str__(self):
        msg = repr(self.message) + " note: "
        if self.isConfigured:
            msg += " login attempt was made."
        else:
            msg += " not currently authenticated"
        return msg


class OpenSSLVersionException(Exception):
    def __init__(self, version):
        self.version = version

    def __str__(self):
        return "The version of openssl (" + self.version + ") is too old!"