Mercurial > repos > bcclaywell > argo_navis

comparison venv/lib/python2.7/site-packages/boto/gs/key.py @ 0:d67268158946 draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
planemo upload commit a3f181f5f126803c654b3a66dd4e83a48f7e203b
author bcclaywell
date Mon, 12 Oct 2015 17:43:33 -0400
parents
children
comparison
equal deleted inserted replaced
-1:000000000000 0:d67268158946
1 # Copyright 2010 Google Inc.
2 #
3 # Permission is hereby granted, free of charge, to any person obtaining a
4 # copy of this software and associated documentation files (the
5 # "Software"), to deal in the Software without restriction, including
6 # without limitation the rights to use, copy, modify, merge, publish, dis-
7 # tribute, sublicense, and/or sell copies of the Software, and to permit
8 # persons to whom the Software is furnished to do so, subject to the fol-
9 # lowing conditions:
10 #
11 # The above copyright notice and this permission notice shall be included
12 # in all copies or substantial portions of the Software.
13 #
14 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
15 # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
16 # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
17 # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
18 # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
20 # IN THE SOFTWARE.
21
22 import base64
23 import binascii
24 import os
25 import re
26
27 from boto.compat import StringIO
28 from boto.exception import BotoClientError
29 from boto.s3.key import Key as S3Key
30 from boto.s3.keyfile import KeyFile
31 from boto.utils import compute_hash
32 from boto.utils import get_utf8_value
33
34 class Key(S3Key):
35 """
36 Represents a key (object) in a GS bucket.
37
38 :ivar bucket: The parent :class:`boto.gs.bucket.Bucket`.
39 :ivar name: The name of this Key object.
40 :ivar metadata: A dictionary containing user metadata that you
41 wish to store with the object or that has been retrieved from
42 an existing object.
43 :ivar cache_control: The value of the `Cache-Control` HTTP header.
44 :ivar content_type: The value of the `Content-Type` HTTP header.
45 :ivar content_encoding: The value of the `Content-Encoding` HTTP header.
46 :ivar content_disposition: The value of the `Content-Disposition` HTTP
47 header.
48 :ivar content_language: The value of the `Content-Language` HTTP header.
49 :ivar etag: The `etag` associated with this object.
50 :ivar last_modified: The string timestamp representing the last
51 time this object was modified in GS.
52 :ivar owner: The ID of the owner of this object.
53 :ivar storage_class: The storage class of the object. Currently, one of:
54 STANDARD | DURABLE_REDUCED_AVAILABILITY.
55 :ivar md5: The MD5 hash of the contents of the object.
56 :ivar size: The size, in bytes, of the object.
57 :ivar generation: The generation number of the object.
58 :ivar metageneration: The generation number of the object metadata.
59 :ivar encrypted: Whether the object is encrypted while at rest on
60 the server.
61 :ivar cloud_hashes: Dictionary of checksums as supplied by the storage
62 provider.
63 """
64
65 def __init__(self, bucket=None, name=None, generation=None):
66 super(Key, self).__init__(bucket=bucket, name=name)
67 self.generation = generation
68 self.meta_generation = None
69 self.cloud_hashes = {}
70 self.component_count = None
71
72 def __repr__(self):
73 if self.generation and self.metageneration:
74 ver_str = '#%s.%s' % (self.generation, self.metageneration)
75 else:
76 ver_str = ''
77 if self.bucket:
78 return '<Key: %s,%s%s>' % (self.bucket.name, self.name, ver_str)
79 else:
80 return '<Key: None,%s%s>' % (self.name, ver_str)
81
82 def endElement(self, name, value, connection):
83 if name == 'Key':
84 self.name = value
85 elif name == 'ETag':
86 self.etag = value
87 elif name == 'IsLatest':
88 if value == 'true':
89 self.is_latest = True
90 else:
91 self.is_latest = False
92 elif name == 'LastModified':
93 self.last_modified = value
94 elif name == 'Size':
95 self.size = int(value)
96 elif name == 'StorageClass':
97 self.storage_class = value
98 elif name == 'Owner':
99 pass
100 elif name == 'VersionId':
101 self.version_id = value
102 elif name == 'Generation':
103 self.generation = value
104 elif name == 'MetaGeneration':
105 self.metageneration = value
106 else:
107 setattr(self, name, value)
108
109 def handle_version_headers(self, resp, force=False):
110 self.metageneration = resp.getheader('x-goog-metageneration', None)
111 self.generation = resp.getheader('x-goog-generation', None)
112
113 def handle_restore_headers(self, response):
114 return
115
116 def handle_addl_headers(self, headers):
117 for key, value in headers:
118 if key == 'x-goog-hash':
119 for hash_pair in value.split(','):
120 alg, b64_digest = hash_pair.strip().split('=', 1)
121 self.cloud_hashes[alg] = binascii.a2b_base64(b64_digest)
122 elif key == 'x-goog-component-count':
123 self.component_count = int(value)
124 elif key == 'x-goog-generation':
125 self.generation = value
126 # Use x-goog-stored-content-encoding and
127 # x-goog-stored-content-length to indicate original content length
128 # and encoding, which are transcoding-invariant (so are preferable
129 # over using content-encoding and size headers).
130 elif key == 'x-goog-stored-content-encoding':
131 self.content_encoding = value
132 elif key == 'x-goog-stored-content-length':
133 self.size = int(value)
134
135 def open_read(self, headers=None, query_args='',
136 override_num_retries=None, response_headers=None):
137 """
138 Open this key for reading
139
140 :type headers: dict
141 :param headers: Headers to pass in the web request
142
143 :type query_args: string
144 :param query_args: Arguments to pass in the query string
145 (ie, 'torrent')
146
147 :type override_num_retries: int
148 :param override_num_retries: If not None will override configured
149 num_retries parameter for underlying GET.
150
151 :type response_headers: dict
152 :param response_headers: A dictionary containing HTTP
153 headers/values that will override any headers associated
154 with the stored object in the response. See
155 http://goo.gl/EWOPb for details.
156 """
157 # For GCS we need to include the object generation in the query args.
158 # The rest of the processing is handled in the parent class.
159 if self.generation:
160 if query_args:
161 query_args += '&'
162 query_args += 'generation=%s' % self.generation
163 super(Key, self).open_read(headers=headers, query_args=query_args,
164 override_num_retries=override_num_retries,
165 response_headers=response_headers)
166
167 def get_file(self, fp, headers=None, cb=None, num_cb=10,
168 torrent=False, version_id=None, override_num_retries=None,
169 response_headers=None, hash_algs=None):
170 query_args = None
171 if self.generation:
172 query_args = ['generation=%s' % self.generation]
173 self._get_file_internal(fp, headers=headers, cb=cb, num_cb=num_cb,
174 override_num_retries=override_num_retries,
175 response_headers=response_headers,
176 hash_algs=hash_algs,
177 query_args=query_args)
178
179 def get_contents_to_file(self, fp, headers=None,
180 cb=None, num_cb=10,
181 torrent=False,
182 version_id=None,
183 res_download_handler=None,
184 response_headers=None,
185 hash_algs=None):
186 """
187 Retrieve an object from GCS using the name of the Key object as the
188 key in GCS. Write the contents of the object to the file pointed
189 to by 'fp'.
190
191 :type fp: File -like object
192 :param fp:
193
194 :type headers: dict
195 :param headers: additional HTTP headers that will be sent with
196 the GET request.
197
198 :type cb: function
199 :param cb: a callback function that will be called to report
200 progress on the upload. The callback should accept two
201 integer parameters, the first representing the number of
202 bytes that have been successfully transmitted to GCS and
203 the second representing the size of the to be transmitted
204 object.
205
206 :type cb: int
207 :param num_cb: (optional) If a callback is specified with the
208 cb parameter this parameter determines the granularity of
209 the callback by defining the maximum number of times the
210 callback will be called during the file transfer.
211
212 :type torrent: bool
213 :param torrent: If True, returns the contents of a torrent
214 file as a string.
215
216 :type res_upload_handler: ResumableDownloadHandler
217 :param res_download_handler: If provided, this handler will
218 perform the download.
219
220 :type response_headers: dict
221 :param response_headers: A dictionary containing HTTP
222 headers/values that will override any headers associated
223 with the stored object in the response. See
224 http://goo.gl/sMkcC for details.
225 """
226 if self.bucket is not None:
227 if res_download_handler:
228 res_download_handler.get_file(self, fp, headers, cb, num_cb,
229 torrent=torrent,
230 version_id=version_id,
231 hash_algs=hash_algs)
232 else:
233 self.get_file(fp, headers, cb, num_cb, torrent=torrent,
234 version_id=version_id,
235 response_headers=response_headers,
236 hash_algs=hash_algs)
237
238 def compute_hash(self, fp, algorithm, size=None):
239 """
240 :type fp: file
241 :param fp: File pointer to the file to hash. The file
242 pointer will be reset to the same position before the
243 method returns.
244
245 :type algorithm: zero-argument constructor for hash objects that
246 implements update() and digest() (e.g. hashlib.md5)
247
248 :type size: int
249 :param size: (optional) The Maximum number of bytes to read
250 from the file pointer (fp). This is useful when uploading
251 a file in multiple parts where the file is being split
252 in place into different parts. Less bytes may be available.
253 """
254 hex_digest, b64_digest, data_size = compute_hash(
255 fp, size=size, hash_algorithm=algorithm)
256 # The internal implementation of compute_hash() needs to return the
257 # data size, but we don't want to return that value to the external
258 # caller because it changes the class interface (i.e. it might
259 # break some code), so we consume the third tuple value here and
260 # return the remainder of the tuple to the caller, thereby preserving
261 # the existing interface.
262 self.size = data_size
263 return (hex_digest, b64_digest)
264
265 def send_file(self, fp, headers=None, cb=None, num_cb=10,
266 query_args=None, chunked_transfer=False, size=None,
267 hash_algs=None):
268 """
269 Upload a file to GCS.
270
271 :type fp: file
272 :param fp: The file pointer to upload. The file pointer must
273 point point at the offset from which you wish to upload.
274 ie. if uploading the full file, it should point at the
275 start of the file. Normally when a file is opened for
276 reading, the fp will point at the first byte. See the
277 bytes parameter below for more info.
278
279 :type headers: dict
280 :param headers: The headers to pass along with the PUT request
281
282 :type num_cb: int
283 :param num_cb: (optional) If a callback is specified with the
284 cb parameter this parameter determines the granularity of
285 the callback by defining the maximum number of times the
286 callback will be called during the file
287 transfer. Providing a negative integer will cause your
288 callback to be called with each buffer read.
289
290 :type query_args: string
291 :param query_args: Arguments to pass in the query string.
292
293 :type chunked_transfer: boolean
294 :param chunked_transfer: (optional) If true, we use chunked
295 Transfer-Encoding.
296
297 :type size: int
298 :param size: (optional) The Maximum number of bytes to read
299 from the file pointer (fp). This is useful when uploading
300 a file in multiple parts where you are splitting the file
301 up into different ranges to be uploaded. If not specified,
302 the default behaviour is to read all bytes from the file
303 pointer. Less bytes may be available.
304
305 :type hash_algs: dictionary
306 :param hash_algs: (optional) Dictionary of hash algorithms and
307 corresponding hashing class that implements update() and digest().
308 Defaults to {'md5': hashlib.md5}.
309 """
310 self._send_file_internal(fp, headers=headers, cb=cb, num_cb=num_cb,
311 query_args=query_args,
312 chunked_transfer=chunked_transfer, size=size,
313 hash_algs=hash_algs)
314
315 def delete(self, headers=None):
316 return self.bucket.delete_key(self.name, version_id=self.version_id,
317 generation=self.generation,
318 headers=headers)
319
320 def add_email_grant(self, permission, email_address):
321 """
322 Convenience method that provides a quick way to add an email grant to a
323 key. This method retrieves the current ACL, creates a new grant based on
324 the parameters passed in, adds that grant to the ACL and then PUT's the
325 new ACL back to GS.
326
327 :type permission: string
328 :param permission: The permission being granted. Should be one of:
329 READ|FULL_CONTROL
330 See http://code.google.com/apis/storage/docs/developer-guide.html#authorization
331 for more details on permissions.
332
333 :type email_address: string
334 :param email_address: The email address associated with the Google
335 account to which you are granting the permission.
336 """
337 acl = self.get_acl()
338 acl.add_email_grant(permission, email_address)
339 self.set_acl(acl)
340
341 def add_user_grant(self, permission, user_id):
342 """
343 Convenience method that provides a quick way to add a canonical user
344 grant to a key. This method retrieves the current ACL, creates a new
345 grant based on the parameters passed in, adds that grant to the ACL and
346 then PUT's the new ACL back to GS.
347
348 :type permission: string
349 :param permission: The permission being granted. Should be one of:
350 READ|FULL_CONTROL
351 See http://code.google.com/apis/storage/docs/developer-guide.html#authorization
352 for more details on permissions.
353
354 :type user_id: string
355 :param user_id: The canonical user id associated with the GS account to
356 which you are granting the permission.
357 """
358 acl = self.get_acl()
359 acl.add_user_grant(permission, user_id)
360 self.set_acl(acl)
361
362 def add_group_email_grant(self, permission, email_address, headers=None):
363 """
364 Convenience method that provides a quick way to add an email group
365 grant to a key. This method retrieves the current ACL, creates a new
366 grant based on the parameters passed in, adds that grant to the ACL and
367 then PUT's the new ACL back to GS.
368
369 :type permission: string
370 :param permission: The permission being granted. Should be one of:
371 READ|FULL_CONTROL
372 See http://code.google.com/apis/storage/docs/developer-guide.html#authorization
373 for more details on permissions.
374
375 :type email_address: string
376 :param email_address: The email address associated with the Google
377 Group to which you are granting the permission.
378 """
379 acl = self.get_acl(headers=headers)
380 acl.add_group_email_grant(permission, email_address)
381 self.set_acl(acl, headers=headers)
382
383 def add_group_grant(self, permission, group_id):
384 """
385 Convenience method that provides a quick way to add a canonical group
386 grant to a key. This method retrieves the current ACL, creates a new
387 grant based on the parameters passed in, adds that grant to the ACL and
388 then PUT's the new ACL back to GS.
389
390 :type permission: string
391 :param permission: The permission being granted. Should be one of:
392 READ|FULL_CONTROL
393 See http://code.google.com/apis/storage/docs/developer-guide.html#authorization
394 for more details on permissions.
395
396 :type group_id: string
397 :param group_id: The canonical group id associated with the Google
398 Groups account you are granting the permission to.
399 """
400 acl = self.get_acl()
401 acl.add_group_grant(permission, group_id)
402 self.set_acl(acl)
403
404 def set_contents_from_file(self, fp, headers=None, replace=True,
405 cb=None, num_cb=10, policy=None, md5=None,
406 res_upload_handler=None, size=None, rewind=False,
407 if_generation=None):
408 """
409 Store an object in GS using the name of the Key object as the
410 key in GS and the contents of the file pointed to by 'fp' as the
411 contents.
412
413 :type fp: file
414 :param fp: The file whose contents are to be uploaded.
415
416 :type headers: dict
417 :param headers: (optional) Additional HTTP headers to be sent with the
418 PUT request.
419
420 :type replace: bool
421 :param replace: (optional) If this parameter is False, the method will
422 first check to see if an object exists in the bucket with the same
423 key. If it does, it won't overwrite it. The default value is True
424 which will overwrite the object.
425
426 :type cb: function
427 :param cb: (optional) Callback function that will be called to report
428 progress on the upload. The callback should accept two integer
429 parameters, the first representing the number of bytes that have
430 been successfully transmitted to GS and the second representing the
431 total number of bytes that need to be transmitted.
432
433 :type num_cb: int
434 :param num_cb: (optional) If a callback is specified with the cb
435 parameter, this parameter determines the granularity of the callback
436 by defining the maximum number of times the callback will be called
437 during the file transfer.
438
439 :type policy: :class:`boto.gs.acl.CannedACLStrings`
440 :param policy: (optional) A canned ACL policy that will be applied to
441 the new key in GS.
442
443 :type md5: tuple
444 :param md5: (optional) A tuple containing the hexdigest version of the
445 MD5 checksum of the file as the first element and the
446 Base64-encoded version of the plain checksum as the second element.
447 This is the same format returned by the compute_md5 method.
448
449 If you need to compute the MD5 for any reason prior to upload, it's
450 silly to have to do it twice so this param, if present, will be
451 used as the MD5 values of the file. Otherwise, the checksum will be
452 computed.
453
454 :type res_upload_handler: :py:class:`boto.gs.resumable_upload_handler.ResumableUploadHandler`
455 :param res_upload_handler: (optional) If provided, this handler will
456 perform the upload.
457
458 :type size: int
459 :param size: (optional) The Maximum number of bytes to read from the
460 file pointer (fp). This is useful when uploading a file in multiple
461 parts where you are splitting the file up into different ranges to
462 be uploaded. If not specified, the default behaviour is to read all
463 bytes from the file pointer. Less bytes may be available.
464
465 Notes:
466
467 1. The "size" parameter currently cannot be used when a
468 resumable upload handler is given but is still useful for
469 uploading part of a file as implemented by the parent class.
470 2. At present Google Cloud Storage does not support multipart
471 uploads.
472
473 :type rewind: bool
474 :param rewind: (optional) If True, the file pointer (fp) will be
475 rewound to the start before any bytes are read from it. The default
476 behaviour is False which reads from the current position of the
477 file pointer (fp).
478
479 :type if_generation: int
480 :param if_generation: (optional) If set to a generation number, the
481 object will only be written to if its current generation number is
482 this value. If set to the value 0, the object will only be written
483 if it doesn't already exist.
484
485 :rtype: int
486 :return: The number of bytes written to the key.
487
488 TODO: At some point we should refactor the Bucket and Key classes,
489 to move functionality common to all providers into a parent class,
490 and provider-specific functionality into subclasses (rather than
491 just overriding/sharing code the way it currently works).
492 """
493 provider = self.bucket.connection.provider
494 if res_upload_handler and size:
495 # could use size instead of file_length if provided but...
496 raise BotoClientError(
497 '"size" param not supported for resumable uploads.')
498 headers = headers or {}
499 if policy:
500 headers[provider.acl_header] = policy
501
502 if rewind:
503 # caller requests reading from beginning of fp.
504 fp.seek(0, os.SEEK_SET)
505 else:
506 # The following seek/tell/seek logic is intended
507 # to detect applications using the older interface to
508 # set_contents_from_file(), which automatically rewound the
509 # file each time the Key was reused. This changed with commit
510 # 14ee2d03f4665fe20d19a85286f78d39d924237e, to support uploads
511 # split into multiple parts and uploaded in parallel, and at
512 # the time of that commit this check was added because otherwise
513 # older programs would get a success status and upload an empty
514 # object. Unfortuantely, it's very inefficient for fp's implemented
515 # by KeyFile (used, for example, by gsutil when copying between
516 # providers). So, we skip the check for the KeyFile case.
517 # TODO: At some point consider removing this seek/tell/seek
518 # logic, after enough time has passed that it's unlikely any
519 # programs remain that assume the older auto-rewind interface.
520 if not isinstance(fp, KeyFile):
521 spos = fp.tell()
522 fp.seek(0, os.SEEK_END)
523 if fp.tell() == spos:
524 fp.seek(0, os.SEEK_SET)
525 if fp.tell() != spos:
526 # Raise an exception as this is likely a programming
527 # error whereby there is data before the fp but nothing
528 # after it.
529 fp.seek(spos)
530 raise AttributeError('fp is at EOF. Use rewind option '
531 'or seek() to data start.')
532 # seek back to the correct position.
533 fp.seek(spos)
534
535 if hasattr(fp, 'name'):
536 self.path = fp.name
537 if self.bucket is not None:
538 if isinstance(fp, KeyFile):
539 # Avoid EOF seek for KeyFile case as it's very inefficient.
540 key = fp.getkey()
541 size = key.size - fp.tell()
542 self.size = size
543 # At present both GCS and S3 use MD5 for the etag for
544 # non-multipart-uploaded objects. If the etag is 32 hex
545 # chars use it as an MD5, to avoid having to read the file
546 # twice while transferring.
547 if (re.match('^"[a-fA-F0-9]{32}"$', key.etag)):
548 etag = key.etag.strip('"')
549 md5 = (etag, base64.b64encode(binascii.unhexlify(etag)))
550 if size:
551 self.size = size
552 else:
553 # If md5 is provided, still need to size so
554 # calculate based on bytes to end of content
555 spos = fp.tell()
556 fp.seek(0, os.SEEK_END)
557 self.size = fp.tell() - spos
558 fp.seek(spos)
559 size = self.size
560
561 if md5 is None:
562 md5 = self.compute_md5(fp, size)
563 self.md5 = md5[0]
564 self.base64md5 = md5[1]
565
566 if self.name is None:
567 self.name = self.md5
568
569 if not replace:
570 if self.bucket.lookup(self.name):
571 return
572
573 if if_generation is not None:
574 headers['x-goog-if-generation-match'] = str(if_generation)
575
576 if res_upload_handler:
577 res_upload_handler.send_file(self, fp, headers, cb, num_cb)
578 else:
579 # Not a resumable transfer so use basic send_file mechanism.
580 self.send_file(fp, headers, cb, num_cb, size=size)
581
582 def set_contents_from_filename(self, filename, headers=None, replace=True,
583 cb=None, num_cb=10, policy=None, md5=None,
584 reduced_redundancy=None,
585 res_upload_handler=None,
586 if_generation=None):
587 """
588 Store an object in GS using the name of the Key object as the
589 key in GS and the contents of the file named by 'filename'.
590 See set_contents_from_file method for details about the
591 parameters.
592
593 :type filename: string
594 :param filename: The name of the file that you want to put onto GS.
595
596 :type headers: dict
597 :param headers: (optional) Additional headers to pass along with the
598 request to GS.
599
600 :type replace: bool
601 :param replace: (optional) If True, replaces the contents of the file
602 if it already exists.
603
604 :type cb: function
605 :param cb: (optional) Callback function that will be called to report
606 progress on the upload. The callback should accept two integer
607 parameters, the first representing the number of bytes that have
608 been successfully transmitted to GS and the second representing the
609 total number of bytes that need to be transmitted.
610
611 :type num_cb: int
612 :param num_cb: (optional) If a callback is specified with the cb
613 parameter this parameter determines the granularity of the callback
614 by defining the maximum number of times the callback will be called
615 during the file transfer.
616
617 :type policy: :py:attribute:`boto.gs.acl.CannedACLStrings`
618 :param policy: (optional) A canned ACL policy that will be applied to
619 the new key in GS.
620
621 :type md5: tuple
622 :param md5: (optional) A tuple containing the hexdigest version of the
623 MD5 checksum of the file as the first element and the
624 Base64-encoded version of the plain checksum as the second element.
625 This is the same format returned by the compute_md5 method.
626
627 If you need to compute the MD5 for any reason prior to upload, it's
628 silly to have to do it twice so this param, if present, will be
629 used as the MD5 values of the file. Otherwise, the checksum will be
630 computed.
631
632 :type res_upload_handler: :py:class:`boto.gs.resumable_upload_handler.ResumableUploadHandler`
633 :param res_upload_handler: (optional) If provided, this handler will
634 perform the upload.
635
636 :type if_generation: int
637 :param if_generation: (optional) If set to a generation number, the
638 object will only be written to if its current generation number is
639 this value. If set to the value 0, the object will only be written
640 if it doesn't already exist.
641 """
642 # Clear out any previously computed hashes, since we are setting the
643 # content.
644 self.local_hashes = {}
645
646 with open(filename, 'rb') as fp:
647 self.set_contents_from_file(fp, headers, replace, cb, num_cb,
648 policy, md5, res_upload_handler,
649 if_generation=if_generation)
650
651 def set_contents_from_string(self, s, headers=None, replace=True,
652 cb=None, num_cb=10, policy=None, md5=None,
653 if_generation=None):
654 """
655 Store an object in GCS using the name of the Key object as the
656 key in GCS and the string 's' as the contents.
657 See set_contents_from_file method for details about the
658 parameters.
659
660 :type headers: dict
661 :param headers: Additional headers to pass along with the
662 request to AWS.
663
664 :type replace: bool
665 :param replace: If True, replaces the contents of the file if
666 it already exists.
667
668 :type cb: function
669 :param cb: a callback function that will be called to report
670 progress on the upload. The callback should accept
671 two integer parameters, the first representing the
672 number of bytes that have been successfully
673 transmitted to GCS and the second representing the
674 size of the to be transmitted object.
675
676 :type cb: int
677 :param num_cb: (optional) If a callback is specified with
678 the cb parameter this parameter determines the
679 granularity of the callback by defining
680 the maximum number of times the callback will
681 be called during the file transfer.
682
683 :type policy: :class:`boto.gs.acl.CannedACLStrings`
684 :param policy: A canned ACL policy that will be applied to the
685 new key in GCS.
686
687 :type md5: A tuple containing the hexdigest version of the MD5
688 checksum of the file as the first element and the
689 Base64-encoded version of the plain checksum as the
690 second element. This is the same format returned by
691 the compute_md5 method.
692 :param md5: If you need to compute the MD5 for any reason prior
693 to upload, it's silly to have to do it twice so this
694 param, if present, will be used as the MD5 values
695 of the file. Otherwise, the checksum will be computed.
696
697 :type if_generation: int
698 :param if_generation: (optional) If set to a generation number, the
699 object will only be written to if its current generation number is
700 this value. If set to the value 0, the object will only be written
701 if it doesn't already exist.
702 """
703
704 # Clear out any previously computed md5 hashes, since we are setting the content.
705 self.md5 = None
706 self.base64md5 = None
707
708 fp = StringIO(get_utf8_value(s))
709 r = self.set_contents_from_file(fp, headers, replace, cb, num_cb,
710 policy, md5,
711 if_generation=if_generation)
712 fp.close()
713 return r
714
715 def set_contents_from_stream(self, *args, **kwargs):
716 """
717 Store an object using the name of the Key object as the key in
718 cloud and the contents of the data stream pointed to by 'fp' as
719 the contents.
720
721 The stream object is not seekable and total size is not known.
722 This has the implication that we can't specify the
723 Content-Size and Content-MD5 in the header. So for huge
724 uploads, the delay in calculating MD5 is avoided but with a
725 penalty of inability to verify the integrity of the uploaded
726 data.
727
728 :type fp: file
729 :param fp: the file whose contents are to be uploaded
730
731 :type headers: dict
732 :param headers: additional HTTP headers to be sent with the
733 PUT request.
734
735 :type replace: bool
736 :param replace: If this parameter is False, the method will first check
737 to see if an object exists in the bucket with the same key. If it
738 does, it won't overwrite it. The default value is True which will
739 overwrite the object.
740
741 :type cb: function
742 :param cb: a callback function that will be called to report
743 progress on the upload. The callback should accept two integer
744 parameters, the first representing the number of bytes that have
745 been successfully transmitted to GS and the second representing the
746 total number of bytes that need to be transmitted.
747
748 :type num_cb: int
749 :param num_cb: (optional) If a callback is specified with the
750 cb parameter, this parameter determines the granularity of
751 the callback by defining the maximum number of times the
752 callback will be called during the file transfer.
753
754 :type policy: :class:`boto.gs.acl.CannedACLStrings`
755 :param policy: A canned ACL policy that will be applied to the new key
756 in GS.
757
758 :type size: int
759 :param size: (optional) The Maximum number of bytes to read from
760 the file pointer (fp). This is useful when uploading a
761 file in multiple parts where you are splitting the file up
762 into different ranges to be uploaded. If not specified,
763 the default behaviour is to read all bytes from the file
764 pointer. Less bytes may be available.
765
766 :type if_generation: int
767 :param if_generation: (optional) If set to a generation number, the
768 object will only be written to if its current generation number is
769 this value. If set to the value 0, the object will only be written
770 if it doesn't already exist.
771 """
772 if_generation = kwargs.pop('if_generation', None)
773 if if_generation is not None:
774 headers = kwargs.get('headers', {})
775 headers['x-goog-if-generation-match'] = str(if_generation)
776 kwargs['headers'] = headers
777 super(Key, self).set_contents_from_stream(*args, **kwargs)
778
779 def set_acl(self, acl_or_str, headers=None, generation=None,
780 if_generation=None, if_metageneration=None):
781 """Sets the ACL for this object.
782
783 :type acl_or_str: string or :class:`boto.gs.acl.ACL`
784 :param acl_or_str: A canned ACL string (see
785 :data:`~.gs.acl.CannedACLStrings`) or an ACL object.
786
787 :type headers: dict
788 :param headers: Additional headers to set during the request.
789
790 :type generation: int
791 :param generation: If specified, sets the ACL for a specific generation
792 of a versioned object. If not specified, the current version is
793 modified.
794
795 :type if_generation: int
796 :param if_generation: (optional) If set to a generation number, the acl
797 will only be updated if its current generation number is this value.
798
799 :type if_metageneration: int
800 :param if_metageneration: (optional) If set to a metageneration number,
801 the acl will only be updated if its current metageneration number is
802 this value.
803 """
804 if self.bucket is not None:
805 self.bucket.set_acl(acl_or_str, self.name, headers=headers,
806 generation=generation,
807 if_generation=if_generation,
808 if_metageneration=if_metageneration)
809
810 def get_acl(self, headers=None, generation=None):
811 """Returns the ACL of this object.
812
813 :param dict headers: Additional headers to set during the request.
814
815 :param int generation: If specified, gets the ACL for a specific
816 generation of a versioned object. If not specified, the current
817 version is returned.
818
819 :rtype: :class:`.gs.acl.ACL`
820 """
821 if self.bucket is not None:
822 return self.bucket.get_acl(self.name, headers=headers,
823 generation=generation)
824
825 def get_xml_acl(self, headers=None, generation=None):
826 """Returns the ACL string of this object.
827
828 :param dict headers: Additional headers to set during the request.
829
830 :param int generation: If specified, gets the ACL for a specific
831 generation of a versioned object. If not specified, the current
832 version is returned.
833
834 :rtype: str
835 """
836 if self.bucket is not None:
837 return self.bucket.get_xml_acl(self.name, headers=headers,
838 generation=generation)
839
840 def set_xml_acl(self, acl_str, headers=None, generation=None,
841 if_generation=None, if_metageneration=None):
842 """Sets this objects's ACL to an XML string.
843
844 :type acl_str: string
845 :param acl_str: A string containing the ACL XML.
846
847 :type headers: dict
848 :param headers: Additional headers to set during the request.
849
850 :type generation: int
851 :param generation: If specified, sets the ACL for a specific generation
852 of a versioned object. If not specified, the current version is
853 modified.
854
855 :type if_generation: int
856 :param if_generation: (optional) If set to a generation number, the acl
857 will only be updated if its current generation number is this value.
858
859 :type if_metageneration: int
860 :param if_metageneration: (optional) If set to a metageneration number,
861 the acl will only be updated if its current metageneration number is
862 this value.
863 """
864 if self.bucket is not None:
865 return self.bucket.set_xml_acl(acl_str, self.name, headers=headers,
866 generation=generation,
867 if_generation=if_generation,
868 if_metageneration=if_metageneration)
869
870 def set_canned_acl(self, acl_str, headers=None, generation=None,
871 if_generation=None, if_metageneration=None):
872 """Sets this objects's ACL using a predefined (canned) value.
873
874 :type acl_str: string
875 :param acl_str: A canned ACL string. See
876 :data:`~.gs.acl.CannedACLStrings`.
877
878 :type headers: dict
879 :param headers: Additional headers to set during the request.
880
881 :type generation: int
882 :param generation: If specified, sets the ACL for a specific generation
883 of a versioned object. If not specified, the current version is
884 modified.
885
886 :type if_generation: int
887 :param if_generation: (optional) If set to a generation number, the acl
888 will only be updated if its current generation number is this value.
889
890 :type if_metageneration: int
891 :param if_metageneration: (optional) If set to a metageneration number,
892 the acl will only be updated if its current metageneration number is
893 this value.
894 """
895 if self.bucket is not None:
896 return self.bucket.set_canned_acl(
897 acl_str,
898 self.name,
899 headers=headers,
900 generation=generation,
901 if_generation=if_generation,
902 if_metageneration=if_metageneration
903 )
904
905 def compose(self, components, content_type=None, headers=None):
906 """Create a new object from a sequence of existing objects.
907
908 The content of the object representing this Key will be the
909 concatenation of the given object sequence. For more detail, visit
910
911 https://developers.google.com/storage/docs/composite-objects
912
913 :type components list of Keys
914 :param components List of gs.Keys representing the component objects
915
916 :type content_type (optional) string
917 :param content_type Content type for the new composite object.
918 """
919 compose_req = []
920 for key in components:
921 if key.bucket.name != self.bucket.name:
922 raise BotoClientError(
923 'GCS does not support inter-bucket composing')
924
925 generation_tag = ''
926 if key.generation:
927 generation_tag = ('<Generation>%s</Generation>'
928 % str(key.generation))
929 compose_req.append('<Component><Name>%s</Name>%s</Component>' %
930 (key.name, generation_tag))
931 compose_req_xml = ('<ComposeRequest>%s</ComposeRequest>' %
932 ''.join(compose_req))
933 headers = headers or {}
934 if content_type:
935 headers['Content-Type'] = content_type
936 resp = self.bucket.connection.make_request(
937 'PUT', get_utf8_value(self.bucket.name), get_utf8_value(self.name),
938 headers=headers, query_args='compose',
939 data=get_utf8_value(compose_req_xml))
940 if resp.status < 200 or resp.status > 299:
941 raise self.bucket.connection.provider.storage_response_error(
942 resp.status, resp.reason, resp.read())
943
944 # Return the generation so that the result URI can be built with this
945 # for automatic parallel uploads.
946 return resp.getheader('x-goog-generation')