diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 91424948fc..e704aa19c0 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,25 @@ Changelog ========= +0.0.6 - 2014-12-18 +------------------ + +* feature:Amazon SQS: Add ``purge`` action to queue resources +* feature:Waiters: Add documentation for client and resource waiters + (`issue 44 `__) +* feature:Waiters: Add support for resource waiters + (`issue 43 `__) +* bugfix:Installation: Remove dependency on the unused ``six`` module + (`issue 42 `__) +* feature:Botocore: Update to Botocore 0.80.0 + + * Update Amazon Simple Workflow Service (SWF) to the latest version + * Update AWS Storage Gateway to the latest version + * Update AWS Elastic MapReduce (EMR) to the latest version + * Update AWS Elastic Transcoder to the latest version + * Enable use of ``page_size`` for clients + (`botocore issue 408 `__) + 0.0.5 - 2014-12-09 ------------------ diff --git a/boto3/__init__.py b/boto3/__init__.py index bbb29d97cb..50eb9632c1 100644 --- a/boto3/__init__.py +++ b/boto3/__init__.py @@ -17,7 +17,7 @@ __author__ = 'Amazon Web Services' -__version__ = '0.0.5' +__version__ = '0.0.6' # The default Boto3 session; autoloaded when needed. diff --git a/boto3/compat.py b/boto3/compat.py deleted file mode 100644 index a59ee37a88..0000000000 --- a/boto3/compat.py +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). You -# may not use this file except in compliance with the License. A copy of -# the License is located at -# -# http://aws.amazon.com/apache2.0/ -# -# or in the "license" file accompanying this file. This file is -# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF -# ANY KIND, either express or implied. See the License for the specific -# language governing permissions and limitations under the License. - -import six diff --git a/boto3/data/resources/sqs-2012-11-05.resources.json b/boto3/data/resources/sqs-2012-11-05.resources.json index 6d889e89ec..9b061fbf3f 100644 --- a/boto3/data/resources/sqs-2012-11-05.resources.json +++ b/boto3/data/resources/sqs-2012-11-05.resources.json @@ -21,15 +21,6 @@ } }, "hasMany": { - "DeadLetterSourceQueues": { - "request": { "operation": "ListDeadLetterSourceQueues" }, - "resource": { - "type": "Queue", - "identifiers": [ - { "target": "Url", "sourceType": "responsePath", "source": "QueueUrls[]" } - ] - } - }, "Queues": { "request": { "operation": "ListQueues" }, "resource": { @@ -45,7 +36,7 @@ "Message": { "identifiers": [ { "name": "QueueUrl" }, - { "name": "ReceiptHandle" } + { "name": "ReceiptHandle", "memberName": "ReceiptHandle" } ], "shape": "Message", "actions": { @@ -129,6 +120,14 @@ ] } }, + "Purge": { + "request": { + "operation": "PurgeQueue", + "params": [ + { "target": "QueueUrl", "sourceType": "identifier", "source": "Url" } + ] + } + }, "ReceiveMessages": { "request": { "operation": "ReceiveMessage", @@ -181,6 +180,22 @@ "subResources": { "resources": [ "Message" ], "identifiers": { "Url": "QueueUrl" } + }, + "hasMany" : { + "DeadLetterSourceQueues": { + "request": { + "operation": "ListDeadLetterSourceQueues", + "params": [ + { "target": "QueueUrl", "sourceType": "identifier", "source": "Url" } + ] + }, + "resource": { + "type": "Queue", + "identifiers": [ + { "target": "Url", "sourceType": "responsePath", "source": "QueueUrls[]" } + ] + } + } } } } diff --git a/boto3/docs.py b/boto3/docs.py index 55c7004460..2c6f6032cb 100644 --- a/boto3/docs.py +++ b/boto3/docs.py @@ -146,9 +146,11 @@ def docs_for(service_name): docs = '{0}\n{1}\n\n'.format(official_name, '=' * len(official_name)) - docs += '.. contents:: Table of Contents\n\n' + docs += '.. contents:: Table of Contents\n :depth: 2\n\n' docs += document_client(service_name, official_name, service_model) + docs += document_client_waiter(session, official_name, service_name, + service_model) filename = (os.path.dirname(__file__) + '/data/resources/' '{0}-{1}.resources.json').format(service_name, @@ -167,7 +169,7 @@ def docs_for(service_name): } docs += document_resource(service_name, official_name, model, - service_model) + service_model, session) # First, collect all the models... for name, model in sorted(data['resources'].items(), @@ -189,7 +191,7 @@ def docs_for(service_name): model = item['model'] if item['type'] == 'resource': docs += document_resource(service_name, official_name, - model, service_model) + model, service_model, session) elif item['type'] == 'collection': docs += document_collection( service_name, official_name, model, @@ -223,7 +225,7 @@ def document_client(service_name, official_name, service_model): wdoc += ' .. py:method:: get_waiter(name)\n\n' wdoc += ' Get a waiter by name. Available waiters:\n\n' for waiter in client.waiter_names: - wdoc += ' * {0}\n'.format(waiter) + wdoc += ' * `{0}`_\n'.format(waiter) wdoc += '\n' waiter_included = False @@ -240,8 +242,40 @@ def document_client(service_name, official_name, service_model): return docs +def document_client_waiter(session, official_name, service_name, + service_model): + client = boto3.client(service_name, aws_access_key_id='dummy', + aws_secret_access_key='dummy', + region_name='us-east-1') + waiter_spec_doc = '' + if client.waiter_names: + waiter_spec_doc = 'Waiter\n------\n\n' + service_waiter_model = session.get_waiter_model(service_name) + for waiter in service_waiter_model.waiter_names: + snake_cased = xform_name(waiter) + waiter_spec_doc += '{0}\n{1}\n\n'.format(snake_cased, + '~' * len(snake_cased)) + waiter_model = service_waiter_model.get_waiter(waiter) + operation_model = service_model.operation_model( + waiter_model.operation) + description = ( + ' This polls :py:meth:`{0}.Client.{1}` every {2} ' + 'seconds until a successful state is reached. An error is ' + 'returned after {3} failed checks.'.format( + service_name, xform_name(waiter_model.operation), + waiter_model.delay, waiter_model.max_attempts) + ) + waiter_spec_doc += document_operation( + operation_model=operation_model, service_name=service_name, + operation_name='wait', rtype=None, description=description, + example_instance='client.get_waiter(\'{0}\')'.format( + snake_cased)) + waiter_spec_doc += '\n' + + return waiter_spec_doc + def document_resource(service_name, official_name, resource_model, - service_model): + service_model, session): """ Generate reference documentation from a resource model. """ @@ -369,6 +403,16 @@ def document_resource(service_name, official_name, resource_model, collection.resource.type, service_name, xform_name(collection.request.operation)) + if resource_model.waiters: + docs += (' .. rst-class:: admonition-title\n\n Waiters\n\n' + ' Waiters provide an interface to wait for a resource' + ' to reach a specific state.\n\n') + service_waiter_model = session.get_waiter_model(service_name) + for waiter in sorted(resource_model.waiters, + key=lambda i: i.resource_waiter_name): + docs += document_waiter(waiter, service_name, resource_model, + service_model, service_waiter_model) + return docs def document_collection(service_name, official_name, collection_model, @@ -407,6 +451,42 @@ def document_collection(service_name, official_name, collection_model, return docs +def document_waiter(waiter, service_name, resource_model, service_model, + service_waiter_model): + """ + Document a resource waiter, including the low-level client waiter + and parameters. + """ + try: + waiter_model = service_waiter_model.get_waiter(waiter.waiter_name) + except: + print('Cannot get waiter ' + waiter.waiter_name) + return '' + + try: + operation_model = service_model.operation_model(waiter_model.operation) + except: + print('Cannot get operation ' + action.request.operation + + ' for waiter ' + waiter.waiter_name) + return '' + description = (' Waits until this {0} is {1}.\n' + ' This method calls ``wait()`` on' + ' :py:meth:`{2}.Client.get_waiter` using `{3}`_ .').format( + resource_model.name, + xform_name(waiter.name).replace('_', ' '), + service_name, + xform_name(waiter.waiter_name)) + + # Here we split because we only care about top-level parameter names + ignore_params = [p.target.split('.')[0].strip('[]') for p in waiter.params] + + return document_operation( + operation_model=operation_model, service_name=service_name, + operation_name=xform_name(waiter.resource_waiter_name), + description=description, + example_instance = xform_name(resource_model.name), + ignore_params=ignore_params, rtype=None) + def document_action(action, service_name, resource_model, service_model, action_type='action'): """ @@ -421,7 +501,8 @@ def document_action(action, service_name, resource_model, service_model, return '' # Here we split because we only care about top-level parameter names - ignore_params = [p.target.split('.')[0] for p in action.request.params] + ignore_params = [p.target.split('.')[0].strip('[]') + for p in action.request.params] rtype = 'dict' if action_type == 'action': @@ -442,7 +523,8 @@ def document_action(action, service_name, resource_model, service_model, return document_operation( operation_model, service_name, operation_name=xform_name(action.name), description=description, ignore_params=ignore_params, rtype=rtype, - example_instance=service_name, example_response=example_response) + example_instance=xform_name(resource_model.name), + example_response=example_response) def document_operation(operation_model, service_name, operation_name=None, description=None, ignore_params=None, rtype='dict', @@ -497,8 +579,10 @@ def document_operation(operation_model, service_name, operation_name=None, default = py_default(value.type_name) dummy_params.append('{0}={1}'.format( key, default)) - docs += ' Example::\n\n {0} = {1}.{2}({3})\n\n'.format( - example_response, example_instance, operation_name, + docs += ' Example::\n\n ' + if example_response is not None: + docs += '{0} = '.format(example_response) + docs += '{0}.{1}({2})\n\n'.format(example_instance, operation_name, ', '.join(dummy_params)) for key, value in params.items(): @@ -510,7 +594,7 @@ def document_operation(operation_model, service_name, operation_name=None, docs += (' :param {0} {1}: *{2}* - {3}\n'.format( param_type, key, required, html_to_rst(value.documentation, indent=9))) - - docs += '\n\n :rtype: {0}\n\n'.format(rtype) + if rtype is not None: + docs += '\n\n :rtype: {0}\n\n'.format(rtype) return docs diff --git a/boto3/resources/action.py b/boto3/resources/action.py index b491f8322b..f4ac96af3d 100644 --- a/boto3/resources/action.py +++ b/boto3/resources/action.py @@ -149,3 +149,48 @@ def __call__(self, parent, *args, **kwargs): self._response_handler(parent, params, response)) return responses + + +class WaiterAction(object): + """ + A class representing a callable waiter action on a resource, for example + ``s3.Bucket('foo').wait_until_bucket_exists()``. + The waiter action may construct parameters from existing resource + identifiers. + + :type waiter_model: :py:class`~boto3.resources.model.Waiter` + :param waiter_model: The action waiter. + :type waiter_resource_name: string + :param waiter_resource_name: The name of the waiter action for the + resource. It usually begins with a + ``wait_until_`` + """ + def __init__(self, waiter_model, waiter_resource_name): + self._waiter_model = waiter_model + self._waiter_resource_name = waiter_resource_name + + def __call__(self, parent, *args, **kwargs): + """ + Perform the wait operation after building operation + parameters. + + :type parent: :py:class:`~boto3.resources.base.ServiceResource` + :param parent: The resource instance to which this action is attached. + """ + client_waiter_name = xform_name(self._waiter_model.waiter_name) + + # First, build predefined params and then update with the + # user-supplied kwargs, which allows overriding the pre-built + # params if needed. + params = create_request_parameters(parent, self._waiter_model) + params.update(kwargs) + + logger.info('Calling %s:%s with %r', + parent.meta['service_name'], + self._waiter_resource_name, params) + + client = parent.meta['client'] + waiter = client.get_waiter(client_waiter_name) + response = waiter.wait(**params) + + logger.debug('Response: %r', response) diff --git a/boto3/resources/factory.py b/boto3/resources/factory.py index da54a2b3a1..7b8926b1cb 100644 --- a/boto3/resources/factory.py +++ b/boto3/resources/factory.py @@ -17,6 +17,7 @@ from botocore import xform_name from .action import ServiceAction +from .action import WaiterAction from .base import ServiceResource from .collection import CollectionFactory from .model import ResourceModel @@ -87,6 +88,7 @@ def load_from_definition(self, service_name, resource_name, model, service_model) self._load_references(attrs, service_name, resource_name, resource_model, resource_defs, service_model) + self._load_waiters(attrs, resource_model) # Create the name based on the requested service and resource cls_name = resource_name @@ -214,6 +216,18 @@ def _load_references(self, attrs, service_name, resource_name, reference.resource.type, snake_cased, reference, service_name, resource_name, model, resource_defs, service_model) + def _load_waiters(self, attrs, model): + """ + Load resource waiters from the model. Each waiter allows you to + wait until a resource reaches a specific state by polling the state + of the resource. + """ + for waiter in model.waiters: + snake_cased = xform_name(waiter.resource_waiter_name) + snake_cased = self._check_allowed_name( + attrs, snake_cased, 'waiter', model.name) + attrs[snake_cased] = self._create_waiter(waiter, snake_cased) + def _check_allowed_name(self, attrs, name, category, resource_name): """ Determine if a given name is allowed on the instance, and if not, @@ -271,6 +285,19 @@ def property_loader(self): property_loader.__doc__ = 'TODO' return property(property_loader) + def _create_waiter(factory_self, waiter_model, snake_cased): + """ + Creates a new wait method for each resource where both a waiter and + resource model is defined. + """ + waiter = WaiterAction(waiter_model, waiter_resource_name=snake_cased) + def do_waiter(self, *args, **kwargs): + waiter(self, *args, **kwargs) + + do_waiter.__name__ = str(snake_cased) + do_waiter.__doc__ = 'TODO' + return do_waiter + def _create_collection(factory_self, service_name, resource_name, snake_cased, collection_model, resource_defs, service_model): diff --git a/boto3/resources/model.py b/boto3/resources/model.py index 938e63d6ea..d1b3019a71 100644 --- a/boto3/resources/model.py +++ b/boto3/resources/model.py @@ -150,6 +150,9 @@ def __init__(self, name, definition): #: (``string``) The name of this waiter self.name = name + #: (``string``) The name of the waiter in the resource + self.resource_waiter_name = 'WaitUntil' + name + #: (``string``) The name of the underlying event waiter self.waiter_name = definition.get('waiterName') diff --git a/docs/source/guide/clients.rst b/docs/source/guide/clients.rst index 107bd8b19a..4a8ed2d671 100644 --- a/docs/source/guide/clients.rst +++ b/docs/source/guide/clients.rst @@ -66,3 +66,44 @@ The ``response`` in the example above looks something like this: "http://url3" ] } + +Waiters +------- +Waiters use a client's service operations to poll the status of an AWS resource +and suspend execution until the AWS resource reaches the state that the +waiter is polling for or a failure occurs while polling. +Using clients, you can learn the name of each waiter that a client has access +to:: + + import boto3 + + s3 = boto3.client('s3') + sqs = boto3.client('sqs') + + # List all of the possible waiters for both clients + print("s3 waiters:") + s3.waiter_names + + print("sqs waiters:") + sqs.waiter_names + +Note if a client does not have any waiters, it will return an empty list when +accessing its ``waiter_names`` attribute:: + + s3 waiters: + [u'bucket_exists', u'bucket_not_exists', u'object_exists', u'object_not_exists'] + sqs waiters: + [] + +Using a client's ``get_waiter()`` method, you can obtain a specific waiter +from its list of possible waiters:: + + # Retrieve waiter instance that will wait till a specified + # S3 bucket exists + s3_bucket_exists_waiter = s3.get_waiter('bucket_exists') + +Then to actually start waiting, you must call the waiter's ``wait()`` method +with the method's appropriate parameters passed in:: + + # Begin waiting for the S3 bucket, mybucket, to exist + s3_bucket_exists_waiter.wait(Bucket='mybucket') diff --git a/docs/source/guide/resources.rst b/docs/source/guide/resources.rst index 9f2183fe90..ecbd2b0fc4 100644 --- a/docs/source/guide/resources.rst +++ b/docs/source/guide/resources.rst @@ -170,3 +170,18 @@ can be considered one-to-many. Examples of sub-resources:: Because an SQS message cannot exist without a queue, and an S3 object cannot exist without a bucket, these are parent to child relationships. + +Waiters +------- +A waiter is similiar to an action. A waiter will poll the status of a +resource and suspend execution until the resource reaches the state that is +being polling for or a failure occurs while polling. +Waiters automatically set the resource +identifiers as parameters, but allow you to pass additional parameters via +keyword arguments. Examples of waiters include:: + + # S3: Wait for a bucket to exist. + bucket.wait_until_exists() + + # EC2: Wait for an instance to reach the running state. + instance.wait_until_running() diff --git a/requirements.txt b/requirements.txt index 7d82c342ae..b70bfa29e9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,5 @@ -e git://github.com/boto/botocore.git@develop#egg=botocore -e git://github.com/boto/bcdoc.git@develop#egg=bcdoc -e git://github.com/boto/jmespath.git@develop#egg=jmespath -six==1.7.3 nose==1.3.3 mock==1.0.1 diff --git a/setup.py b/setup.py index 2b226b0e18..a96e9498b4 100644 --- a/setup.py +++ b/setup.py @@ -26,10 +26,9 @@ def get_version(): ] requires = [ - 'botocore==0.78.0', + 'botocore==0.80.0', 'bcdoc==0.12.2', 'jmespath==0.5.0', - 'six==1.7.3', ] setup( diff --git a/tests/integration/test_s3.py b/tests/integration/test_s3.py index ca76d5a7af..197de819d1 100644 --- a/tests/integration/test_s3.py +++ b/tests/integration/test_s3.py @@ -18,22 +18,30 @@ class TestS3Resource(unittest.TestCase): def setUp(self): - self.session = boto3.session.Session(region_name='us-west-2') + self.region = 'us-west-2' + self.session = boto3.session.Session(region_name=self.region) self.s3 = self.session.resource('s3') self.bucket_name = unique_id('boto3-test') + def create_bucket_resource(self, bucket_name, region=None): + if region is None: + region = self.region + kwargs = {'Bucket': bucket_name} + if region != 'us-east-1': + kwargs['CreateBucketConfiguration'] = { + 'LocationConstraint': region + } + bucket = self.s3.create_bucket(**kwargs) + self.addCleanup(bucket.delete) + return bucket + def test_s3(self): client = self.s3.meta['client'] # Create a bucket (resource action with a resource response) - bucket = self.s3.create_bucket( - Bucket=self.bucket_name, - CreateBucketConfiguration={ - 'LocationConstraint': 'us-west-2' - }) + bucket = self.create_bucket_resource(self.bucket_name) waiter = client.get_waiter('bucket_exists') waiter.wait(Bucket=self.bucket_name) - self.addCleanup(bucket.delete) # Create an object obj = bucket.Object('test.txt') @@ -52,3 +60,26 @@ def test_s3(self): # Perform a resource action with a low-level response self.assertEqual(b'hello, world', obj.get()['Body'].read()) + + def test_s3_resource_waiter(self): + # Create a bucket + bucket = self.create_bucket_resource(self.bucket_name) + # Wait till the bucket exists + bucket.wait_until_exists() + # Confirm the bucket exists by finding it in a list of all of our + # buckets + self.assertIn(self.bucket_name, + [b.name for b in self.s3.buckets.all()]) + + + # Create an object + obj = bucket.Object('test.txt') + obj.put( + Body='hello, world') + self.addCleanup(obj.delete) + + # Wait till the bucket exists + obj.wait_until_exists() + + # List objects and make sure ours is present + self.assertIn('test.txt', [o.key for o in bucket.objects.all()]) diff --git a/tests/unit/resources/test_action.py b/tests/unit/resources/test_action.py index f0e07e043e..8420c6525b 100644 --- a/tests/unit/resources/test_action.py +++ b/tests/unit/resources/test_action.py @@ -11,8 +11,8 @@ # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. -from boto3.resources.action import BatchAction, ServiceAction -from boto3.resources.model import Action +from boto3.resources.action import BatchAction, ServiceAction, WaiterAction +from boto3.resources.model import Action, Waiter from tests import BaseTestCase, mock @@ -123,6 +123,57 @@ def test_service_action_calls_resource_handler(self, handler_mock, params_mock): handler_mock.return_value.assert_called_with(resource, {}, 'response') +class TestWaiterActionCall(BaseTestCase): + def setUp(self): + super(TestWaiterActionCall, self).setUp() + self.waiter_resource_name = 'wait_until_exists' + self.waiter_def = { + "waiterName": "FrobExists", + "params": [ + {"target": "Frob", "sourceType": "identifier", + "source": "Name"}] + } + + @property + def waiter(self): + return Waiter('test', self.waiter_def) + + @mock.patch('boto3.resources.action.create_request_parameters', + return_value={}) + def test_service_waiter_creates_params(self, params_mock): + resource = mock.Mock() + resource.meta = { + 'service_name': 'test', + 'client': mock.Mock(), + } + + action = WaiterAction(self.waiter, self.waiter_resource_name) + + action(resource, foo=1) + + self.assertTrue(params_mock.called, + 'Parameters for operation not created') + + @mock.patch('boto3.resources.action.create_request_parameters', + return_value={'bar': 'baz'}) + def test_service_action_calls_operation(self, params_mock): + resource = mock.Mock() + resource.meta = { + 'service_name': 'test', + 'client': mock.Mock(), + } + get_waiter = resource.meta['client'].get_waiter + mock_waiter = mock.Mock() + get_waiter.return_value = mock_waiter + + action = WaiterAction(self.waiter, self.waiter_resource_name) + + action(resource, foo=1) + + get_waiter.assert_called_with('frob_exists') + mock_waiter.wait.assert_called_with(foo=1, bar='baz') + + class TestBatchActionCall(BaseTestCase): def setUp(self): super(TestBatchActionCall, self).setUp() diff --git a/tests/unit/resources/test_factory.py b/tests/unit/resources/test_factory.py index 2ed014049a..b9059dacaf 100644 --- a/tests/unit/resources/test_factory.py +++ b/tests/unit/resources/test_factory.py @@ -16,6 +16,7 @@ from boto3.resources.base import ServiceResource from boto3.resources.collection import CollectionManager from boto3.resources.factory import ResourceFactory +from boto3.resources.action import WaiterAction from tests import BaseTestCase, mock @@ -581,3 +582,49 @@ def test_resource_loads_collections(self, mock_model): 'Resource should expose queues collection') self.assertIsInstance(resource.queues, CollectionManager, 'Queues collection should be a collection manager') + + def test_resource_loads_waiters(self): + model = { + "waiters": { + "Exists": { + "waiterName": "BucketExists", + "params": [ + {"target": "Bucket", "sourceType": "identifier", + "source": "Name"}] + } + } + } + + defs = { + 'Bucket': {} + } + service_model = ServiceModel({}) + + resource = self.load('test', 'test', model, defs, service_model)() + + self.assertTrue(hasattr(resource, 'wait_until_exists'), + 'Resource should expose resource waiter: wait_until_exists') + + @mock.patch('boto3.resources.factory.WaiterAction') + def test_resource_waiter_calls_waiter_method(self, waiter_action_cls): + model = { + "waiters": { + "Exists": { + "waiterName": "BucketExists", + "params": [ + {"target": "Bucket", "sourceType": "identifier", + "source": "Name"}] + } + } + } + + defs = { + 'Bucket': {} + } + service_model = ServiceModel({}) + + waiter_action = waiter_action_cls.return_value + resource = self.load('test', 'test', model, defs, service_model)() + + resource.wait_until_exists('arg1', arg2=2) + waiter_action.assert_called_with(resource, 'arg1', arg2=2) diff --git a/tests/unit/resources/test_model.py b/tests/unit/resources/test_model.py index 94e21abbfe..d53e103013 100644 --- a/tests/unit/resources/test_model.py +++ b/tests/unit/resources/test_model.py @@ -255,4 +255,5 @@ def test_waiter(self): self.assertIsInstance(waiter, Waiter) self.assertEqual(waiter.name, 'Exists') self.assertEqual(waiter.waiter_name, 'ObjectExists') + self.assertEqual(waiter.resource_waiter_name, 'WaitUntilExists') self.assertEqual(waiter.params[0].target, 'Bucket')