Merge branch 'release-0.0.12'

* release-0.0.12:
  Bump version to 0.0.12
  Update changelog [ci skip]
  Remove unneeded backslash [ci skip]
  Add nested structure descriptions to params & return value.
  Elaborate more on the developer preview note in README
  Modify test to use public property
  Minor review feedback
  Load reference data if a resource path is defined

CHANGELOG.rst

@@ -1,6 +1,15 @@
 Changelog
 =========
 
+Unreleased
+----------
+
+* feature:Resources: Add the ability to load resource data from a
+  ``has`` relationship. This saves a call to ``load`` when available,
+  and otherwise fixes a problem where there was no way to get at
+  certain resource data.
+  (`issue 74 <https://github.com/boto/boto3/pull/72>`__,
+
 0.0.11 - 2015-03-24
 -------------------
 

README.rst

@@ -10,9 +10,11 @@ of services like Amazon S3 and Amazon EC2. You can find the latest, most
 up to date, documentation at `Read the Docs`_, including a list of
 services that are supported.
 
-**WARNING**: Boto 3 is in *developer preview* and **should not** be used in
-production yet! Please try it out and give feedback by opening issues or
-pull requests on this repository. Thanks!
+Boto 3 is in **developer preview**.  This means that until a 1.0 release
+occurs, some of the interfaces may change based on your feedback.
+Once a 1.0 release happens, we guarantee backwards compatibility
+for all future 1.x.x releases.  Try out boto3 and give us
+`feedback <https://github.com/boto/boto3/issues>`__ today!
 
 .. _boto: https://docs.pythonboto.org/
 .. _`Read the Docs`: https://boto3.readthedocs.org/en/latest/

boto3/__init__.py

@@ -17,7 +17,7 @@
 
 
 __author__ = 'Amazon Web Services'
-__version__ = '0.0.11'
+__version__ = '0.0.12'
 
 
 # The default Boto3 session; autoloaded when needed.

boto3/docs.py

@@ -52,6 +52,7 @@ def py_type_name(type_name):
     :rtype: string
     """
     return {
+        'blob': 'bytes',
         'character': 'string',
         'double': 'float',
         'long': 'integer',
@@ -88,16 +89,16 @@ def py_default(type_name):
     }.get(type_name, '...')
 
 
-def html_to_rst(html, indent=0, indentFirst=False):
+def html_to_rst(html, indent=0, indent_first=False):
     """
     Use bcdoc to convert html to rst.
 
     :type html: string
     :param html: Input HTML to be converted
     :type indent: int
     :param indent: Number of spaces to indent each line
-    :type indentFirst: boolean
-    :param indentFirst: Whether to indent the first line
+    :type indent_first: boolean
+    :param indent_first: Whether to indent the first line
     :rtype: string
     """
     doc = ReSTDocument()
@@ -113,7 +114,7 @@ def html_to_rst(html, indent=0, indentFirst=False):
     if indent:
         rst = '\n'.join([(' ' * indent) + line for line in rst.splitlines()])
 
-        if not indentFirst:
+        if not indent_first:
             rst = rst.strip()
 
     return rst
@@ -563,7 +564,7 @@ def document_operation(operation_model, service_name, operation_name=None,
     if description is None:
         description = html_to_rst(
             operation_model._operation_model.get('documentation', ''),
-            indent=6, indentFirst=True)
+            indent=6, indent_first=True)
 
     docs = '   .. py:method:: {0}({1})\n\n{2}\n\n'.format(
                 operation_name, param_desc, description)
@@ -591,11 +592,97 @@ def document_operation(operation_model, service_name, operation_name=None,
         if key in ignore_params:
             continue
         param_type = py_type_name(value.type_name)
+
+        # Convert the description from HTML to RST (to later be converted
+        # into HTML... don't ask). If the parameter is a nested structure
+        # then we also describe its members.
+        param_desc = html_to_rst(
+            value.documentation, indent=9, indent_first=True)
+        if param_type in ['list', 'dict']:
+            param_desc = ('\n         Structure description::\n\n' +
+                          '            ' + key + ' = ' +
+                          document_structure(
+                            key, value, indent=12, indent_first=False) +
+                          '\n' + param_desc)
         required = key in required_params and 'Required' or 'Optional'
         docs += ('      :param {0} {1}: *{2}* - {3}\n'.format(
-            param_type, key, required,
-            html_to_rst(value.documentation, indent=9)))
+            param_type, key, required, param_desc))
     if rtype is not None:
-        docs += '\n\n      :rtype: {0}\n\n'.format(rtype)
+        docs += '      :rtype: {0}\n\n'.format(rtype)
+
+    # Only document the return structure if it isn't a resource. Usually
+    # this means either a list or structure.
+    output_shape = operation_model.output_shape
+    if rtype in ['list', 'dict'] and output_shape is not None:
+        docs += ('      :return:\n         Structure description::\n\n' +
+                 document_structure(None, output_shape, indent=12) + '\n')
+
+    return docs
+
+
+def document_structure(name, shape, indent=0, indent_first=True,
+                       parent_type=None, eol='\n'):
+    """
+    Document a nested structure (list or dict) parameter or return value as
+    a snippet of Python code with dummy placeholders. For example:
+
+        {
+            'Param1': [
+                STRING,
+                ...
+            ],
+            'Param2': BOOLEAN,
+            'Param3': {
+                'Param4': FLOAT,
+                'Param5': INTEGER
+            }
+        }
+
+    """
+    docs = ''
+
+    # Add spaces if the first line is indented.
+    if indent_first:
+        docs += (' ' * indent)
+
+    if shape.type_name == 'structure':
+        # Only include the name if the parent is also a structure.
+        if parent_type == 'structure':
+            docs += "'" + name + '\': {\n'
+        else:
+            docs += '{\n'
+
+        # Go through each member and recursively process them.
+        for i, member_name in enumerate(shape.members):
+            member_eol = '\n'
+            if i < len(shape.members) - 1:
+                member_eol = ',\n'
+            docs += document_structure(
+                member_name, shape.members[member_name],
+                indent=indent + 2, parent_type=shape.type_name,
+                eol=member_eol)
+        docs += (' ' * indent) + '}' + eol
+    elif shape.type_name == 'list':
+        # Only include the name if the parent is a structure.
+        if parent_type == 'structure':
+            docs += "'" + name + '\': [\n'
+        else:
+            docs += '[\n'
+
+        # Lists have only a single member. Here we document it, plus add
+        # an ellipsis to signify that more of the same member type can be
+        # added in a list.
+        docs += document_structure(
+            None, shape.member, indent=indent + 2, eol=',\n')
+        docs += (' ' * indent) + '  ...\n'
+        docs += (' ' * indent) + ']' + eol
+    else:
+        # It's not a structure or list, so document the type. Here we
+        # try to use the equivalent Python type name for clarity.
+        if name is not None:
+            docs += ("'" + name + '\': ' +
+                     py_type_name(shape.type_name).upper() + eol)
+        else:
+            docs += py_type_name(shape.type_name).upper() + eol
 
     return docs

boto3/resources/factory.py

@@ -246,16 +246,27 @@ def _create_reference(factory_self, name, reference, service_name,
         # References are essentially an action with no request
         # or response, so we can re-use the response handlers to
         # build up resources from identifiers and data members.
-        handler = ResourceHandler('', factory_self, resource_defs,
-                                  service_model, reference.resource)
+        handler = ResourceHandler(reference.resource.path, factory_self,
+                                  resource_defs, service_model,
+                                  reference.resource)
+
+        # Are there any identifiers that need access to data members?
+        # This is important when building the resource below since
+        # it requires the data to be loaded.
+        needs_data = any(i.source == 'data' for i in
+                         reference.resource.identifiers)
 
         def get_reference(self):
             # We need to lazy-evaluate the reference to handle circular
             # references between resources. We do this by loading the class
             # when first accessed.
-            # First, though, we need to see if we have the required
-            # identifiers to instantiate the resource reference.
-            return handler(self, {}, {})
+            # This is using a *response handler* so we need to make sure
+            # our data is loaded (if possible) and pass that data into
+            # the handler as if it were a response. This allows references
+            # to have their data loaded properly.
+            if needs_data and self.meta.data is None and hasattr(self, 'load'):
+                self.load()
+            return handler(self, {}, self.meta.data)
 
         get_reference.__name__ = str(reference.name)
         get_reference.__doc__ = 'TODO'

setup.py

@@ -26,7 +26,7 @@ def get_version():
 ]
 
 requires = [
-    'botocore==0.97.0',
+    'botocore==0.99.0',
     'bcdoc==0.12.2',
     'jmespath==0.6.1',
 ]
@@ -50,7 +50,7 @@ def get_version():
     install_requires=requires,
     license=open("LICENSE").read(),
     classifiers=[
-        'Development Status :: 3 - Alpha',
+        'Development Status :: 4 - Beta',
         'Intended Audience :: Developers',
         'Natural Language :: English',
         'License :: OSI Approved :: Apache Software License',

tests/unit/resources/test_factory.py

@@ -553,7 +553,7 @@ def test_resource_loads_waiters(self):
                 }
             }
         }
-        
+
         defs = {
             'Bucket': {}
         }
@@ -686,6 +686,67 @@ def test_dangling_resource_inequality(self):
         self.assertNotEqual(q1, q2)
         self.assertNotEqual(q1, m)
 
+    def test_dangling_resource_loads_data(self):
+        # Given a loadable resource instance that contains a reference
+        # to another resource which has a resource data path, the
+        # referenced resource should be loaded with all of the data
+        # contained at that path. This allows loading references
+        # which would otherwise not be loadable (missing load method)
+        # and prevents extra load calls for others when we already
+        # have the data available.
+        self.defs = {
+            'Instance': {
+                'identifiers': [{'name': 'Id'}],
+                'has': {
+                    'NetworkInterface': {
+                        'resource': {
+                            'type': 'NetworkInterface',
+                            'identifiers': [
+                                {'target': 'Id', 'source': 'data',
+                                 'path': 'NetworkInterface.Id'}
+                            ],
+                            'path': 'NetworkInterface'
+                        }
+                    }
+                }
+            },
+            'NetworkInterface': {
+                'identifiers': [{'name': 'Id'}],
+                'shape': 'NetworkInterfaceShape'
+            }
+        }
+        self.model = self.defs['Instance']
+        shape = DenormalizedStructureBuilder().with_members({
+            'Id': {
+                'type': 'string',
+            },
+            'PublicIp': {
+                'type': 'string'
+            }
+        }).build_model()
+        service_model = mock.Mock()
+        service_model.shape_for.return_value = shape
+
+        cls = self.load('test', 'Instance', self.model, self.defs,
+                        service_model)
+        instance = cls('instance-id')
+
+        # Set some data as if we had completed a load action.
+        def set_meta_data():
+            instance.meta.data = {
+                'NetworkInterface': {
+                    'Id': 'network-interface-id',
+                    'PublicIp': '127.0.0.1'
+                }
+            }
+        instance.load = mock.Mock(side_effect=set_meta_data)
+
+        # Now, get the reference and make sure it has its data
+        # set as expected.
+        interface = instance.network_interface
+        self.assertIsNotNone(interface.meta.data)
+        self.assertEqual(interface.public_ip, '127.0.0.1')
+
 
 class TestServiceResourceSubresources(BaseTestResourceFactory):
     def setUp(self):