Enable use of page_size for clients

botocore/paginate.py

@@ -21,15 +21,16 @@
 
 class PageIterator(object):
     def __init__(self, method, input_token, output_token, more_results,
-                 result_keys, non_aggregate_keys, max_items, starting_token,
-                 page_size, op_kwargs):
+                 result_keys, non_aggregate_keys, limit_key, max_items,
+                 starting_token, page_size, op_kwargs):
         self._method = method
         self._op_kwargs = op_kwargs
         self._input_token = input_token
         self._output_token = output_token
         self._more_results = more_results
         self._result_keys = result_keys
         self._max_items = max_items
+        self._limit_key = limit_key
         self._starting_token = starting_token
         self._page_size = page_size
         self._op_kwargs = op_kwargs
@@ -135,11 +136,9 @@ def _inject_starting_params(self, op_kwargs):
             next_token = self._parse_starting_token()[0]
             self._inject_token_into_kwargs(op_kwargs, next_token)
         if self._page_size is not None:
-            # Determine the proper parameter name for limiting
-            # page size.
-            page_size_name = self._operation.pagination['limit_key']
-            # Pass the page size as the determined parameter name.
-            op_kwargs[page_size_name] = self._page_size
+            # Pass the page size as the parameter name for limiting
+            # page size, also known as the limit_key.
+            op_kwargs[self._limit_key] = self._page_size
 
     def _inject_token_into_kwargs(self, op_kwargs, next_token):
         for name, token in zip(self._input_token, next_token):
@@ -268,6 +267,7 @@ def __init__(self, method, pagination_config):
         self._non_aggregate_keys = self._get_non_aggregate_keys(
             self._pagination_cfg)
         self._result_keys = self._get_result_keys(self._pagination_cfg)
+        self._limit_key = self._get_limit_key(self._pagination_cfg)
 
     @property
     def result_keys(self):
@@ -307,6 +307,9 @@ def _get_result_keys(self, config):
             result_key = [jmespath.compile(rk) for rk in result_key]
             return result_key
 
+    def _get_limit_key(self, config):
+        return config.get('limit_key')
+
     def paginate(self, **kwargs):
         """Create paginator object for an operation.
 
@@ -320,6 +323,7 @@ def paginate(self, **kwargs):
             self._method, self._input_token,
             self._output_token, self._more_results,
             self._result_keys, self._non_aggregate_keys,
+            self._limit_key,
             page_params['max_items'],
             page_params['starting_token'],
             page_params['page_size'],
@@ -373,12 +377,12 @@ def __iter__(self):
 class DeprecatedPageIterator(PageIterator):
     def __init__(self, operation, endpoint, input_token,
                  output_token, more_results,
-                 result_keys, non_aggregate_keys, max_items, starting_token,
-                 page_size, op_kwargs):
+                 result_keys, non_aggregate_keys, limit_key, max_items,
+                 starting_token, page_size, op_kwargs):
         super(DeprecatedPageIterator, self).__init__(
             None, input_token, output_token, more_results, result_keys,
-            non_aggregate_keys, max_items, starting_token, page_size,
-            op_kwargs)
+            non_aggregate_keys, limit_key, max_items,
+            starting_token, page_size, op_kwargs)
         self._operation = operation
         self._endpoint = endpoint
 
@@ -410,6 +414,7 @@ def paginate(self, endpoint, **kwargs):
             self._operation, endpoint, self._input_token,
             self._output_token, self._more_results,
             self._result_keys, self._non_aggregate_keys,
+            self._limit_key,
             page_params['max_items'],
             page_params['starting_token'],
             page_params['page_size'],

tests/integration/test_s3.py

@@ -38,7 +38,8 @@ class BaseS3Test(unittest.TestCase):
     def setUp(self):
         self.session = botocore.session.get_session()
         self.service = self.session.get_service('s3')
-        self.endpoint = self.service.get_endpoint('us-east-1')
+        self.region = 'us-east-1'
+        self.endpoint = self.service.get_endpoint(self.region)
         self.keys = []
 
     def create_object(self, key_name, body='foo'):
@@ -204,6 +205,23 @@ def test_can_paginate_with_page_size(self):
                      for el in data]
         self.assertEqual(key_names, ['key0', 'key1', 'key2', 'key3', 'key4'])
 
+    def test_client_can_paginate_with_page_size(self):
+        for i in range(5):
+            key_name = 'key%s' % i
+            self.create_object(key_name)
+        # Eventual consistency.
+        time.sleep(3)
+        client = self.session.create_client('s3', region_name=self.region)
+        paginator = client.get_paginator('list_objects')
+        generator = paginator.paginate(page_size=1,
+                                       Bucket=self.bucket_name)
+        responses = list(generator)
+        self.assertEqual(len(responses), 5, responses)
+        data = [r for r in responses]
+        key_names = [el['Contents'][0]['Key']
+                     for el in data]
+        self.assertEqual(key_names, ['key0', 'key1', 'key2', 'key3', 'key4'])
+
     def test_result_key_iters(self):
         for i in range(5):
             key_name = 'key/%s/%s' % (i, i)

tests/unit/test_paginate.py

@@ -12,6 +12,7 @@
 # language governing permissions and limitations under the License.
 
 from tests import unittest
+from botocore.paginate import Paginator as FuturePaginator
 from botocore.paginate import DeprecatedPaginator as Paginator
 from botocore.exceptions import PaginationError
 from botocore.operation import Operation
@@ -149,6 +150,36 @@ def test_more_tokens_is_path_expression(self):
              mock.call(None, NextToken='token1'),])
 
 
+class TestFuturePaginator(unittest.TestCase):
+    def setUp(self):
+        self.method = mock.Mock()
+        self.paginate_config = {
+            "output_token": "Marker",
+            "input_token": "Marker",
+            "result_key": "Users",
+            "limit_key": "MaxKeys",
+        }
+
+        self.paginator = FuturePaginator(self.method, self.paginate_config)
+
+    def test_with_page_size(self):
+        responses = [
+            {"Users": ["User1"], "Marker": "m1"},
+            {"Users": ["User2"], "Marker": "m2"},
+            {"Users": ["User3"]},
+        ]
+        self.method.side_effect = responses
+        users = []
+        for page in self.paginator.paginate(page_size=1):
+            users += page['Users']
+        self.assertEqual(
+            self.method.call_args_list,
+            [mock.call(MaxKeys=1),
+             mock.call(Marker='m1', MaxKeys=1),
+             mock.call(Marker='m2', MaxKeys=1)]
+        )
+
+
 class TestPaginatorObjectConstruction(unittest.TestCase):
     def test_pagination_delegates_to_paginator(self):
         paginator_cls = mock.Mock()