Properly percent encode values in URI strings

botocore/auth.py

@@ -24,9 +24,10 @@
 import time
 
 from botocore.exceptions import NoCredentialsError
-from botocore.utils import normalize_url_path
+from botocore.utils import normalize_url_path, percent_encode_sequence
+from botocore.utils import percent_encode
 from botocore.compat import HTTPHeaders
-from botocore.compat import quote, unquote, urlsplit, parse_qs, urlencode
+from botocore.compat import quote, unquote, urlsplit, parse_qs
 from botocore.compat import urlunsplit
 
 logger = logging.getLogger(__name__)
@@ -379,7 +380,7 @@ def _modify_request_before_signing(self, request):
         # You can't mix the two types of params together, i.e just keep doing
         # new_query_params.update(op_params)
         # new_query_params.update(auth_params)
-        # urlencode(new_query_params)
+        # percent_encode_sequence(new_query_params)
         operation_params = ''
         if request.data:
             # We also need to move the body params into the query string.
@@ -389,8 +390,9 @@ def _modify_request_before_signing(self, request):
             query_dict.update(request.data)
             request.data = ''
         if query_dict:
-            operation_params = urlencode(query_dict) + '&'
-        new_query_string = operation_params + urlencode(auth_params)
+            operation_params = percent_encode_sequence(query_dict) + '&'
+        new_query_string = operation_params + \
+                percent_encode_sequence(auth_params)
         # url_parts is a tuple (and therefore immutable) so we need to create
         # a new url_parts with the new query string.
         # <part>   - <index>

botocore/utils.py

@@ -12,17 +12,21 @@
 # language governing permissions and limitations under the License.
 import logging
 
+from six import string_types, text_type
 
-from .exceptions import InvalidExpressionError, ConfigNotFound
-from .compat import json
-from .vendored import requests
+from botocore.exceptions import InvalidExpressionError, ConfigNotFound
+from botocore.compat import json, quote
+from botocore.vendored import requests
 
 
 logger = logging.getLogger(__name__)
 DEFAULT_METADATA_SERVICE_TIMEOUT = 1
 METADATA_SECURITY_CREDENTIALS_URL = (
     'http://169.254.169.254/latest/meta-data/iam/security-credentials/'
 )
+# These are chars that do not need to be urlencoded.
+# Based on rfc2986, section 2.3
+SAFE_CHARS = '-._~'
 
 
 class _RetriesExceededError(Exception):
@@ -223,3 +227,43 @@ def parse_key_val_file_contents(contents):
         val = val.strip()
         final[key] = val
     return final
+
+
+def percent_encode_sequence(mapping, safe=SAFE_CHARS):
+    """Urlencode a dict or list into a string.
+
+    This is similar to urllib.urlencode except that:
+
+    * It uses quote, and not quote_plus
+    * It has a default list of safe chars that don't need
+      to be encoded, which matches what AWS services expect.
+
+    This function should be preferred over the stdlib
+    ``urlencode()`` function.
+
+    :param mapping: Either a dict to urlencode or a list of
+        ``(key, value)`` pairs.
+
+    """
+    encoded_pairs = []
+    if hasattr(mapping, 'items'):
+        pairs = mapping.items()
+    else:
+        pairs = mapping
+    for key, value in pairs:
+        encoded_pairs.append('%s=%s' % (percent_encode(key),
+                                        percent_encode(value)))
+    return '&'.join(encoded_pairs)
+
+
+def percent_encode(input_str, safe=SAFE_CHARS):
+    """Urlencodes a string.
+
+    Whereas percent_encode_sequence handles taking a dict/sequence and
+    producing a percent encoded string, this function deals only with
+    taking a string (not a dict/sequence) and percent encoding it.
+
+    """
+    if not isinstance(input_str, string_types):
+        input_str = text_type(input_str)
+    return quote(text_type(input_str), safe=safe)

tests/unit/auth/test_signers.py

@@ -228,6 +228,14 @@ def test_operation_params_before_auth_params_in_body(self):
         self.assertIn(
             '?Action=MyOperation&X-Amz', request.url)
 
+    def test_presign_with_spaces_in_param(self):
+        request = AWSRequest()
+        request.url = 'https://ec2.us-east-1.amazonaws.com/'
+        request.data = {'Action': 'MyOperation', 'Description': 'With Spaces'}
+        self.auth.add_auth(request)
+        # Verify we encode spaces as '%20, and we don't use '+'.
+        self.assertIn('Description=With%20Spaces', request.url)
+
     def test_s3_sigv4_presign(self):
         auth = botocore.auth.S3SigV4QueryAuth(
             self.credentials, self.service_name, self.region_name, expires=60)