Add compat layer for text based stream writers

In python2, if a stream has no encoding, unicode is encoded
using the default encoding.  In python3, sys.stdout is already
text based (it expects str() types).

In python2, we introduce a compat layer that will give you a
stream writer that accepts unicode and encodes the contents into
the local.preferredencoding().

Simple repro: tag an ec2 instance with a unicode char and run:

    aws ec2 describe-instances | grep TAGS

You'll get a ascii encode error.  With this change you won't.

Fixes aws#742.

awscli/compat.py

@@ -0,0 +1,23 @@
+# Copyright 2012-2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+
+#     http://aws.amazon.com/apache2.0/
+
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+import sys
+import six
+
+if six.PY3:
+    def get_stdout_text_writer():
+        return sys.stdout
+else:
+    import codecs
+    import locale
+    def get_stdout_text_writer():
+        return codecs.getwriter(locale.getpreferredencoding())(sys.stdout)

awscli/formatter.py

@@ -13,15 +13,19 @@
 import logging
 import sys
 import json
-import jmespath
+import codecs
+import locale
 
+import jmespath
 from botocore.utils import set_value_from_jmespath
 
 from awscli.table import MultiTable, Styler, ColorizedStyler
 from awscli import text
+from awscli import compat
 
 
 LOG = logging.getLogger(__name__)
+DEFAULT_ENCODING = 'utf-8'
 
 
 class Formatter(object):
@@ -40,14 +44,24 @@ def _remove_request_id(self, response_data):
                     LOG.debug('RequestId: %s', request_id)
                 del response_data['ResponseMetadata']
 
+    def _get_default_stream(self):
+        if getattr(sys.stdout, 'encoding', None) is None:
+            # In python3, sys.stdout.encoding is always set.
+            # In python2, if you redirect to stdout, then
+            # encoding is not None.  In this case we'll default
+            # to utf-8.
+            return compat.get_stdout_text_writer()
+        else:
+            return sys.stdout
+
 
 class FullyBufferedFormatter(Formatter):
     def __call__(self, operation, response, stream=None):
         if stream is None:
             # Retrieve stdout on invocation instead of at import time
             # so that if anything wraps stdout we'll pick up those changes
             # (specifically colorama on windows wraps stdout).
-            stream = sys.stdout
+            stream = self._get_default_stream()
         # I think the interfaces between non-paginated
         # and paginated responses can still be cleaned up.
         if operation.can_paginate and self._args.paginate:
@@ -207,7 +221,7 @@ class TextFormatter(Formatter):
 
     def __call__(self, operation, response, stream=None):
         if stream is None:
-            stream = sys.stdout
+            stream = self._get_default_stream()
         try:
             if operation.can_paginate and self._args.paginate:
                 result_keys = response.result_keys

tests/unit/output/test_text_output.py

@@ -12,14 +12,19 @@
 # ANY KIND, either express or implied. See the License for the specific
 # language governing permissions and limitations under the License.
 from tests.unit import BaseAWSCommandParamsTest
+from tests import unittest
 import json
 import os
 import sys
 import re
+import locale
 
 from six.moves import cStringIO
+import six
 import mock
 
+from awscli.formatter import Formatter
+
 
 class TestListUsers(BaseAWSCommandParamsTest):
 
@@ -73,3 +78,22 @@ def test_text_response(self):
         self.assertEqual(
             output,
             'ENGINEDEFAULTS\tNone\n')
+
+
+class CustomFormatter(Formatter):
+    def __call__(self, operation, response, stream=None):
+        self.stream = self._get_default_stream()
+
+
+class TestDefaultStream(BaseAWSCommandParamsTest):
+    @unittest.skipIf(six.PY3, "Text writer only vaild on py3.")
+    def test_default_stream_with_table_output(self):
+        formatter = CustomFormatter(None)
+        stream = cStringIO()
+        with mock.patch('sys.stdout', stream):
+            formatter(None, None)
+            formatter.stream.write(u'\u2713')
+        # Ensure the unicode data is written as UTF-8 by default.
+        self.assertEqual(
+            formatter.stream.getvalue(),
+            u'\u2713'.encode(locale.getpreferredencoding()))