Merge branch 'fix-s3-download-mtime' into develop

* fix-s3-download-mtime:
  Set mtime of S3 downloaded object when file is closed

awscli/customizations/s3/executor.py

@@ -10,6 +10,7 @@
 # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
 # ANY KIND, either express or implied. See the License for the specific
 # language governing permissions and limitations under the License.
+import os
 import logging
 import sys
 import threading
@@ -175,6 +176,9 @@ def run(self):
                 if fileobj is not None:
                     fileobj.close()
                     del self.fd_descriptor_cache[task.filename]
+                if task.desired_mtime is not None:
+                    os.utime(task.filename, (task.desired_mtime,
+                                             task.desired_mtime))
 
     def _cleanup(self):
         for fileobj in self.fd_descriptor_cache.values():

awscli/customizations/s3/tasks.py

@@ -310,12 +310,12 @@ def __call__(self):
         self._context.wait_for_completion()
         last_update_tuple = self._filename.last_update.timetuple()
         mod_timestamp = time.mktime(last_update_tuple)
-        os.utime(self._filename.dest, (int(mod_timestamp), int(mod_timestamp)))
+        desired_mtime = int(mod_timestamp)
         message = print_operation(self._filename, False,
                                   self._parameters['dryrun'])
         print_task = {'message': message, 'error': False}
         self._result_queue.put(PrintTask(**print_task))
-        self._io_queue.put(IOCloseRequest(self._filename.dest))
+        self._io_queue.put(IOCloseRequest(self._filename.dest, desired_mtime))
 
 
 class DownloadPartTask(OrderableTask):

awscli/customizations/s3/utils.py

@@ -452,4 +452,7 @@ def __new__(cls, message, error=False, total_parts=None, warning=None):
                        ['filename', 'offset', 'data', 'is_stream'])
 # Used to signal that IO for the filename is finished, and that
 # any associated resources may be cleaned up.
-IOCloseRequest = namedtuple('IOCloseRequest', ['filename'])
+_IOCloseRequest = namedtuple('IOCloseRequest', ['filename', 'desired_mtime'])
+class IOCloseRequest(_IOCloseRequest):
+    def __new__(cls, filename, desired_mtime=None):
+        return super(IOCloseRequest, cls).__new__(cls, filename, desired_mtime)

tests/unit/customizations/s3/test_executor.py

@@ -13,12 +13,13 @@
 import os
 import tempfile
 import shutil
-from awscli.compat import six
-from six.moves import queue
+import time
 import sys
 
 import mock
 
+from awscli.compat import six
+from six.moves import queue
 from awscli.testutils import unittest, temporary_file
 from awscli.customizations.s3.executor import IOWriterThread
 from awscli.customizations.s3.executor import ShutdownThreadRequest
@@ -74,6 +75,17 @@ def test_multiple_files_in_queue(self):
         with open(second_file, 'rb') as f:
             self.assertEqual(f.read(), b'otherstuff')
 
+    def test_mtime_set_at_file_close_time(self):
+        # We're picking something other than the close time so that can verify
+        # that the IOCloseRequest can specify what the mtime should be.
+        now_time = int(time.time() - 100)
+        self.queue.put(IORequest(self.filename, 0, b'foobar', False))
+        self.queue.put(IOCloseRequest(self.filename, now_time))
+        self.queue.put(ShutdownThreadRequest())
+        self.io_thread.run()
+        actual_mtime = int(os.stat(self.filename).st_mtime)
+        self.assertEqual(actual_mtime, now_time)
+
     def test_stream_requests(self):
         # Test that offset has no affect on the order in which requests
         # are written to stdout. The order of requests for a stream are