From 6c43a77ca8fb362975e975702944dcc4f74444f4 Mon Sep 17 00:00:00 2001 From: Andrew Caldwell Date: Mon, 23 Jun 2014 18:12:26 -0500 Subject: [PATCH] Optional s3 sync flag to address timestamp issues described in https://github.com/aws/aws-cli/issues/599 --- awscli/customizations/s3/comparator.py | 9 ++ awscli/customizations/s3/s3.py | 6 ++ .../unit/customizations/s3/test_comparator.py | 99 +++++++++++++++++++ 3 files changed, 114 insertions(+) diff --git a/awscli/customizations/s3/comparator.py b/awscli/customizations/s3/comparator.py index 555bd24bf2c4..9c86eeee9e96 100644 --- a/awscli/customizations/s3/comparator.py +++ b/awscli/customizations/s3/comparator.py @@ -38,6 +38,10 @@ def __init__(self, params=None): if 'size_only' in params: self.compare_on_size_only = params['size_only'] + self.match_exact_timestamps = False + if 'exact_timestamps' in params: + self.match_exact_timestamps = params['exact_timestamps'] + def call(self, src_files, dest_files): """ This function preforms the actual comparisons. The parameters it takes @@ -198,6 +202,11 @@ def compare_time(self, src_file, dest_file): # at the source location. return False elif cmd == "download": + if self.match_exact_timestamps: + # An update is needed unless the + # timestamps match exactly. + return total_seconds(delta) == 0 + if total_seconds(delta) <= 0: return True else: diff --git a/awscli/customizations/s3/s3.py b/awscli/customizations/s3/s3.py index f501c732836e..087e26bb145f 100644 --- a/awscli/customizations/s3/s3.py +++ b/awscli/customizations/s3/s3.py @@ -796,6 +796,7 @@ def add_verify_ssl(self, parsed_globals): 'sse', 'storage-class', 'content-type', 'cache-control', 'content-disposition', 'content-encoding', 'content-language', + 'exact-timestamps', 'expires', 'size-only']}, 'ls': {'options': {'nargs': '?', 'default': 's3://'}, 'params': ['recursive'], 'default': 's3://', @@ -850,6 +851,11 @@ def add_verify_ssl(self, parsed_globals): 'size-only': {'options': {'action': 'store_true'}, 'documents': ('Makes the size of each key the only criteria used to ' 'decide whether to sync from source to destination.')}, + 'exact-timestamps': {'options': {'action': 'store_true'}, 'documents': + ('When syncing from S3 to local, same-sized items will be ' + 'ignored only when the timestamps match exactly. The ' + 'default behavior is to ignore same-sized items unless ' + 'the local version is newer than the S3 version.')}, 'index-document': {'options': {}, 'documents': ('A suffix that is appended to a request that is for a ' 'directory on the website endpoint (e.g. if the suffix ' diff --git a/tests/unit/customizations/s3/test_comparator.py b/tests/unit/customizations/s3/test_comparator.py index e0a983e90517..d0d5652d1bc9 100644 --- a/tests/unit/customizations/s3/test_comparator.py +++ b/tests/unit/customizations/s3/test_comparator.py @@ -351,5 +351,104 @@ def test_compare_size_only_src_older_than_dest(self): self.assertEqual(sum(1 for _ in files), 0) +class ComparatorExactTimestampsTest(unittest.TestCase): + def setUp(self): + self.comparator = Comparator({'exact_timestamps': True}) + + def test_compare_exact_timestamps_dest_older(self): + """ + Confirm that same-sized files are synced when + the destination is older than the source and + `exact_timestamps` is set. + """ + time_src = datetime.datetime.now() + time_dst = time_src - datetime.timedelta(days=1) + + src_file = FileInfo(src='', dest='', + compare_key='test.py', size=10, + last_update=time_src, src_type='s3', + dest_type='local', operation_name='download', + service=None, endpoint=None) + + dst_file = FileInfo(src='', dest='', + compare_key='test.py', size=10, + last_update=time_dst, src_type='local', + dest_type='s3', operation_name='', + service=None, endpoint=None) + + files = self.comparator.call(iter([src_file]), iter([dst_file])) + self.assertEqual(sum(1 for _ in files), 1) + + def test_compare_exact_timestamps_src_older(self): + """ + Confirm that same-sized files are synced when + the source is older than the destination and + `exact_timestamps` is set. + """ + time_src = datetime.datetime.now() - datetime.timedelta(days=1) + time_dst = datetime.datetime.now() + + src_file = FileInfo(src='', dest='', + compare_key='test.py', size=10, + last_update=time_src, src_type='s3', + dest_type='local', operation_name='download', + service=None, endpoint=None) + + dst_file = FileInfo(src='', dest='', + compare_key='test.py', size=10, + last_update=time_dst, src_type='local', + dest_type='s3', operation_name='', + service=None, endpoint=None) + + files = self.comparator.call(iter([src_file]), iter([dst_file])) + self.assertEqual(sum(1 for _ in files), 1) + + def test_compare_exact_timestamps_same_age_same_size(self): + """ + Confirm that same-sized files are not synced when + the source and destination are the same age and + `exact_timestamps` is set. + """ + time_both = datetime.datetime.now() + + src_file = FileInfo(src='', dest='', + compare_key='test.py', size=10, + last_update=time_both, src_type='s3', + dest_type='local', operation_name='download', + service=None, endpoint=None) + + dst_file = FileInfo(src='', dest='', + compare_key='test.py', size=10, + last_update=time_both, src_type='local', + dest_type='s3', operation_name='', + service=None, endpoint=None) + + files = self.comparator.call(iter([src_file]), iter([dst_file])) + self.assertEqual(sum(1 for _ in files), 0) + + def test_compare_exact_timestamps_same_age_diff_size(self): + """ + Confirm that files of differing sizes are synced when + the source and destination are the same age and + `exact_timestamps` is set. + """ + time_both = datetime.datetime.now() + + src_file = FileInfo(src='', dest='', + compare_key='test.py', size=20, + last_update=time_both, src_type='s3', + dest_type='local', operation_name='download', + service=None, endpoint=None) + + dst_file = FileInfo(src='', dest='', + compare_key='test.py', size=10, + last_update=time_both, src_type='local', + dest_type='s3', operation_name='', + service=None, endpoint=None) + + files = self.comparator.call(iter([src_file]), iter([dst_file])) + self.assertEqual(sum(1 for _ in files), 1) + + if __name__ == "__main__": unittest.main()