Resource refactor

.gitignore

@@ -33,3 +33,4 @@ nosetests.xml
 AUTHORS
 ChangeLog
 dist
+__pycache__

cinq_collector_aws/account.py

@@ -1,7 +1,7 @@
 from collections import defaultdict
+from datetime import datetime, timedelta
 
 from botocore.exceptions import ClientError
-from datetime import datetime, timedelta
 from cloud_inquisitor import get_aws_session
 from cloud_inquisitor.config import dbconfig
 from cloud_inquisitor.database import db
@@ -99,11 +99,22 @@ def update_s3buckets(self):
                     tags = {}
 
                 try:
-                    bucket_size = self._get_bucket_statistics(data.name, bucket_region, 'StandardStorage',
-                                                              'BucketSizeBytes', 3)
+                    # bucket_name, bucket_region, storage_type, statistic, days
+                    bucket_size = self._get_bucket_statistics(
+                        bucket_name=data.name,
+                        bucket_region=bucket_region,
+                        storage_type='StandardStorage',
+                        statistic='BucketSizeBytes',
+                        days=3
+                    )
 
-                    bucket_obj_count = self._get_bucket_statistics(data.name, bucket_region, 'AllStorageTypes',
-                                                                   'NumberOfObjects', 3)
+                    bucket_obj_count = self._get_bucket_statistics(
+                        bucket_name=data.name,
+                        bucket_region=bucket_region,
+                        storage_type='AllStorageTypes',
+                        statistic='NumberOfObjects',
+                        days=3
+                    )
 
                     metrics = {'size': bucket_size, 'object_count': bucket_obj_count}
 
@@ -112,25 +123,22 @@ def update_s3buckets(self):
                     metrics = {'found': False}
 
                 properties = {
-                    'bucket_policy': bucket_policy,
                     'creation_date': data.creation_date,
-                    'location': bucket_region,
+                    'bucket_policy': bucket_policy,
                     'website_enabled': website_enabled,
                     'metrics': metrics,
-                    'tags': tags
                 }
 
                 if data.name in existing_buckets:
                     bucket = existing_buckets[data.name]
-                    if bucket.update(data, properties):
+                    if bucket.update_resource(properties=properties, tags=tags):
                         self.log.debug('Change detected for S3Bucket {}/{}'.format(
                             self.account.account_name,
                             bucket.id
                         ))
                         bucket.save()
                 else:
                     # If a bucket has no tags, a boto3 error is thrown. We treat this as an empty tag set
-
                     S3Bucket.create(
                         data.name,
                         account_id=self.account.account_id,
@@ -214,6 +222,7 @@ def update_cloudfront(self):
                                 )
 
                         data = {
+                            'id': get_resource_id('cfd', dist['ARN']),
                             'arn': dist['ARN'],
                             'name': dist['DomainName'],
                             'origins': origins,
@@ -242,6 +251,7 @@ def update_cloudfront(self):
                 if 'Items' in dl:
                     dists += [
                         {
+                            'id': get_resource_id('cfd', x['ARN']),
                             'arn': x['ARN'],
                             'name': x['DomainName'],
                             'origins': [{'type': 's3', 'source': x['S3Origin']['DomainName']}],
@@ -254,45 +264,47 @@ def update_cloudfront(self):
             # endregion
 
             for data in dists:
-                if data['arn'] in existing_dists:
-                    dist = existing_dists[data['arn']]
-                    if dist.update(data):
+                properties = {
+                    'arn': data['arn'],
+                    'domain_name': data['name'],
+                    'origins': data['origins'],
+                    'enabled': data['enabled'],
+                    'type': data['type']
+                }
+
+                if data['id'] in existing_dists:
+                    dist = existing_dists[data['id']]
+
+                    if dist.update_resource(properties=properties, tags=data['tags']):
                         self.log.debug('Updated CloudFrontDist {}/{}'.format(
                             self.account.account_name,
                             data['name']
                         ))
                         dist.save()
 
                 else:
-                    properties = {
-                        'domain_name': data['name'],
-                        'origins': data['origins'],
-                        'enabled': data['enabled'],
-                        'type': data['type']
-                    }
-
                     CloudFrontDist.create(
-                        data['arn'],
+                        data['id'],
                         account_id=self.account.account_id,
                         properties=properties,
                         tags=data['tags']
                     )
 
                     self.log.debug('Added new CloudFrontDist {}/{}'.format(
                         self.account.account_name,
-                        data['name']
+                        data['id']
                     ))
             db.session.commit()
 
-            dk = set(x['arn'] for x in dists)
+            dk = set(x['id'] for x in dists)
             edk = set(existing_dists.keys())
 
             try:
                 for resource_id in edk - dk:
                     db.session.delete(existing_dists[resource_id].resource)
                     self.log.debug('Deleted CloudFrontDist {}/{}'.format(
+                        self.account.account_name,
                         resource_id,
-                        self.account.account_name
                     ))
                 db.session.commit()
             except:
@@ -313,16 +325,17 @@ def update_route53(self):
         existing_zones = DNSZone.get_all(self.account)
         zones = self.__fetch_route53_zones()
         for resource_id, data in zones.items():
+            tags = data.pop('tags')
             if resource_id in existing_zones:
                 zone = DNSZone.get(resource_id)
-                if zone.update(data):
+                if zone.update_resource(properties=data, tags=tags):
                     self.log.debug('Change detected for Route53 zone {}/{}'.format(
                         self.account,
                         zone.name
                     ))
                     zone.save()
             else:
-                tags = data.pop('tags')
+
                 DNSZone.create(
                     resource_id,
                     account_id=self.account.account_id,
@@ -352,50 +365,46 @@ def update_route53(self):
         # endregion
 
         # region Update resource records
-        try:
-            for zone_id, zone in DNSZone.get_all(self.account).items():
-                existing_records = {rec.id: rec for rec in zone.records}
-                records = self.__fetch_route53_zone_records(zone.get_property('zone_id').value)
-
-                for data in records:
-                    if data['id'] in existing_records:
-                        record = existing_records[data['id']]
-                        if record.update(data):
-                            self.log.debug('Changed detected for DNSRecord {}/{}/{}'.format(
-                                self.account,
-                                zone.name,
-                                data['name']
-                            ))
-                            record.save()
-                    else:
-                        record = DNSRecord.create(
-                            data['id'],
-                            account_id=self.account.account_id,
-                            properties={k: v for k, v in data.items() if k != 'id'},
-                            tags={}
-                        )
-                        self.log.debug('Added new DNSRecord {}/{}/{}'.format(
+        for zone_id, zone in DNSZone.get_all(self.account).items():
+            existing_records = {rec.id: rec for rec in zone.records}
+            records = self.__fetch_route53_zone_records(zone.get_property('zone_id').value)
+
+            for record_id, data in records.items():
+                if record_id in existing_records:
+                    record = existing_records[record_id]
+                    if record.update_resource(properties=data):
+                        self.log.debug('Changed detected for DNSRecord {}/{}/{}'.format(
                             self.account,
                             zone.name,
                             data['name']
                         ))
-                        zone.add_record(record)
-                db.session.commit()
+                        record.save()
+                else:
+                    record = DNSRecord.create(
+                        record_id,
+                        account_id=self.account.account_id,
+                        properties=data
+                    )
+                    self.log.debug('Added new DNSRecord {}/{}/{}'.format(
+                        self.account,
+                        zone.name,
+                        data['name']
+                    ))
+                    zone.add_child(record)
+            db.session.commit()
 
-                rk = set(x['id'] for x in records)
-                erk = set(existing_records.keys())
+            rk = set(records.keys())
+            erk = set(existing_records.keys())
 
-                for resource_id in erk - rk:
-                    record = existing_records[resource_id]
-                    zone.delete_record(record)
-                    self.log.debug('Deleted Route53 record {}/{}/{}'.format(
-                        self.account.account_name,
-                        zone_id,
-                        record.name
-                    ))
-                db.session.commit()
-        except:
-            raise
+            for resource_id in erk - rk:
+                record = existing_records[resource_id]
+                zone.delete_record(record)
+                self.log.debug('Deleted Route53 record {}/{}/{}'.format(
+                    self.account.account_name,
+                    zone_id,
+                    record.name
+                ))
+            db.session.commit()
         # endregion
 
     # region Helper functions
@@ -412,8 +421,8 @@ def __get_distribution_tags(self, client, arn):
         """
         return {
             t['Key']: t['Value'] for t in client.list_tags_for_resource(
-            Resource=arn
-        )['Tags']['Items']
+                Resource=arn
+            )['Tags']['Items']
         }
 
     @retry
@@ -467,23 +476,23 @@ def __fetch_route53_zone_records(self, zone_id):
         route53 = self.session.client('route53')
 
         done = False
-        nextName = nextType = None
+        next_name = next_type = None
         records = {}
 
         try:
             while not done:
-                if nextName and nextType:
+                if next_name and next_type:
                     response = route53.list_resource_record_sets(
                         HostedZoneId=zone_id,
-                        StartRecordName=nextName,
-                        StartRecordType=nextType
+                        StartRecordName=next_name,
+                        StartRecordType=next_type
                     )
                 else:
                     response = route53.list_resource_record_sets(HostedZoneId=zone_id)
 
                 if response['IsTruncated']:
-                    nextName = response['NextRecordName']
-                    nextType = response['NextRecordType']
+                    next_name = response['NextRecordName']
+                    next_type = response['NextRecordType']
                 else:
                     done = True
 
@@ -496,7 +505,6 @@ def __fetch_route53_zone_records(self, zone_id):
                         if 'AliasTarget' in record:
                             value = record['AliasTarget']['DNSName']
                             records[record_id] = {
-                                'id': record_id,
                                 'name': record['Name'].rstrip('.'),
                                 'type': 'ALIAS',
                                 'ttl': 0,
@@ -505,14 +513,13 @@ def __fetch_route53_zone_records(self, zone_id):
                         else:
                             value = [y['Value'] for y in record['ResourceRecords']]
                             records[record_id] = {
-                                'id': record_id,
                                 'name': record['Name'].rstrip('.'),
                                 'type': record['Type'],
                                 'ttl': record['TTL'],
                                 'value': value
                             }
 
-            return list(records.values())
+            return records
         finally:
             del route53
 
@@ -580,9 +587,6 @@ def _get_bucket_statistics(self, bucket_name, bucket_region, storage_type, stati
         """
 
         cw = self.session.client('cloudwatch', region_name=bucket_region)
-
-        # gather cw stats
-
         try:
             obj_stats = cw.get_metric_statistics(
                 Namespace='AWS/S3',