template.yaml

AWSTemplateFormatVersion: 2010-09-09
Transform: 'AWS::Serverless-2016-10-31'

Parameters:
  Stage:
    Type: String
    Description: The stage where the application is running in, e.g., dev, prod.
    AllowedPattern: ^[a-z0-9_]+$
    Default: 'test'

Resources:
  DDBFanoutApp:
    Type: AWS::Serverless::Application
    Properties:
      Location:
        ApplicationId: arn:aws:serverlessrepo:us-east-1:646794253159:applications/aws-dynamodb-stream-eventbridge-fanout
        SemanticVersion: 0.1.1
      Parameters:
        DynamoDBStreamArn: !Sub '{{resolve:ssm:/applications/apprepo/${Stage}/ddb/Applications/StreamArn:1}}'

  DataBucket:
    Type: AWS::S3::Bucket
    Properties:
      BucketEncryption:
        ServerSideEncryptionConfiguration:
          - ServerSideEncryptionByDefault:
              SSEAlgorithm: aws:kms

  FirehoseLogGroup:
    Type: AWS::Logs::LogGroup
    Properties:
      RetentionInDays: 3653

  FirehoseLogStream:
    Type: AWS::Logs::LogStream
    Properties:
      LogGroupName: !Ref FirehoseLogGroup

  FirehoseS3Role:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Statement:
          - Action: sts:AssumeRole
            Effect: Allow
            Principal:
              Service: firehose.amazonaws.com

  EventBridgeToFirehoseRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Statement:
          - Action: sts:AssumeRole
            Effect: Allow
            Principal:
              Service: events.amazonaws.com

  EventBridgeToFirehosePolicy:
    Type: AWS::IAM::Policy
    Properties:
      Roles:
        - !Ref EventBridgeToFirehoseRole
      PolicyName: "eventbridge-to-firehose"
      PolicyDocument:
        Version: "2012-10-17"
        Statement:
          - Effect: Allow
            Action:
              - firehose:PutRecord
              - firehose:PutRecordBatch
            Resource:
              - !GetAtt Firehose.Arn

  FirehoseToS3Policy:
    Type: AWS::IAM::Policy
    Properties:
      Roles:
        - !Ref FirehoseS3Role
      PolicyName: "firehose-to-s3"
      PolicyDocument:
        Version: "2012-10-17"
        Statement:
          - Effect: Allow
            Action:
              - s3:PutObject
              - s3:AbortMultipartUpload
              - s3:GetBucketLocation
              - s3:GetObject
              - s3:ListBucket
              - s3:ListBucketMultipartUploads
              - s3:PutEncryptionConfiguration
            Resource:
              - !Sub arn:${AWS::Partition}:s3:::*
              - !Sub arn:${AWS::Partition}:s3:::*/*
          - Action:
              - glue:GetTable
              - glue:GetTableVersion
              - glue:GetTableVersions
            Effect: Allow
            Resource:
              - "*"
          - Effect: Allow
            Action: logs:PutLogEvents
            Resource: !GetAtt FirehoseLogGroup.Arn

  GlueDatabase:
    Type: AWS::Glue::Database
    Properties:
      CatalogId: !Ref AWS::AccountId
      DatabaseInput:
        # Using a name compatible with Apache Hive (alphanumeric and underscore characters) since the intent is to use this database with Athena.
        # Names auto generated by CloudFormation do not follow this pattern.
        # See https://docs.aws.amazon.com/athena/latest/ug/create-table.html for more details
        Name: !Sub aws_serverless_app_repo_reference_analytics_${Stage}
        Description: AWS Serverless App Repo reference implementation analytics database

  GlueTable:
    Type: AWS::Glue::Table
    Properties:
      DatabaseName: !Ref GlueDatabase
      CatalogId: !Ref AWS::AccountId
      TableInput:
        Name: applications
        StorageDescriptor:
          Columns:
            - Name: version
              Type: string
            - Name: id
              Type: string
            - Name: detail-type
              Type: string
            - Name: source
              Type: string
            - Name: account
              Type: string
            - Name: time
              Type: string
            - Name: region
              Type: string
            - Name: resources
              Type: array<string>
            - Name: detail
              Type: struct<eventID:string,eventName:string,eventVersion:string,eventSource:string,awsRegion:string,dynamodb:struct<approximateCreationDateTime:bigint,keys:struct<applicationId:struct<s:string>,userId:struct<s:string>>,newImage:struct<createdAt:struct<s:string>,author:struct<s:string>,description:struct<s:string>,applicationId:struct<s:string>,userId:struct<s:string>,version:struct<n:string>,homePageUrl:struct<s:string>>,sequenceNumber:string,sizeBytes:int,streamViewType:string,oldImage:struct<createdAt:struct<s:string>,author:struct<s:string>,description:struct<s:string>,applicationId:struct<s:string>,userId:struct<s:string>,version:struct<n:string>,homePageUrl:struct<s:string>>>,eventSourceARN:string>
          Location: !Sub s3://${DataBucket}/Applications/
          InputFormat: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat
          OutputFormat: org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat
          Compressed: false
          NumberOfBuckets: -1
          SerdeInfo:
            SerializationLibrary: org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe
            Parameters:
              serialization.format: '1'
          BucketColumns: []
          SortColumns: []
          StoredAsSubDirectories: false
        PartitionKeys:
          - Name: year
            Type: string
          - Name: month
            Type: string
          - Name: day
            Type: string
          - Name: hour
            Type: string
        TableType: EXTERNAL_TABLE

  Firehose:
    Type: AWS::KinesisFirehose::DeliveryStream
    Properties:
      DeliveryStreamType: DirectPut
      ExtendedS3DestinationConfiguration:
        BucketARN: !GetAtt DataBucket.Arn
        BufferingHints:
          IntervalInSeconds: 60
          SizeInMBs: 128
        CloudWatchLoggingOptions:
          Enabled: true
          LogGroupName: !Ref FirehoseLogGroup
          LogStreamName: !Ref FirehoseLogStream
        CompressionFormat: UNCOMPRESSED
        DataFormatConversionConfiguration:
          Enabled: true
          InputFormatConfiguration:
            Deserializer:
              OpenXJsonSerDe:
                CaseInsensitive: true
          OutputFormatConfiguration:
            Serializer:
              ParquetSerDe:
                Compression: SNAPPY
                EnableDictionaryCompression: false
          SchemaConfiguration:
            CatalogId: !Ref AWS::AccountId
            DatabaseName: !Ref GlueDatabase
            Region: !Ref AWS::Region
            RoleARN: !GetAtt FirehoseS3Role.Arn
            TableName: !Ref GlueTable
            VersionId: LATEST
        Prefix: "Applications/year=!{timestamp:YYYY}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/"
        ErrorOutputPrefix: "Errors/!{firehose:random-string}/!{firehose:error-output-type}/!{timestamp:yyyy/MM/dd}/"
        RoleARN: !GetAtt FirehoseS3Role.Arn

  EventRule:
    Type: AWS::Events::Rule
    Properties:
      Description: "Match DDB streams events and send them to Firehose for analytics"
      EventPattern: { "source": [ "aws-dynamodb-stream-eventbridge-fanout" ], "resources":[ !Sub '{{resolve:ssm:/applications/apprepo/${Stage}/ddb/Applications/StreamArn:1}}' ] }
      Targets:
        - Id: "firehose-analytics"
          Arn: !GetAtt Firehose.Arn
          RoleArn: !GetAtt EventBridgeToFirehoseRole.Arn

  # Storing resources names to SSM
  GlueDatabaseName:
    Type: "AWS::SSM::Parameter"
    Properties:
      Name: !Sub "/applications/apprepo/${Stage}/glue/database"
      Type: "String"
      Value: !Ref GlueDatabase
      Description: "Analytics database name"

  GlueTableName:
    Type: "AWS::SSM::Parameter"
    Properties:
      Name: !Sub "/applications/apprepo/${Stage}/glue/table/applications"
      Type: "String"
      Value: !Ref GlueTable
      Description: "Analytics applications table name"

  DataBucketName:
    Type: "AWS::SSM::Parameter"
    Properties:
      Name: !Sub "/applications/apprepo/${Stage}/s3/DataBucket/Name"
      Type: "String"
      Value: !Ref DataBucket
      Description: "DataBucket bucket name"