stack.yaml

AWSTemplateFormatVersion: 2010-09-09

Description: Setup infrastructure for Rate My Post project

Parameters:
  ProjectName:
    Type: String
    Default: ratemypost
  PreBucket:
    Type: String
    Default: ratemypost-pre
  AppBucket:
    Type: String
    Default: ratemypost-app

Mappings:
  TaskMap:
    upload:
      name: zip-to-s3
    triggerUnzip:
      name: s3-to-ecs
    unzip:
      name: unzip-on-s3
    parse:
      name: parse-xml
    featureEngineering:
      name: feature-engineering
    createFeatures:
      name: get-features
    triggerPredict:
      name: run-prediction
    predict:
      name: post-predict
  DirMap:
    unzip:
      name: unzip
    zip:
      name: zip
    data:
      name: data
    glue:
      name: glue
    jars:
      name: jars
    lambda:
      name: lambda
    modelTraining:
      name: training

Resources:
  Bucket:
    Type: AWS::S3::Bucket
    Properties:
      BucketName: !Ref ProjectName
      PublicAccessBlockConfiguration:
        BlockPublicAcls: true
        BlockPublicPolicy: true
        IgnorePublicAcls: true
        RestrictPublicBuckets: true
      NotificationConfiguration:
        LambdaConfigurations:
          - Event: s3:ObjectCreated:*
            Function: !GetAtt TriggerUnzipLambda.Arn
            Filter:
              S3Key:
                Rules:
                  - Name: prefix
                    Value: !Sub
                      - ${path}/
                      - path: !FindInMap [DirMap, zip, name]
    DependsOn:
      - TriggerUnzipLambdaPermission

  ECSCluster:
    Type: AWS::ECS::Cluster
    Properties:
      CapacityProviders:
        - FARGATE
      ClusterName: !Ref ProjectName

  GlueDB:
    Type: AWS::Glue::Database
    Properties:
      CatalogId: !Ref AWS::AccountId
      DatabaseInput:
        Name: default
        LocationUri: !Sub
          - s3://${bucketName}/${directoryName}
          - bucketName: !Ref ProjectName
            directoryName: !FindInMap [DirMap, data, name]

  GlueDBTraining:
    Type: AWS::Glue::Database
    Properties:
      CatalogId: !Ref AWS::AccountId
      DatabaseInput:
        Name: training
        LocationUri: !Sub
          - s3://${bucketName}/${directoryName}
          - bucketName: !Ref ProjectName
            directoryName: !FindInMap [DirMap, modelTraining, name]

  ECSRoleS3:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: 2012-10-17
        Statement:
          - Effect: Allow
            Principal:
              Service:
                - ecs-tasks.amazonaws.com
            Action:
              - sts:AssumeRole
      ManagedPolicyArns:
        - arn:aws:iam::aws:policy/AmazonS3FullAccess
      RoleName: ecs-run-task-s3

  LambdaRoleS3:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: 2012-10-17
        Statement:
          - Effect: Allow
            Principal:
              Service:
                - lambda.amazonaws.com
            Action:
              - sts:AssumeRole
      ManagedPolicyArns:
        - arn:aws:iam::aws:policy/CloudWatchLogsFullAccess
        - arn:aws:iam::aws:policy/AmazonEC2ContainerServiceFullAccess
        - arn:aws:iam::aws:policy/AmazonS3ReadOnlyAccess
      RoleName: lambda-get-s3-run-ecs

  GlueRoleS3:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: 2012-10-17
        Statement:
          - Effect: Allow
            Principal:
              Service:
                - glue.amazonaws.com
            Action:
              - sts:AssumeRole
      ManagedPolicyArns:
        - arn:aws:iam::aws:policy/AmazonS3FullAccess
        - arn:aws:iam::aws:policy/service-role/AWSGlueServiceNotebookRole
        - arn:aws:iam::aws:policy/service-role/AWSGlueServiceRole
      RoleName: glue-run-job-s3

  GlueRoleS3DynamoDB:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: 2012-10-17
        Statement:
          - Effect: Allow
            Principal:
              Service:
                - glue.amazonaws.com
            Action:
              - sts:AssumeRole
      ManagedPolicyArns:
        - arn:aws:iam::aws:policy/AmazonS3FullAccess
        - arn:aws:iam::aws:policy/service-role/AWSGlueServiceNotebookRole
        - arn:aws:iam::aws:policy/service-role/AWSGlueServiceRole
        - arn:aws:iam::aws:policy/AmazonDynamoDBFullAccess
      RoleName: glue-run-job-s3-dynamodb

  ECSExecutionRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: 2012-10-17
        Statement:
          - Effect: Allow
            Principal:
              Service:
                - ecs-tasks.amazonaws.com
            Action:
              - sts:AssumeRole
      ManagedPolicyArns:
        - arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy
      RoleName: ecs-execute-task

  UploadTaskRepo:
    Type: AWS::ECR::Repository
    Properties:
      RepositoryName: !FindInMap [TaskMap, upload, name]

  UnzipTaskRepo:
    Type: AWS::ECR::Repository
    Properties:
      RepositoryName: !FindInMap [TaskMap, unzip, name]

  UploadTaskDefinition:
    Type: AWS::ECS::TaskDefinition
    Properties:
      ContainerDefinitions:
        - Name: !FindInMap [TaskMap, upload, name]
          Image: !Sub
            - ${AccountId}.dkr.ecr.${Region}.amazonaws.com/${RepoName}
            - AccountId: !Ref AWS::AccountId
              Region: !Ref AWS::Region
              RepoName: !Ref UploadTaskRepo
          Essential: true
          LogConfiguration:
            LogDriver: awslogs
            Options:
              awslogs-group: !Sub
                - /ecs/${TaskName}
                - TaskName: !FindInMap [TaskMap, upload, name]
              awslogs-region: !Ref AWS::Region
              awslogs-stream-prefix: ecs
      Cpu: 2vCPU
      ExecutionRoleArn: !Ref ECSExecutionRole
      Family: !FindInMap [TaskMap, upload, name]
      Memory: 4GB
      NetworkMode: awsvpc
      RequiresCompatibilities:
        - FARGATE
      TaskRoleArn: !GetAtt ECSRoleS3.Arn
    DependsOn:
      - ECSRoleS3
      - UploadTaskRepo
      - UploadTaskLogGroup

  UnzipTaskDefinition:
    Type: AWS::ECS::TaskDefinition
    Properties:
      ContainerDefinitions:
        - Name: !FindInMap [TaskMap, unzip, name]
          Image: !Sub
            - ${AccountId}.dkr.ecr.${Region}.amazonaws.com/${RepoName}
            - AccountId: !Ref AWS::AccountId
              Region: !Ref AWS::Region
              RepoName: !Ref UnzipTaskRepo
          Essential: true
          LogConfiguration:
            LogDriver: awslogs
            Options:
              awslogs-group: !Sub
                - /ecs/${TaskName}
                - TaskName: !FindInMap [TaskMap, unzip, name]
              awslogs-region: !Ref AWS::Region
              awslogs-stream-prefix: ecs
      Cpu: 1vCPU
      ExecutionRoleArn: !Ref ECSExecutionRole
      Family: !FindInMap [TaskMap, unzip, name]
      Memory: 2GB
      NetworkMode: awsvpc
      RequiresCompatibilities:
        - FARGATE
      TaskRoleArn: !GetAtt ECSRoleS3.Arn
    DependsOn:
      - ECSRoleS3
      - UnzipTaskRepo
      - UploadTaskLogGroup

  UploadTaskLogGroup:
    Type: AWS::Logs::LogGroup
    Properties:
      LogGroupName: !Sub
        - /ecs/${TaskName}
        - TaskName: !FindInMap [TaskMap, upload, name]

  UnzipTaskLogGroup:
    Type: AWS::Logs::LogGroup
    Properties:
      LogGroupName: !Sub
        - /ecs/${TaskName}
        - TaskName: !FindInMap [TaskMap, unzip, name]

  TriggerUnzipLambda:
    Type: AWS::Lambda::Function
    Properties:
      Code:
        S3Bucket: !Ref PreBucket
        S3Key: !Sub
          - ${lambda_dir}/${task_name}/lambda_handler.py
          - lambda_dir: !FindInMap [DirMap, lambda, name]
            task_name: !FindInMap [TaskMap, triggerUnzip, name]
      Description: Run ECS task on S3 file
      Environment:
        Variables:
          CLUSTER_NAME: !Ref ECSCluster
          TASK_UNZIP: !FindInMap [TaskMap, unzip, name]
          UNZIP_DIR: !FindInMap [DirMap, unzip, name]
      FunctionName: !FindInMap [TaskMap, triggerUnzip, name]
      Handler: lambda_handler.lambda_handler
      MemorySize: 128
      PackageType: Zip
      Role: !GetAtt LambdaRoleS3.Arn
      Runtime: python3.7
      Timeout: 10
    DependsOn:
      - LambdaRoleS3

  TriggerUnzipLambdaPermission:
    Type: AWS::Lambda::Permission
    Properties:
      Action: lambda:InvokeFunction
      FunctionName: !GetAtt TriggerUnzipLambda.Arn
      Principal: s3.amazonaws.com
      SourceArn: !Sub arn:aws:s3:::${ProjectName}
    DependsOn:
      - TriggerUnzipLambda

  ParseXMLJob:
    Type: AWS::Glue::Job
    Properties:
      Command:
        Name: glueetl
        PythonVersion: 3
        ScriptLocation: !Sub
          - s3://${bucket_name}/${glue_dir}/${job_name}/glue_job.py
          - bucket_name: !Ref PreBucket
            glue_dir: !FindInMap [DirMap, glue, name]
            job_name: !FindInMap [TaskMap, parse, name]
      DefaultArguments:
        --enable-glue-datacatalog: ""
        --enable-metrics: ""
        --enable-spark-ui: true
        --enable-continuous-cloudwatch-log: true
        --enable-continuous-log-filter: true
        --job-language: python
        --extra-jars: !Sub
          - s3://${bucket_name}/${jar_dir}/spark-xml_2.11-0.11.0.jar
          - bucket_name: !Ref PreBucket
            jar_dir: !FindInMap [DirMap, jars, name]
        --bucket_name: !Ref ProjectName
        --input_dir: !FindInMap [DirMap, unzip, name]
        --output_dir: !FindInMap [DirMap, data, name]
        --database_name: default
      Description: Parse XML data and save to parquet
      ExecutionProperty:
        MaxConcurrentRuns: 1
      GlueVersion: 2.0
      MaxRetries: 0
      Name: !FindInMap [TaskMap, parse, name]
      NumberOfWorkers: 4
      Role: !GetAtt GlueRoleS3.Arn
      Timeout: 120
      WorkerType: G.1X
    DependsOn:
      - GlueRoleS3
      - Bucket

  GetFeaturesJob:
    Type: AWS::Glue::Job
    Properties:
      Command:
        Name: glueetl
        PythonVersion: 3
        ScriptLocation: !Sub
          - s3://${bucket_name}/${glue_dir}/${job_name}/glue_job.py
          - bucket_name: !Ref PreBucket
            glue_dir: !FindInMap [DirMap, glue, name]
            job_name: !FindInMap [TaskMap, createFeatures, name]
      DefaultArguments:
        --enable-glue-datacatalog: ""
        --enable-metrics: ""
        --enable-spark-ui: true
        --enable-continuous-cloudwatch-log: true
        --enable-continuous-log-filter: true
        --job-language: python
        --database_name: default
      Description: Create DynamoDB with features for prediction
      ExecutionProperty:
        MaxConcurrentRuns: 1
      GlueVersion: 2.0
      MaxRetries: 0
      Name: !FindInMap [TaskMap, createFeatures, name]
      NumberOfWorkers: 4
      Role: !GetAtt GlueRoleS3DynamoDB.Arn
      Timeout: 120
      WorkerType: G.1X
    DependsOn:
      - GlueRoleS3DynamoDB
      - Bucket

  FeatureEngineeringJob:
    Type: AWS::Glue::Job
    Properties:
      Command:
        Name: glueetl
        PythonVersion: 3
        ScriptLocation: !Sub
          - s3://${bucket_name}/${glue_dir}/${job_name}/glue_job.py
          - bucket_name: !Ref PreBucket
            glue_dir: !FindInMap [DirMap, glue, name]
            job_name: !FindInMap [TaskMap, featureEngineering, name]
      DefaultArguments:
        --enable-glue-datacatalog: ""
        --enable-metrics: ""
        --enable-spark-ui: true
        --enable-continuous-cloudwatch-log: true
        --enable-continuous-log-filter: true
        --job-language: python
        --extra-jars: !Sub
          - s3://${bucket_name}/${jar_dir}/spark-nlp-assembly-2.6.2.jar
          - bucket_name: !Ref PreBucket
            jar_dir: !FindInMap [DirMap, jars, name]
        --additional-python-modules: spark-nlp==2.6.2
        --input_db: default
        --output_db: training
        --model_bucket: !Ref PreBucket
      Description: Feature engineering for model training
      ExecutionProperty:
        MaxConcurrentRuns: 1
      GlueVersion: 2.0
      MaxRetries: 0
      Name: !FindInMap [TaskMap, featureEngineering, name]
      NumberOfWorkers: 4
      Role: !GetAtt GlueRoleS3.Arn
      Timeout: 480
      WorkerType: G.1X
    DependsOn:
      - GlueRoleS3
      - Bucket

  DataProcessingWorkflow:
    Type: AWS::Glue::Workflow
    Properties:
      Description: Parse XML and create tables
      Name: data-processing

  ParseXMLTrigger:
    Type: AWS::Glue::Trigger
    Properties:
      Actions:
        - JobName: !Ref GetFeaturesJob
        - JobName: !Ref FeatureEngineeringJob
      StartOnCreation: true
      Name: trigger
      Predicate:
        Conditions:
          - JobName: !Ref ParseXMLJob
            LogicalOperator: EQUALS
            State: SUCCEEDED
        Logical: AND
      Type: CONDITIONAL
      WorkflowName: !Ref DataProcessingWorkflow
    DependsOn:
      - GetFeaturesJob
      - ParseXMLJob
      - DataProcessingWorkflow

  ParseXMLStart:
    Type: AWS::Glue::Trigger
    Properties:
      Actions:
        - JobName: !FindInMap [TaskMap, parse, name]
      Name: start-workflow
      Type: ON_DEMAND
      WorkflowName: !Ref DataProcessingWorkflow
    DependsOn:
      - DataProcessingWorkflow

  SagemakerRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: 2012-10-17
        Statement:
          - Effect: Allow
            Principal:
              AWS:
                - !Sub arn:aws:iam::${AWS::AccountId}:root
            Action:
              - sts:AssumeRole
          - Effect: Allow
            Principal:
              Service:
                - sagemaker.amazonaws.com
            Action:
              - sts:AssumeRole
      ManagedPolicyArns:
        - arn:aws:iam::aws:policy/AmazonS3FullAccess
        - arn:aws:iam::aws:policy/AmazonSageMakerFullAccess
      RoleName: sagemaker-role

  LambdaRolePrediction:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: 2012-10-17
        Statement:
          - Effect: Allow
            Principal:
              Service:
                - lambda.amazonaws.com
            Action:
              - sts:AssumeRole
      ManagedPolicyArns:
        - arn:aws:iam::aws:policy/CloudWatchLogsFullAccess
        - arn:aws:iam::aws:policy/AmazonDynamoDBReadOnlyAccess
      Policies:
        - PolicyName: LambdaSagemakerInvoke
          PolicyDocument:
            Version: 2012-10-17
            Statement:
              - Effect: Allow
                Action: "sagemaker:InvokeEndpoint"
                Resource: "*"
      RoleName: lambda-predict

  TriggerPredictLambda:
    Type: AWS::Lambda::Function
    Properties:
      Code:
        S3Bucket: !Ref PreBucket
        S3Key: !Sub
          - ${lambda_dir}/${task_name}.zip
          - lambda_dir: !FindInMap [DirMap, lambda, name]
            task_name: !FindInMap [TaskMap, triggerPredict, name]
      Description: Run prediction from user request
      Environment:
        Variables:
          ENDPOINT_NAME: !FindInMap [TaskMap, predict, name]
          NLTK_DATA: nltk_data
      FunctionName: !FindInMap [TaskMap, triggerPredict, name]
      Handler: lambda_handler.lambda_handler
      MemorySize: 256
      PackageType: Zip
      Role: !GetAtt LambdaRolePrediction.Arn
      Runtime: python3.7
      Timeout: 30
    DependsOn:
      - LambdaRolePrediction

  TriggerPredictLambdaLogGroup:
    Type: AWS::Logs::LogGroup
    Properties:
      LogGroupName: !Sub
        - /aws/lambda/${lambdaFunctionName}
        - lambdaFunctionName: !Ref TriggerPredictLambda
      RetentionInDays: 30
    DependsOn:
      - TriggerPredictLambda

  ModelApi:
    Type: AWS::ApiGateway::RestApi
    Properties:
      EndpointConfiguration:
        Types:
          - REGIONAL
      Name: !Ref ProjectName

  ModelApiPost:
    Type: AWS::ApiGateway::Method
    Properties:
      AuthorizationType: NONE
      HttpMethod: POST
      Integration:
        IntegrationHttpMethod: POST
        Type: AWS_PROXY
        Uri: !Sub
          - arn:aws:apigateway:${AWS::Region}:lambda:path/2015-03-31/functions/${lambdaArn}/invocations
          - lambdaArn: !GetAtt TriggerPredictLambda.Arn
      ResourceId: !GetAtt ModelApi.RootResourceId
      RestApiId: !Ref ModelApi
    DependsOn:
      - ModelApi
      - TriggerPredictLambda

  ModelApiDeployment:
    Type: AWS::ApiGateway::Deployment
    Properties:
      RestApiId: !Ref ModelApi
      StageName: prod
    DependsOn:
      - ModelApiPost

  TriggerPredictLambdaPermission:
    Type: AWS::Lambda::Permission
    Properties:
      Action: lambda:InvokeFunction
      FunctionName: !GetAtt TriggerPredictLambda.Arn
      Principal: apigateway.amazonaws.com
      SourceArn: !Sub arn:aws:execute-api:${AWS::Region}:${AWS::AccountId}:${ModelApi}/prod/POST/
    DependsOn:
      - ModelApiDeployment

  UserTable:
    Type: AWS::DynamoDB::Table
    Properties:
      AttributeDefinitions:
        - AttributeName: "id"
          AttributeType: "S"
      BillingMode: PAY_PER_REQUEST
      KeySchema:
        - AttributeName: "id"
          KeyType: "HASH"
      TableName: users

  TagTable:
    Type: AWS::DynamoDB::Table
    Properties:
      AttributeDefinitions:
        - AttributeName: "id"
          AttributeType: "S"
      BillingMode: PAY_PER_REQUEST
      KeySchema:
        - AttributeName: "id"
          KeyType: "HASH"
      TableName: tags