AWSTemplateFormatVersion: 2010-09-09
Description: >
  Example for AWS Blog post "Monitor Apache Spark applications on Amazon EMR
  with Amazon Cloudwatch".

  WARNING This template creates IAM role, IAM instance profile, Cloudwatch
  Dashboard and EMR cluster.  You will  be billed for the AWS resources used if
  you create a stack from this template.
Parameters:
  InstanceType:
    Type: String
    Default: m5.2xlarge
    Description: >-
      Type of instances for all instance groups
      (https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-supported-instance-types.html)
  InstanceCountCore:
    Type: Number
    Default: 4
    MinValue: 1
    MaxValue: 4
    Description: Number of instances in core instance group
  EMRReleaseLabel:
    Type: String
    Default: emr-6.9.0
    Description: >-
      EMR release label
      (https://docs.aws.amazon.com/emr/latest/ReleaseGuide/emr-release-components.html)
  BootstrapScriptPath:
    Type: String
    Description: S3 path of CloudWatch Agent installation bootstrap script
    AllowedPattern: '^s3://([^/]+)/(.*?([^/]+))$'
    ConstraintDescription: 'BootstrapScriptPath must match pattern s3://bucket/folder/.../file'
    Default: 's3://aws-blogs-artifacts-public/artifacts/BDB-3297/installer.sh'
  MetricFilterPath:
    Type: String
    Description: S3 path of Metricfilter.json file
    AllowedPattern: '^s3://([^/]+)/(.*?([^/]+))$'
    ConstraintDescription: 'MetricFilterPath must match pattern s3://bucket/folder/.../file'
    Default: 's3://aws-blogs-artifacts-public/artifacts/BDB-3297/Metricfilter.json'
  MetricsLibraryPath:
    Type: String
    Description: S3 path of spark-emr-cloudwatch-sink-0.0.1.jar file matching the EMR Release
    AllowedPattern: '^s3://([^/]+)/(.*?([^/]+))$'
    ConstraintDescription: 'MetricsLibraryPath must match pattern s3://bucket/folder/.../file'
    Default: >-
      s3://aws-blogs-artifacts-public/artifacts/BDB-3297/emr-6.x.x/spark-emr-cloudwatch-sink-0.0.1.jar
  CloudWatchNamespace:
    Type: String
    Description: The name of the Cloudwatch Namespace where the metrics should be published
    AllowedPattern: '[^:].*'
    ConstraintDescription: CloudWatchNamespace must have a minimum length of 1. Maximum length of 255
    Default: EMRCustomSparkCloudWatchSink
  SparkDemoApplicationPath:
    Type: String
    Description: S3 path of spark application file
    AllowedPattern: '^s3://([^/]+)/(.*?([^/]+))$'
    ConstraintDescription: 'SparkDemoApplicationPath must match pattern s3://bucket/folder/.../file'
    Default: 's3://aws-blogs-artifacts-public/artifacts/BDB-3297/exampleJob.sh'
  Subnet:
    Type: 'AWS::EC2::Subnet::Id'
    Description: EC2 subnet to launch cluster into
  EC2KeyPairName:
    Type: String
    Description: >-
      Optional EC2 keypair for connecting to cluster nodes, as an alternative to
      Session Manager
Conditions:
  UseKeyPair: !Not 
    - !Equals 
      - !Ref EC2KeyPairName
      - ''
Resources:
  DemoEMRInstanceProfile:
    Type: 'AWS::IAM::InstanceProfile'
    Properties:
      Roles:
        - !Ref DemoEMRRole
  DemoEMRRole:
    Type: 'AWS::IAM::Role'
    Properties:
      AssumeRolePolicyDocument:
        Version: 2012-10-17
        Statement:
          - Effect: Allow
            Principal:
              Service:
                - ec2.amazonaws.com
            Action:
              - 'sts:AssumeRole'
      Policies:
        - PolicyName: DemoAmazonElasticMapReduceforEC2Role
          PolicyDocument:
            Version: "2012-10-17"
            Statement:
              - Effect: Allow
                Resource: "*"
                Action:
                  - cloudwatch:PutDashboard
                  - cloudwatch:PutMetricData
                  - cloudwatch:GetDashboard
                  - cloudwatch:GetMetricData
                  - cloudwatch:ListDashboards
                  - cloudwatch:ListMetrics
                  - elasticmapreduce:DescribeNotebookExecution
                  - elasticmapreduce:DescribeStudio
                  - elasticmapreduce:DescribeCluster
                  - elasticmapreduce:DescribeStep
                  - elasticmapreduce:DescribePersistentAppUI
                  - elasticmapreduce:DescribeEditor
                  - elasticmapreduce:DescribeSecurityConfiguration
                  - elasticmapreduce:DescribeReleaseLabel
                  - elasticmapreduce:DescribeRepository
                  - elasticmapreduce:DescribeJobFlows
                  - elasticmapreduce:ListBootstrapActions
                  - elasticmapreduce:ListClusters
                  - elasticmapreduce:ListInstanceGroups
                  - elasticmapreduce:ListInstances
                  - elasticmapreduce:ListSteps
                  - s3:GetBucketVersioning
                  - s3:GetObject
                  - s3:GetObjectTagging
                  - s3:GetObjectVersion
                  - s3:ListBucket
                  - s3:ListBucketMultipartUploads
                  - s3:ListBucketVersions
                  - s3:ListMultipartUploadPart
  DemoEMR:
    Type: 'AWS::EMR::Cluster'
    Properties:
      Name: Demo EMR Spark CloudWatch Sink
      ReleaseLabel: !Ref EMRReleaseLabel
      Applications:
        - Name: spark
      Instances:
        CoreInstanceGroup:
          InstanceCount: !Ref InstanceCountCore
          InstanceType: !Ref InstanceType
          Name: Core Group
        Ec2SubnetId: !Ref Subnet
        Ec2KeyName: !If 
          - UseKeyPair
          - !Ref EC2KeyPairName
          - !Ref 'AWS::NoValue'
        KeepJobFlowAliveWhenNoSteps: true
        MasterInstanceGroup:
          InstanceCount: 1
          InstanceType: !Ref InstanceType
          Name: Management Group
        TerminationProtected: false
      JobFlowRole: !Ref DemoEMRInstanceProfile
      ServiceRole: EMR_DefaultRole
      VisibleToAllUsers: true
      Tags:
        - Key: DemoEMRSparkCloudWatchSink
          Value: ''
      BootstrapActions:
        - Name: Install EMRCustomSparkCloudWatchSink
          ScriptBootstrapAction:
            Args:
              - !Ref MetricsLibraryPath
              - !Ref MetricFilterPath
              - !Ref CloudWatchNamespace
            Path: !Ref BootstrapScriptPath
      Configurations:
        - Classification: spark-metrics
          ConfigurationProperties:
            '*.sink.statsd.class': org.apache.spark.amazonaws.emr.metrics.sink.SparkCWStatsdSink
            '*.sink.statsd.prefix': spark
            '*.source.jvm.class': org.apache.spark.metrics.source.JvmSource
            driver.sink.statsd.tags: driver
            executor.sink.statsd.tags: executor
        - Classification: spark-defaults
          ConfigurationProperties:
            spark.metrics.appStatusSource.enabled: 'true'
            spark.metrics.staticSources.enabled: 'true'
  SparkJobStartStep:
    DependsOn: DemoEMR
    Type: 'AWS::EMR::Step'
    Properties:
      Name: DemoJob
      ActionOnFailure: CONTINUE
      HadoopJarStep:
        Jar: !Join
          - ''
          - - 's3://'
            - !Ref 'AWS::Region'
            - '.elasticmapreduce/libs/script-runner/script-runner.jar'
        Args:
          - !Ref SparkDemoApplicationPath
      JobFlowId: !Ref DemoEMR
Outputs:
  EMRClusterOutput:
    Description: The EMR Cluster
    Value: !Ref DemoEMR
