AWSTemplateFormatVersion: '2010-09-09'
Description:  This template create an EMR cluster on Outposts

Parameters:
# -----------------------------------------------------------------------------------
# IAM Roles and policies parameters
#
  AmazonEMROutpostsClusterRole:
    Type: "String"
    Description: "The IAM role name that Amazon EMR assumes to provision resources and perform service-level actions with other AWS services."
    Default: "AmazonEMR-architecture-outposts-cluster-role"

  AmazonEMROutpostsEC2Policy:
    Type: "String"
    Description: "The IAM policy name that will be attached to the AmazonEMROutpostsClusterRole."
    Default: "AmazonEMR-architecture-outposts-EC2-policy"

  AmazonEMROutpostsEC2Role:
    Type: "String"
    Description: "The IAM role name to be associated to every EC2 instance in the EMR cluster."
    Default: "AmazonEMR-architecture-outposts-EC2-role"

  AmazonEMROutpostsClusterPolicy:
    Type: "String"
    Description: "The IAM policy name that will be attached to the AmazonEMROutpostsEC2Role."
    Default: "AmazonEMR-architecture-outposts-cluster-policy"

# -----------------------------------------------------------------------------------
# Instance type used by EMR Cluster
#
  InstanceType:
    Type: String
    Description: The instance type used by EMR cluster
    Default: "m5.xlarge"
# -----------------------------------------------------------------------------------
# EMR Release label
#
  ReleaseLabel:
    Type: String
    Description: The Amazon EMR release label, which determines the version of open-source application packages installed on the cluster.
    Default: "emr-7.1.0"
# -----------------------------------------------------------------------------------
# EC2 Key pair
#
  Ec2KeyName:
    Type: AWS::EC2::KeyPair::KeyName
    Description: Key pair to be used if you need to ssh to the EMR Primary node. You need to precreate this key.
    Default: "emr-outpost-master-kp"
# -----------------------------------------------------------------------------------
# Enable/Disable termination protection of EMR Cluster
#
  TerminationProtected:
    Type: String
    Default: false
    AllowedValues: [true, false]

# -----------------------------------------------------------------------------------
# EMR certificates for encryption in transit
#
  SSlCertsS3BucketName:
    Type: String
    Description: This should be an existing bucket in your account.
    Default: "certsbundlebucket-outpostblog-1"
    
  SSlCertsZipName:
    Type: String
    Description: The certificates zip should be uploaded to the bucket above before deploying the cloudformation stack.
    Default: "my-certs.zip"

Resources:

  # Lambda function to get the access point alias
  GetAccessPointAliasFunction:
    Type: AWS::Lambda::Function
    Metadata:
      cfn_nag:
        rules_to_suppress:
          - id: W89
            reason: "This function is used to get the Accesspoint alias for a s3 bucket and doesn't need to be deployed in a VPC"
          - id: W92
            reason: "This function is simple and doesn't need to use reserve simultaneous executions"
      checkov:
        skip:
          - id: CKV_AWS_115
            comment: "Function-level concurrent execution limit is not required as this function's usage is predictable and well-managed at the account level"
          - id: CKV_AWS_116
            comment: "Dead Letter Queue is not required for this function as it's designed to fail gracefully and retry is not needed"
          - id: CKV_AWS_117
            comment: "This function is used to get the Accesspoint alias for a s3 bucket and doesn't need to be deployed in a VPC"
    Properties:
      Handler: index.lambda_handler
      Role: !GetAtt LambdaExecutionRole.Arn
      Code:
        ZipFile: |
          import boto3
          import cfnresponse
          import json
          import logging


          logger = logging.getLogger()
          logger.setLevel(logging.INFO)

          def lambda_handler(event, context):
            logger.info(f"REQUEST RECEIVED:\n{json.dumps(event)}")

            UniqueId = event['RequestId']
            newPhysicalResourceId = 'LambdaGetAccessPointAlias-' + str(UniqueId)
            customPhysicalResourceId = event['PhysicalResourceId'] if 'PhysicalResourceId' in event else newPhysicalResourceId

            try:
              if event['RequestType'] in ['Create', 'Update']:
                s3control = boto3.client('s3control')
                response = s3control.get_access_point(
                  AccountId=event['ResourceProperties']['AccountId'],
                  Name=event['ResourceProperties']['AccessPointName']
                )
                alias = response['Alias']
                cfnresponse.send(event, context, cfnresponse.SUCCESS, {'Alias': alias}, customPhysicalResourceId)
              else:
                cfnresponse.send(event, context, cfnresponse.SUCCESS, {}, customPhysicalResourceId)
            except Exception as e:
              cfnresponse.send(event, context, cfnresponse.FAILED, {'Error: ': str(e)})

      Runtime: python3.12
      Timeout: 120

  # IAM role for the Lambda function
  LambdaExecutionRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: Allow
            Principal:
              Service: lambda.amazonaws.com
            Action: sts:AssumeRole
      ManagedPolicyArns:
        - 'arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole'
        - 'arn:aws:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole'
      Policies:
        - PolicyName: S3ControlAccess
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              - Effect: Allow
                Action:
                  - s3-outposts:GetAccessPoint
                Resource: !Sub 'arn:${AWS::Partition}:s3-outposts:${AWS::Region}:${AWS::AccountId}:outpost/*/accesspoint/*'

  # Custom resource to invoke the Lambda function
  GetAccessPointAliasResource:
    Type: Custom::GetAccessPointAlias
    Properties:
      ServiceToken: !GetAtt GetAccessPointAliasFunction.Arn
      AccountId: !Ref AWS::AccountId
      AccessPointName: !ImportValue S3OutpostsAccessPoint

  CopyDummyDataLambdaExecutionRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: Allow
            Principal:
              Service:
                - lambda.amazonaws.com
            Action:
              - sts:AssumeRole
      ManagedPolicyArns:
        - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole
      Policies:
        - PolicyName: S3Access
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              - Effect: Allow
                Action:
                  - s3:PutObject
                Resource: 
                  - !Sub
                    - 'arn:${AWS::Partition}:s3:::${bucketinregion}/*'
                    - bucketinregion: !ImportValue S3BucketBlogEMROnOutposts

  CopyDummyDataLambdaFunction:
    Type: AWS::Lambda::Function
    Metadata:
      cfn_nag:
        rules_to_suppress:
          - id: W89
            reason: "This function doesn't need to be deployed inside a VPC"
          - id: W92
            reason: "This function is simple and doesn't need to use reserve simultaneous executions"
      checkov:
        skip:
          - id: CKV_AWS_115
            comment: "Function-level concurrent execution limit is not required as this function's usage is predictable and well-managed at the account level"
          - id: CKV_AWS_116
            comment: "Dead Letter Queue is not required for this function as it's designed to fail gracefully and retry is not needed"
          - id: CKV_AWS_117
            comment: "This Lambda function does not require VPC access for its operation"
          - id: CKV_AWS_173
            comment: "The lambda parameter is not confidential. It is the bucket name used to download the bootstrap script"
    Properties:
      FunctionName: CopyFileToS3
      Handler: index.lambda_handler
      Role: !GetAtt CopyDummyDataLambdaExecutionRole.Arn
      Environment:
        Variables:
          DESTINATION_BUCKET_NAME: !ImportValue S3BucketBlogEMROnOutposts
      Code:
        ZipFile: |
          import json
          import boto3
          import urllib.request
          import os
          import cfnresponse
          

          def lambda_handler(event, context):

              UniqueId = event['RequestId']
              newPhysicalResourceId = 'LambdaCopyDummyData-' + str(UniqueId)
              customPhysicalResourceId = event['PhysicalResourceId'] if 'PhysicalResourceId' in event else newPhysicalResourceId

              try:
                # S3 bucket and file details
                source_files = [
                    {
                        "url": "https://aws-blogs-artifacts-public.s3.amazonaws.com/artifacts/BDB-4146/scripts/copy-dummy-data.sh",
                        "key": "scripts/copy-dummy-data.sh"
                    },
                    {
                        "url": "https://aws-blogs-artifacts-public.s3.amazonaws.com/artifacts/BDB-4146/scripts/BatchInsert.py",
                        "key": "scripts/BatchInsert.py"
                    }
                ]

                destination_bucket = os.environ['DESTINATION_BUCKET_NAME']

                if event['RequestType'] in ['Create', 'Update']:
                    s3 = boto3.client('s3')

                    for file in source_files:
                        # Download file from HTTPS endpoint
                        try:
                            with urllib.request.urlopen(file["url"]) as response:
                                content = response.read()
                                if response.getcode() != 200:
                                    cfnresponse.send(event, context, cfnresponse.FAILED, {'Error': f'Failed to download {file["url"]}'})
                                    return {
                                        'statusCode': response.getcode(),
                                        'body': json.dumps(f'Failed to download file from source URL: {file["url"]}')
                                    }
                        except urllib.error.URLError as e:
                            cfnresponse.send(event, context, cfnresponse.FAILED, {'Error': str(e)})
                            return {
                                'statusCode': 500,
                                'body': json.dumps(f'Failed to download file {file["url"]}: {str(e)}')
                            }

                        # Save file to S3
                        try:
                            s3.put_object(Bucket=destination_bucket, Key=file["key"], Body=content)
                        except Exception as e:
                            print(e)
                            cfnresponse.send(event, context, cfnresponse.FAILED, {'Error': f'Failed to copy {file["key"]} to S3'})
                            return {
                                'statusCode': 500,
                                'body': json.dumps(f'Failed to copy {file["key"]} to S3')
                            }

                    cfnresponse.send(event, context, cfnresponse.SUCCESS, {}, customPhysicalResourceId)
                    return {
                        'statusCode': 200,
                        'body': json.dumps('Files successfully copied to S3')
                    }
                else:
                    cfnresponse.send(event, context, cfnresponse.SUCCESS, {}, customPhysicalResourceId)
              except Exception as e:
                  cfnresponse.send(event, context, cfnresponse.FAILED, {'Error': str(e)})


      Runtime: python3.12
      Timeout: 30
      MemorySize: 128

  # Custom resource to invoke the Lambda function
  CopyDummyData:
    Type: Custom::CopyDummyData
    Properties:
      ServiceToken: !GetAtt CopyDummyDataLambdaFunction.Arn

# -----------------------------------------------------------------------------------
#  IAM roles and policies used to create the EMR Cluster
  # IAM policy that will be attached to the AmazonEMROutpostsClusterRole
  IAMAmazonEMROutpostsClusterPolicy:
    Type: 'AWS::IAM::Policy'
    Properties:
      PolicyName: !Ref AmazonEMROutpostsClusterPolicy
      PolicyDocument:
        Version: "2012-10-17"
        Statement:
          - Effect: Allow
            Sid: CreateInNetwork
            Action:
              - ec2:CreateNetworkInterface
              - ec2:RunInstances
              - ec2:CreateFleet
              - ec2:CreateLaunchTemplate
              - ec2:CreateLaunchTemplateVersion
              - ec2:DeleteNetworkInterface
              - ec2:TerminateInstances
            Resource:
              - !Sub 'arn:${AWS::Partition}:ec2:*:*:subnet/*'
              - !Sub 'arn:${AWS::Partition}:ec2:*:*:security-group/*'
              - !Sub 'arn:${AWS::Partition}:ec2:*:*:instance/*'
              - !Sub 'arn:${AWS::Partition}:ec2:*:*:volume/*'
              - !Sub 'arn:${AWS::Partition}:ec2:*:*:network-interface/*'
          - Effect: Allow
            Sid: ManageSecurityGroups
            Action:
              - ec2:AuthorizeSecurityGroupEgress
              - ec2:AuthorizeSecurityGroupIngress
              - ec2:RevokeSecurityGroupEgress
              - ec2:RevokeSecurityGroupIngress
            Resource:
              - !Sub 'arn:${AWS::Partition}:ec2:*:*:security-group/*'
          - Effect: Allow
            Sid: CreateDefaultSecurityGroupInVPC
            Action:
              - ec2:CreateSecurityGroup
            Resource:
              - !Sub 'arn:${AWS::Partition}:ec2:*:*:vpc/*'
          - Effect: Allow
            Sid: PassRoleForEC2
            Action:
              - iam:PassRole
            Resource:
              - !Sub 'arn:${AWS::Partition}:iam::*:role/AmazonEMR-architecture-outposts-EC2-role'
            Condition:
              StringLike:
                "iam:PassedToService": "ec2.amazonaws.com"
      Roles:
        - !Ref IAMAmazonEMROutpostsClusterRole

  # IAM role that Amazon EMR assumes to provision resources and perform service-level actions with other AWS services.
  IAMAmazonEMROutpostsClusterRole:
    Type: 'AWS::IAM::Role'
    Metadata:
      cfn_nag:
        rules_to_suppress:
          - id: W28
            reason: "The rolename will be used to create the EMR Cluster"
    Properties:
      AssumeRolePolicyDocument:
        Version: "2012-10-17"
        Statement:
          - Effect: Allow
            Principal:
              Service:
                - elasticmapreduce.amazonaws.com
            Action:
              - sts:AssumeRole
      ManagedPolicyArns:
        - !Sub 'arn:${AWS::Partition}:iam::aws:policy/service-role/AmazonEMRServicePolicy_v2'
        - !Sub 'arn:${AWS::Partition}:iam::aws:policy/SecretsManagerReadWrite'
        - !Sub 'arn:${AWS::Partition}:iam::aws:policy/AmazonS3OutpostsFullAccess'
          
      Path: /
      RoleName: !Ref AmazonEMROutpostsClusterRole

  # IAM policy name that will be attached to the AmazonEMROutpostsEC2Role
  IAMAmazonEMROutpostsEC2Policy:
    Type: 'AWS::IAM::Policy'
    Properties:
      PolicyName: !Ref AmazonEMROutpostsEC2Policy
      PolicyDocument:
        Version: "2012-10-17"
        Statement:
          - Effect: Allow
            Action:
              - s3:AbortMultipartUpload
              - s3:CreateBucket
              - s3:DeleteObject
              - s3:ListBucket
              - s3:ListBucketMultipartUploads
              - s3:ListBucketVersions
              - s3:ListMultipartUploadParts
              - s3:PutBucketVersioning
              - s3:PutObject
              - s3:PutObjectTagging
            Resource:
              - !Join ['',[!Sub 'arn:${AWS::Partition}:s3:::', !ImportValue S3BucketBlogEMROnOutposts, '/logs']]
              - !Join ['',[!Sub 'arn:${AWS::Partition}:s3:::', !ImportValue S3BucketBlogEMROnOutposts, '/logs/*']]
          - Effect: Allow
            Action:
              - s3:GetBucketVersioning
              - s3:GetObject
              - s3:GetObjectTagging
              - s3:GetObjectVersion
              - s3:ListBucket
              - s3:ListBucketMultipartUploads
              - s3:ListBucketVersions
              - s3:ListMultipartUploadParts
            Resource:
              - !Sub 'arn:${AWS::Partition}:s3:::elasticmapreduce'
              - !Sub 'arn:${AWS::Partition}:s3:::elasticmapreduce/*'
              - !Sub 'arn:${AWS::Partition}:s3:::*.elasticmapreduce/*'
              - !Join ['',[!Sub 'arn:${AWS::Partition}:s3:::', !ImportValue S3BucketBlogEMROnOutposts, '/*']]
              - !Join ['',[!Sub 'arn:${AWS::Partition}:s3:::', !ImportValue S3BucketBlogEMROnOutposts]]
          - Effect: Allow
            Action:
              - s3-outposts:PutObject
              - s3-outposts:GetObject
              - s3-outposts:ListBucket
              - s3-outposts:DeleteObject
              - s3:ListBucket
              - s3:ListBucketMultipartUploads
              - s3:ListBucketVersions
              - s3:ListMultipartUploadParts
              - s3:PutBucketVersioning
              - s3:PutObject
              - s3:PutObjectTagging
            Resource:
              - !Sub 'arn:${AWS::Partition}:s3:::elasticmapreduce'
              - !Sub 'arn:${AWS::Partition}:s3:::elasticmapreduce/*'
              - !Sub 'arn:${AWS::Partition}:s3:::*.elasticmapreduce/*'
              - !Join ['',[!Sub 'arn:${AWS::Partition}:s3:::', !ImportValue S3BucketBlogEMROnOutposts, '/*']]
              - !Join ['',[!Sub 'arn:${AWS::Partition}:s3:::', !ImportValue S3BucketBlogEMROnOutposts]]
              - !ImportValue S3OutpostsBucket
              - !ImportValue S3OutpostsAccessPoint
              - !Sub
                - ${S3OutpostsBuck}/*
                - S3OutpostsBuck: !ImportValue S3OutpostsBucket
              - !Sub
                - ${S3AP}/*
                - S3AP: !ImportValue S3OutpostsAccessPoint
      Roles:
        - !Ref IAMAmazonEMROutpostsEC2Role

  # The IAM role to be associated to every EC2 instance in the EMR cluster
  IAMAmazonEMROutpostsEC2Role:
    Type: 'AWS::IAM::Role'
    Metadata:
      cfn_nag:
        rules_to_suppress:
          - id: W28
            reason: "The rolename will be used to create the EMR Cluster"
    Properties:
      RoleName: !Ref AmazonEMROutpostsEC2Role
      Path: /
      AssumeRolePolicyDocument:
        Version: "2012-10-17"
        Statement:
          - Effect: Allow
            Principal:
              Service:
                - ec2.amazonaws.com
            Action:
              - sts:AssumeRole
      ManagedPolicyArns:
        - arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore
        - arn:aws:iam::aws:policy/service-role/AmazonElasticMapReduceforEC2Role
      Policies:
        - PolicyName: RuntimeRolePolicy
          PolicyDocument:
            Version: "2012-10-17"
            Statement:
              - Effect: Allow
                Action:
                  - "sts:AssumeRole"
                  - "sts:TagSession"
                  - "sts:SetSourceIdentity"
                Resource:
                  - !Sub 'arn:${AWS::Partition}:iam::${AWS::AccountId}:role/outpostblog-runtimeRole1'
  # Runtime role
  RuntimeRole1:
    Type: AWS::IAM::Role
    Metadata:
      cfn_nag:
        rules_to_suppress:
          - id: W28
            reason: "The rolename will be used to create the EMR Cluster"
          - id: W11
            reason: "AWS Lake Formation does not support specifying a resource ARN in the Resource element of an IAM policy statement. To allow access to AWS Lake Formation, specify \"Resource\": \"*\" in your policy. Link: https://docs.aws.amazon.com/service-authorization/latest/reference/list_awslakeformation.html"
          - id: F3
            reason: "AWS Lake Formation does not support specifying a resource ARN in the Resource element of an IAM policy statement. To allow access to AWS Lake Formation, specify \"Resource\": \"*\" in your policy. Link: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
    Properties:
      RoleName: "outpostblog-runtimeRole1"
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: Allow
            Principal:
              AWS: !GetAtt IAMAmazonEMROutpostsEC2Role.Arn
            Action:
              - "sts:AssumeRole"
              - "sts:SetSourceIdentity"
      Policies:
        - PolicyName: "LakeFormationGluePolicy"
          PolicyDocument:
            Version: "2012-10-17"
            Statement:
              - Sid: "LakeFormationManagedAccess"
                Effect: "Allow"
                Action:
                  - "lakeformation:GetDataAccess"
                  - "glue:Get*"
                  - "glue:Create*"
                  - "glue:Update*"
                Resource:
                  - !Sub 'arn:${AWS::Partition}:glue:*:*:database/oktank_outpostblog_temp/*'
                  - !Sub 'arn:${AWS::Partition}:glue:*:*:database/oktank_outpostblog_temp'
                  - !Sub 'arn:${AWS::Partition}:glue:*:*:table/oktank_outpostblog_temp/*'
                  - !Sub 'arn:${AWS::Partition}:glue:*:*:database/default'
                  - !Sub 'arn:${AWS::Partition}:glue:*:*:catalog/*'
                  - !Sub 'arn:${AWS::Partition}:glue:*:*:catalog'
        - PolicyName: "LocalS3OutpostBucketAccess"
          PolicyDocument:
            Version: "2012-10-17"
            Statement:
              - Sid: "OutpostBucketAccess"
                Effect: "Allow"
                Action:
                  - "s3-outposts:*"
                Resource: 
                  - !ImportValue S3OutpostsBucket
                  - !ImportValue S3OutpostsAccessPoint
                  - !Sub
                    - ${S3OutpostsBuck}/*
                    - S3OutpostsBuck: !ImportValue S3OutpostsBucket
                  - !Sub
                    - ${S3AP}/*
                    - S3AP: !ImportValue S3OutpostsAccessPoint
        - PolicyName: "RegionaS3BucketAccess"
          PolicyDocument:
            Version: "2012-10-17"
            Statement:
              - Sid: "BucketAccess"
                Effect: "Allow"
                Action:
                  - "s3:ListBucket"
                  - "s3:GetObject"                 
                Resource: 
                  - !Sub
                    - arn:${AWS::Partition}:s3:::${ImportedS3BucketName}/*
                    - ImportedS3BucketName: !ImportValue S3BucketBlogEMROnOutposts 
                  - !Sub
                    - arn:${AWS::Partition}:s3:::${ImportedS3BucketName}
                    - ImportedS3BucketName: !ImportValue S3BucketBlogEMROnOutposts
    # The IAM instance profile role to be associated to every EC2 instance in the EMR cluster
  IAMAmazonEMROutpostsEC2RoleInstanceProfile:
    Type: 'AWS::IAM::InstanceProfile'
    Properties:
      InstanceProfileName: !Ref AmazonEMROutpostsEC2Role
      Path: /
      Roles:
        - !Ref IAMAmazonEMROutpostsEC2Role

  EMRClusterSecurityConfiguration:
    Type: AWS::EMR::SecurityConfiguration
    Properties:
      Name: EMRClusterSecurityConfiguration
      SecurityConfiguration:
        EncryptionConfiguration:
          EnableInTransitEncryption: true
          EnableAtRestEncryption: true
          InTransitEncryptionConfiguration:
            TLSCertificateConfiguration:
              CertificateProviderType: "PEM"
              S3Object: !Sub "s3://${SSlCertsS3BucketName}/${SSlCertsZipName}"
          AtRestEncryptionConfiguration:
            S3EncryptionConfiguration:
              EncryptionMode: "SSE-S3"
        AuthorizationConfiguration:
          IAMConfiguration:
            EnableApplicationScopedIAMRole: true
            ApplicationScopedIAMRoleConfiguration:
              PropagateSourceIdentity: true
 
  EMRCluster:
    DependsOn:
         - CopyDummyDataLambdaFunction
    Type: AWS::EMR::Cluster
    DeletionPolicy: Delete
    UpdateReplacePolicy: Delete
    Properties:
      EbsRootVolumeSize: 20
      SecurityConfiguration: !Ref EMRClusterSecurityConfiguration
      Configurations:
        - Classification: core-site
          ConfigurationProperties:
            fs.AbstractFileSystem.s3.impl: org.apache.hadoop.fs.s3a.S3A
            fs.s3.impl: org.apache.hadoop.fs.s3a.S3AFileSystem
            fs.s3a.aws.credentials.provider: software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider
            fs.s3a.bucket.outpostsbucketa.accesspoint.arn: !ImportValue S3OutpostsAccessPoint
            fs.s3a.change.detection.mode: none
            fs.s3a.committer.name: magic
            fs.s3a.endpoint: !Sub 
              - "s3.${awsregion}.amazonaws.com"
              - awsregion: !Ref "AWS::Region"
            fs.s3a.endpoint.region: !Ref "AWS::Region"
            fs.s3a.select.enabled: false
        - Classification: hadoop-env
          ConfigurationProperties:
            Classification: export
            Properties: "JAVA_HOME: /usr/lib/jvm/java-17-amazon-corretto.x86_64"
        - Classification: spark-env
          ConfigurationProperties:
            Classification: export
            Properties: "JAVA_HOME: /usr/lib/jvm/java-17-amazon-corretto.x86_64"
        - Classification: spark-defaults
          ConfigurationProperties:
            spark.executorEnv.JAVA_HOME: /usr/lib/jvm/java-17-amazon-corretto.x86_64
            spark.sql.sources.fastS3PartitionDiscovery.enabled: false
        - Classification: hive-site
          ConfigurationProperties:
            hive.metastore.client.factory.class: com.amazonaws.glue.catalog.metastore.AWSGlueDataCatalogHiveClientFactory
        - Classification: spark-hive-site
          ConfigurationProperties:
            hive.metastore.client.factory.class: com.amazonaws.glue.catalog.metastore.AWSGlueDataCatalogHiveClientFactory
      Instances:
        AdditionalMasterSecurityGroups:
          - !Ref 'EMRStudioEngineSecurityGroup'
        Ec2KeyName: !Ref Ec2KeyName
        MasterInstanceGroup:
          Name: cfnMaster
          InstanceCount: 1
          InstanceType: !Ref InstanceType
          EbsConfiguration:
            EbsOptimized: true
            EbsBlockDeviceConfigs:
                 - VolumeSpecification:
                    SizeInGB: 100
                    VolumeType: gp2
        CoreInstanceGroup:
          Name: cfnCore
          InstanceCount: 2
          InstanceType: !Ref InstanceType
          EbsConfiguration:
            EbsOptimized: true
            EbsBlockDeviceConfigs:
                 - VolumeSpecification:
                    SizeInGB: 100
                    VolumeType: gp2  
        TerminationProtected: !Ref TerminationProtected
        Ec2SubnetIds:
          - !ImportValue 'OutpostsPrivateSubnet'
      LogUri: !Sub 
        - "s3://${bucketname}/logs"
        - bucketname: !ImportValue S3BucketBlogEMROnOutposts
      Name: EMROutpostBlog
      JobFlowRole: !Ref IAMAmazonEMROutpostsEC2RoleInstanceProfile
      ServiceRole: !Ref IAMAmazonEMROutpostsClusterRole
      ReleaseLabel: !Ref ReleaseLabel
      VisibleToAllUsers: true
      BootstrapActions:
        - Name: DownloadAndCopyFile
          ScriptBootstrapAction:
            Path: !Sub 
                    - "s3://${bucketname}/scripts/copy-dummy-data.sh"
                    - bucketname: !ImportValue S3BucketBlogEMROnOutposts
            Args: 
              - !GetAtt GetAccessPointAliasResource.Alias
              - !ImportValue S3BucketBlogEMROnOutposts
      Applications:
        - Name: Spark
        - Name: Hadoop
        - Name: Livy
        - Name: JupyterEnterpriseGateway
      Tags:
        - Key: "for-use-with-amazon-emr-managed-policies"
          Value: "true"
# -----------------------------------------------------------------------------------
#  Glue  set ups
#
  ##Glue Database
  GlueDatabase:
    Type: 'AWS::Glue::Database'
    Properties:
      DatabaseInput:
        Name: 'oktank_outpostblog_temp'
      CatalogId: !Sub ${AWS::AccountId}
      
  ## StockHolding Table - Confidential
  GlueTableCustomerStockHolding:
    Type: 'AWS::Glue::Table'
    DependsOn: GlueDatabase
    Properties:
      CatalogId: !Sub ${AWS::AccountId}
      DatabaseName: 'oktank_outpostblog_temp'
      TableInput:
        Name: 'stockholdings_info'
        TableType: EXTERNAL_TABLE
        Parameters: {
          "EXTERNAL": "TRUE",
          'skip.header.line.count': '1'
        }
        
        StorageDescriptor:
          Columns:
            - Name: CustomerID
              Type: bigint
            - Name: CustomerName
              Type: string
            - Name: AccountNumber
              Type: string
            - Name: StockID
              Type: string
            - Name: SharesHeld
              Type: bigint
            - Name: PurchaseDate
              Type: date
            # Add more columns as needed
          Location: !Sub 
            - s3a://${ImportedS3OutpostBucketName}/customerholdings/
            - ImportedS3OutpostBucketName: !GetAtt GetAccessPointAliasResource.Alias
          InputFormat: 'org.apache.hadoop.mapred.TextInputFormat'
          OutputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'
          SerdeInfo:
            SerializationLibrary: 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
            Parameters:
              'field.delim': ',' 

  ## StockHolding Detailed Table - Confidential
  GlueTableCustomerStockHoldingDetailed:
    Type: 'AWS::Glue::Table'
    DependsOn: GlueDatabase
    Properties:
      CatalogId: !Sub ${AWS::AccountId}
      DatabaseName: 'oktank_outpostblog_temp'
      TableInput:
        Name: 'stockholdings_info_detailed'
        TableType: EXTERNAL_TABLE
        Parameters: {
          "EXTERNAL": "TRUE",
          
        }
        
        StorageDescriptor:
          Location: !Sub 
            - s3a://${ImportedS3OutpostBucketName}/customerholdingsdetailed/
            - ImportedS3OutpostBucketName: !GetAtt GetAccessPointAliasResource.Alias
          InputFormat: "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat"
          OutputFormat: "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat"
          SerdeInfo:
            SerializationLibrary: "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe"
          Columns:
            - Name: customerid
              Type: string
            - Name: sharesheld
              Type: int
            - Name: purchasedate
              Type: date
            - Name: stockid
              Type: string
            - Name: stockname
              Type: string
            - Name: category
              Type: string
            - Name: currentprice
              Type: string

  ## Stock Details Table - Non-Confidential/Public data. This bucket wil be managed by Lake Formation.
  GlueTableStockDetails:
    Type: 'AWS::Glue::Table'
    DependsOn: GlueDatabase
    Properties:
      CatalogId: !Sub ${AWS::AccountId}
      DatabaseName: 'oktank_outpostblog_temp'
      TableInput:
        Name: 'stock_details'
        TableType: EXTERNAL_TABLE
        Parameters: {
          "EXTERNAL": "TRUE",
          'skip.header.line.count': '1' 

        }
        
        StorageDescriptor:
          Columns:
            - Name: StockID
              Type: string
            - Name: StockName
              Type: string
            - Name: Category
              Type: string
            - Name: CurrentPrice
              Type: string
            - Name: MarketCapInMillions
              Type: bigint
            # Add more columns as needed
          Location: !Sub
            - s3://${ImportedS3BucketName}/stockdetails/
            - ImportedS3BucketName: !ImportValue S3BucketBlogEMROnOutposts
          InputFormat: 'org.apache.hadoop.mapred.TextInputFormat'
          OutputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'
          SerdeInfo:
            SerializationLibrary: 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
            Parameters:
              'field.delim': ',' 

  ### Security Groups for Studios
  # Create Engine and Workspace Security groups
  # https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-studio-security-groups.html
  EMRStudioEngineSecurityGroup:
    Type: AWS::EC2::SecurityGroup
    Metadata:
      cfn_nag:
        rules_to_suppress:
          - id: W5
            reason: "Allow traffic to the internet to link publicly git repositories to Workspaces. https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-studio-security-groups.html"
    Properties:
      GroupDescription: "EMR Studio Engine Security Group"
      VpcId: !ImportValue VPCID
      SecurityGroupEgress:
        - IpProtocol: -1
          CidrIp: 127.0.0.1/32
          Description: "Explicitly deny all outbound traffic except to localhost"
      Tags:
        - Key: Name
          Value: EMRTIP-Engine-SG
        - Key: for-use-with-amazon-emr-managed-policies
          Value: 'true'

  # Handle cyclic dependency with engine and workspace sg
  EngineSGIngressFromWorkspaceSG:
    Type: 'AWS::EC2::SecurityGroupIngress'
    Properties:
      FromPort: 18888
      ToPort: 18888
      GroupId: !GetAtt EMRStudioEngineSecurityGroup.GroupId
      SourceSecurityGroupId: !GetAtt EMRStudioWorkspaceSecurityGroup.GroupId
      IpProtocol: 'tcp'
      Description: "Allow inbound traffic from EMR Studio Workspace to Engine on port 18888"

  EMRStudioWorkspaceSecurityGroup:
    Type: AWS::EC2::SecurityGroup
    Metadata:
      cfn_nag:
        rules_to_suppress:
          - id: W5
            reason: "Allow traffic to the internet to link publicly git repositories to Workspaces. https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-studio-security-groups.html"
    Properties:
      GroupDescription: "EMR Studio Workspace Security Group"
      VpcId: !ImportValue VPCID
      SecurityGroupEgress:
      - IpProtocol: tcp
        FromPort: 443
        ToPort: 443
        CidrIp: 0.0.0.0/0
        Description: "Allow outbound HTTPS traffic to the internet"
      - IpProtocol: tcp
        FromPort: 18888
        ToPort: 18888
        DestinationSecurityGroupId:
          Fn::GetAtt:
          - EMRStudioEngineSecurityGroup
          - GroupId
        Description: "Allow outbound traffic to EMR Studio Engine on port 18888"
      Tags:
      - Key: Name
        Value: EMRTIP-Workspace-SG
      - Key: for-use-with-amazon-emr-managed-policies
        Value: 'true'

Outputs:
  EMREC2Role:
    Description: The IAM role name to attach to the EMR Nodes
    Value: !Ref IAMAmazonEMROutpostsEC2Role
    Export:
      Name: "IAMAmazonEMROutpostsEC2Role"  

  EMRStudioEngineSecurityGroupO:
    Description: EMR Studio Engine Security Group
    Value: !Ref EMRStudioEngineSecurityGroup
    Export:
      Name: "EMREngineSecurityGroup" 

  EMRStudioWorkspaceSecurityGroupO:
    Description: EMR Studio Workspace Security Group
    Value: !Ref EMRStudioWorkspaceSecurityGroup
    Export:
      Name: "EMRWorkspaceSecurityGroup" 

  S3OutpostBucketAccessPointAlias1:
    Description: Access point Alias for S3 on Outpost Bucket
    Value: !GetAtt GetAccessPointAliasResource.Alias
    Export:
      Name: "S3OutpostBucketAccessPointAlias"
      
