イベント駆動型のAWS Glueクローラー: SQSベースのトリガーを設定する

1
AWSTemplateFormatVersion: '2010-09-09'
2
Description: Glue crawler test
3

4
Resources:
5
  SqsQueue:
6
    Type: AWS::SQS::Queue
7
    Properties:
8
      SqsManagedSseEnabled: true
9
      QueueName: glue-crawler-test-queue
10

11
  SqsQueuePolicy:
12
    Type: AWS::SQS::QueuePolicy
13
    Properties:
14
      Queues:
15
        - !Ref SqsQueue
16
      PolicyDocument:
17
        Version: '2008-10-17'
18
        Id: __default_policy_ID
19
        Statement:
20
          - Effect: Allow
21
            Principal:
22
              AWS:
23
                - !Sub arn:aws:iam::${AWS::AccountId}:root
24
                - !GetAtt IAMRoleGlueCrawler.Arn
25
            Action: sqs:*
26
            Resource: !GetAtt SqsQueue.Arn
27
          - Effect: Allow
28
            Principal:
29
              Service: s3.amazonaws.com
30
            Action: sqs:*
31
            Resource: !GetAtt SqsQueue.Arn
32

33
  S3Bucket:
34
    Type: AWS::S3::Bucket
35
    DependsOn: SqsQueuePolicy
36
    Properties:
37
      BucketName: !Sub glue-crawler-test-${AWS::AccountId}-${AWS::Region}
38
      BucketEncryption:
39
        ServerSideEncryptionConfiguration:
40
          - ServerSideEncryptionByDefault:
41
              SSEAlgorithm: AES256
42
      NotificationConfiguration:
43
        QueueConfigurations:
44
          - Event: 's3:ObjectCreated:*'
45
            Queue: !GetAtt SqsQueue.Arn
46
      PublicAccessBlockConfiguration:
47
        BlockPublicAcls: TRUE
48
        BlockPublicPolicy: TRUE
49
        IgnorePublicAcls: TRUE
50
        RestrictPublicBuckets: TRUE
51

52
  IAMRoleGlueCrawler:
53
    Type: AWS::IAM::Role
54
    Properties:
55
      Path: /service-role/
56
      RoleName: !Sub glue-crawler-test-service-role
57
      AssumeRolePolicyDocument:
58
        Version: '2012-10-17'
59
        Statement:
60
          - Effect: Allow
61
            Principal:
62
              Service: glue.amazonaws.com
63
            Action: sts:AssumeRole
64
      Policies:
65
        - PolicyName: cw-logs
66
          PolicyDocument:
67
            Version: '2012-10-17'
68
            Statement:
69
              - Effect: Allow
70
                Action:
71
                  - logs:CreateLogGroup
72
                  - logs:CreateLogStream
73
                  - logs:PutLogEvents
74
                Resource: "*"
75
        - PolicyName: glue
76
          PolicyDocument:
77
            Version: '2012-10-17'
78
            Statement:
79
              - Effect: Allow
80
                Action:
81
                  - glue:CreateTable
82
                  - glue:GetDatabase
83
                  - glue:GetTable
84
                  - glue:UpdateTable
85
                Resource: "*"
86
        - PolicyName: s3
87
          PolicyDocument:
88
            Version: '2012-10-17'
89
            Statement:
90
              - Effect: Allow
91
                Action:
92
                  - s3:GetObject
93
                  - s3:ListBucket
94
                  - s3:PutObject
95
                Resource: '*'
96

97
  GlueDatabase:
98
    Type: AWS::Glue::Database
99
    Properties:
100
      CatalogId: !Ref AWS::AccountId
101
      DatabaseInput:
102
        Name: glue-crawler-test-db
103

104
  GlueTable:
105
    Type: AWS::Glue::Table
106
    Properties:
107
      CatalogId: !Ref AWS::AccountId
108
      DatabaseName: !Ref GlueDatabase
109
      TableInput:
110
        Name: glue-crawler-test-table
111
        TableType: EXTERNAL_TABLE
112
        Parameters:
113
          classification: json
114
        StorageDescriptor:
115
          Location: !Sub 's3://${S3Bucket}/'
116
          Compressed: false
117
          InputFormat: org.apache.hadoop.mapred.TextInputFormat
118
          OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
119
          SerdeInfo:
120
            SerializationLibrary: org.openx.data.jsonserde.JsonSerDe
121

122
  GlueCrawler:
123
    Type: AWS::Glue::Crawler
124
    Properties:
125
      Name: glue-crawler-test
126
      Role: !Sub service-role/${IAMRoleGlueCrawler}
127
      Targets:
128
        CatalogTargets:
129
          - DatabaseName: !Ref GlueDatabase
130
            Tables:
131
              - !Ref GlueTable
132
      SchemaChangePolicy:
133
        UpdateBehavior: UPDATE_IN_DATABASE
134
        DeleteBehavior: LOG

1
[
2
  {
3
    "Name": "message",
4
    "Type": "string"
5
  }
6
]

1
[
2
  {
3
    "Name": "message",
4
    "Type": "string"
5
  },
6
  {
7
    "Name": "statuscode",
8
    "Type": "int"
9
  }
10
]

1
{
2
    "Attributes": {
3
        "ApproximateNumberOfMessages": "0"
4
    }
5
}