-
Notifications
You must be signed in to change notification settings - Fork 0
/
template.yaml
180 lines (167 loc) · 5.35 KB
/
template.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
AWSTemplateFormatVersion: '2010-09-09'
Transform: AWS::Serverless-2016-10-31
Description: >
twitter-backup
Backup your twitter account on a daily basis
Globals:
Function:
MemorySize: 256
Runtime: python3.9
Tags:
'lumigo:auto-trace': 'true'
Architectures:
- arm64
Environment:
Variables:
POWERTOOLS_SERVICE_NAME: backup-twitter-service
Parameters:
TwitterBearerParameterName:
Type: String
Default: TwitterBearerToken
TwitterBearerToken:
Type: String
Default: 'bla bla please replace me'
TwitterAccountsValues:
Type: String
Default: TServerless,heitor_lessa,shridharpandey
Resources:
ScheduleBackupFunction:
Type: AWS::Serverless::Function
Properties:
Timeout: 60
CodeUri: schedule_backup/
Handler: app.lambda_handler
Layers:
- !Ref SharedLibsLayer
- 'arn:aws:lambda:us-east-1:177933569100:layer:AWS-Parameters-and-Secrets-Lambda-Extension-Arm64:2'
Policies:
- SQSSendMessagePolicy:
QueueName: !GetAtt BackupJobQueue.QueueName
- SSMParameterReadPolicy:
ParameterName: !Ref TwitterAccountsParameter
Environment:
Variables:
TWITTER_ACCOUNTS_PARAMETER_NAME: !Ref TwitterAccountsParameter
BACKUP_JOB_QUEUE_NAME: !Ref BackupJobQueue
Events:
DailyScheduleBackup:
Type: Schedule
Properties:
Schedule: cron(0 10 * * ? *)
PullTwitterStream:
Type: AWS::Serverless::Function
Properties:
Timeout: 60
CodeUri: pull_twitter_stream/
Handler: app.lambda_handler
Environment:
Variables:
TWITTER_FH_NAME: !Ref TweetsFirehose
TWITTER_BEARER_PARAM_NAME: !Ref TwitterBearerParameterName
Layers:
- !Ref SharedLibsLayer
- 'arn:aws:lambda:us-east-1:177933569100:layer:AWS-Parameters-and-Secrets-Lambda-Extension-Arm64:2'
Policies:
- SQSPollerPolicy:
QueueName: !GetAtt BackupJobQueue.QueueName
- FirehoseWritePolicy:
DeliveryStreamName: !Ref TweetsFirehose
- AWSSecretsManagerGetSecretValuePolicy:
SecretArn: !Ref TwitterBearerTokenSecret
- KMSDecryptPolicy:
KeyId: '*'
Events:
Batch:
Type: SQS
Properties:
Queue: !GetAtt BackupJobQueue.Arn
FunctionResponseTypes:
- ReportBatchItemFailures
SharedLibsLayer:
Type: AWS::Serverless::LayerVersion
Properties:
ContentUri: app_layer
CompatibleRuntimes:
- python3.9
Metadata:
BuildMethod: python3.9
BackupJobDLQ:
Type: AWS::SQS::Queue
BackupJobQueue:
Type: AWS::SQS::Queue
Properties:
VisibilityTimeout: 180
RedrivePolicy:
maxReceiveCount: 2
deadLetterTargetArn: !GetAtt BackupJobDLQ.Arn
TweetsBucket:
Type: AWS::S3::Bucket
FHToS3Policy:
Type: AWS::IAM::Role
Properties:
AssumeRolePolicyDocument:
Version: "2012-10-17"
Statement:
- Effect: Allow
Principal:
Service:
- firehose.amazonaws.com
Action:
- 'sts:AssumeRole'
Policies:
- PolicyName: access-s3
PolicyDocument:
Version: "2012-10-17"
Statement:
- Effect: Allow
Action:
- "s3:AbortMultipartUpload"
- "s3:GetBucketLocation"
- "s3:GetObject"
- "s3:ListBucket"
- "s3:ListBucketMultipartUploads"
- "s3:PutObject"
Resource:
- !GetAtt TweetsBucket.Arn
- !Sub '${TweetsBucket.Arn}/*'
TweetsFirehose:
Type: AWS::KinesisFirehose::DeliveryStream
Properties:
ExtendedS3DestinationConfiguration:
BucketARN: !GetAtt TweetsBucket.Arn
BufferingHints:
IntervalInSeconds: 60
ErrorOutputPrefix: error/result=!{firehose:error-output-type}/!{timestamp:yyyy/MM/dd}
Prefix: TWEET_HANDLE=!{partitionKeyFromQuery:TWEET_HANDLE}/YYYY=!{timestamp:yyyy}/MM=!{timestamp:MM}/!{timestamp:dd}-
RoleARN: !GetAtt FHToS3Policy.Arn
CompressionFormat: GZIP
DynamicPartitioningConfiguration:
Enabled: true
RetryOptions:
DurationInSeconds: 300
ProcessingConfiguration:
Enabled: true
Processors:
- Type: MetadataExtraction
Parameters:
- ParameterName: MetadataExtractionQuery
ParameterValue: '{TWEET_HANDLE : .twitter_account}'
- ParameterName: JsonParsingEngine
ParameterValue: JQ-1.6
- Type: AppendDelimiterToRecord
Parameters:
- ParameterName: Delimiter
ParameterValue: "\\n"
TwitterAccountsParameter:
Type: AWS::SSM::Parameter
Properties:
Name: TwitterAccounts
Description: Twitter accounts to backup on a daily basis.
Type: StringList
Value: !Ref TwitterAccountsValues
TwitterBearerTokenSecret:
Type: AWS::SecretsManager::Secret
Properties:
Name: !Ref TwitterBearerParameterName
Description: Contains oauth bearer token
SecretString: !Ref TwitterBearerToken