-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathvariables.tf
381 lines (321 loc) · 9.53 KB
/
variables.tf
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
variable "glue_database_name" {
type = string
default = ""
description = "Glue database name (This is service database managed by TF databases, used by Kinesis to convert files into Parquet)"
}
variable "glue_location_uri" {
type = string
default = ""
description = "Glue location uri - S3 path for the glue service table"
}
variable "glue_table_name" {
type = string
default = ""
description = "Glue table name - service table name"
}
variable "glue_table_description" {
type = string
default = ""
description = "Glue table description"
}
variable "parquet_schema" {
type = list(any)
description = "Parquet schema, can be different that why type is not so straight"
default = []
}
variable "kinesis_s3_prefix" {
default = ""
type = string
description = "Kinesis s3 prefix - s3 location where the files will be output"
}
variable "kinesis_delivery_stream_name" {
type = string
default = ""
description = "Kinesis delivery stream name"
}
variable "warehouse_bucket_arn" {
type = string
description = "Warehouse bucket arn"
}
variable "warehouse_bucket_name" {
type = string
description = "Warehouse bucket name"
}
variable "kinesis_s3_errors_prefix" {
type = string
default = ""
description = "Kinesiss3 errors prefix - s3 location where the files will be output"
}
variable "lambda_function_name" {
type = string
description = "Lambda function name"
}
variable "events_lambda_function_name" {
type = string
default = "events_lambda"
description = "Events Lambda function name"
}
variable "lambda_description" {
type = string
description = "Lambda description"
default = "Oxbow lambda for converting parquet files to delta tables"
}
variable "lambda_timeout" {
type = number
description = "Lambda timeout"
default = 120
}
variable "lambda_s3_key" {
type = string
description = "Lambda s3 key - lambda path on S3 and file name filename"
}
variable "lambda_s3_bucket" {
type = string
description = "Lambda s3 bucket where lambda is stored"
}
variable "auto_tagging_s3_key" {
type = string
description = "Lambda s3 key - auto tagging lambda path on S3 and file name filename"
default = ""
}
variable "auto_tagging_s3_bucket" {
type = string
description = "s3 bucket where auto tagging lambda is stored"
default = ""
}
variable "lambda_memory_size" {
type = number
default = 128
description = "Lambda memory size"
}
variable "events_lambda_s3_key" {
default = "events_lambda"
type = string
description = "Events Lambda s3 key - lambda path on S3 and file name filename"
}
variable "events_lambda_s3_bucket" {
type = string
default = "events_lambda"
description = "Events Lambda s3 bucket where lambda is stored"
}
variable "lambda_permissions_policy_name" {
type = string
description = ""
}
variable "lambda_reserved_concurrent_executions" {
type = number
description = "Lambda reserved concurrent executions"
default = 1
}
variable "kinesis_policy_name" {
type = string
default = ""
description = "Kinesis policy name"
}
variable "kinesis_policy_description" {
type = string
description = "Kinesis policy description"
default = ""
}
variable "rust_log_deltalake_debug_level" {
type = string
description = "Rust log deltalake debug level"
}
variable "rust_log_oxbow_debug_level" {
type = string
description = "Rust log oxbow debug level"
}
variable "aws_s3_locking_provider" {
type = string
description = "Aws s3 locking provider"
}
variable "dynamodb_table_name" {
type = string
default = ""
description = "Dynamodb table name"
}
variable "logstore_dynamodb_table_name" {
type = string
default = ""
description = "Dynamodb table name"
}
variable "sqs_queue_name" {
type = string
description = "Sqs queue name"
}
variable "sqs_visibility_timeout_seconds" {
type = number
default = 120
description = "Sqs visibility timeout seconds"
}
variable "sqs_delay_seconds" {
type = number
default = 180
description = "Sqs delivery delay seconds"
}
variable "sqs_redrive_policy_maxReceiveCount" {
type = number
default = 10
description = "Sqs maxReceiveCount"
}
variable "sqs_fifo_queue_name" {
type = string
default = "this.fifo"
description = "Sqs FIFO queue name"
}
variable "sqs_fifo_DL_queue_name" {
type = string
default = "this.fifoDL"
description = "Sqs DL FIFO queue name"
}
variable "sqs_group_queue_name" {
type = string
default = "this.group"
description = "Sqs group queue name"
}
variable "sqs_group_DL_queue_name" {
type = string
default = "this.group"
description = "Sqs group queue name"
}
variable "sqs_queue_name_dl" {
type = string
description = "Sqs queue name - dead letters"
}
variable "lambda_kinesis_role_name" {
type = string
description = "Lambda kinesis IAM role name"
}
variable "tags" {
type = map(string)
default = {}
description = "A mapping of tags to assign to the resource"
}
variable "s3_path" {
type = string
description = "s3_path - where the files are going to be stored"
}
# monitoring
variable "enabled_dead_letters_monitoring" {
type = bool
description = "Enable monitoring"
default = false
}
variable "dl_alert_recipients" {
type = list(string)
default = []
description = "List of the alert recipients"
}
variable "dl_alert_message" {
type = string
default = ""
}
variable "dl_warning" {
type = any
default = ""
description = "Dead letters warning value"
}
variable "dl_critical" {
type = any
default = ""
description = "Dead letters critical value"
}
variable "dl_ok" {
type = any
default = ""
description = "Dead letters critical value"
}
variable "tags_monitoring" {
type = list(string)
default = []
description = "A list of tags to assign to the monitoring resource"
}
variable "enable_aws_glue_catalog_table" {
type = bool
description = "Enable glue catalog table"
default = false
}
variable "enable_kinesis_firehose_delivery_stream" {
type = bool
description = "Enable firehose delivery stream"
default = false
}
variable "enable_bucket_notification" {
type = bool
description = "Enable enable_bucket_notification"
default = false
}
variable "enable_group_events" {
type = bool
description = "Enable group events"
default = false
}
variable "enable_auto_tagging" {
type = bool
description = "Whether to turn on Auto Tagging Lambda"
default = false
}
variable "sns_topic_arn" {
type = string
description = "Optional arn to enable the SNS subscription and ENV var for Oxbow"
default = ""
}
variable "enable_glue_create" {
type = bool
description = "Whether to turn on Glue create Lambda"
default = false
}
variable "enable_schema_evolution" {
type = bool
description = "Whether to turn on schema evolution"
default = true
}
variable "glue_create_config" {
type = object({
athena_workgroup_name = string // Name of AWS Athena workgroup
athena_data_source = string // Arn name of AWS Athena data source (catalog)
athena_bucket_name = string // name of AWS Athena bucket.
lambda_s3_key = string // lambda s3 key - lambda path on S3 and file name filename
lambda_s3_bucket = string // lambda s3 bucket where lambda is stored
lambda_function_name = string // lambda function name
path_regex = string // regexp for mapping s3 path to database/table
sns_topic_arn = string // sns topic arn with s3 events (source for lambda)
sqs_queue_name = string // name of sqs queue for glue-sync lambda
sqs_queue_name_dl = string // name dead letter sqs que with not processed s3 events
iam_role_name = string // lambda role name
iam_policy_name = string // lambda policy name
})
description = "Configuration of glue-create lambda"
}
variable "enable_glue_sync" {
type = bool
description = "Whether to turn on Glue create Lambda"
default = false
}
variable "glue_sync_config" {
type = object({
lambda_s3_key = string // lambda s3 key - lambda path on S3 and file name filename
lambda_s3_bucket = string // lambda s3 bucket where lambda is stored
lambda_function_name = string // lambda function name
path_regex = string // regexp for mapping s3 path to database/table
sns_topic_arn = string // sns topic arn with s3 events (source for lambda)
sqs_queue_name = string // name of sqs queue for glue-sync lambda
sqs_queue_name_dl = string // name dead letter sqs que with not processed s3 events
iam_role_name = string // lambda role name
iam_policy_name = string // lambda policy name
})
description = "Configuration of glue-sync lambda"
}
variable "group_event_lambda_batch_size" {
description = "describe your variable"
default = 10
}
variable "group_event_lambda_maximum_batching_window_in_seconds" {
description = "describe your variable"
default = 1
}
variable "architectures" {
description = "Instruction set architecture for your Lambda function. Valid values are [\"x86_64\"] and [\"arm64\"]."
type = list(string)
default = ["x86_64"]
}