Eg: dagger
Let's see how to configure log groups retention if they already exist or create them otherwise.
AWS Glue ETL jobs of type "pythonshell" can only use the default log groups (created implicitly):
/aws-glue/python-jobs/output
/aws-glue/python-jobs/error
By default, these log groups are created without retention configuration.
main.tf
# For checking if Glue pythonshell log groups exist
data "aws_cloudwatch_log_groups" "glue_existing" {
log_group_name_prefix = "/aws-glue/python-jobs/"
}
locals {
glue_existing_log_groups = toset([
for lg in data.aws_cloudwatch_log_groups.glue_existing.log_group_names : lg
if contains(["/aws-glue/python-jobs/output", "/aws-glue/python-jobs/error"], lg)
])
}
# Glue: Import existing log groups.
# Import blocks are only allowed in the root module.
import {
for_each = local.glue_existing_log_groups
to = aws_cloudwatch_log_group.python_jobs[each.value]
id = each.value
}
# Manage log groups (create if not exist, update retention if exist)
resource "aws_cloudwatch_log_group" "python_jobs" {
for_each = toset(["/aws-glue/python-jobs/output", "/aws-glue/python-jobs/error"])
name = each.value
retention_in_days = local.cloudwatch_retention_in_days
}